Merge ~hloeung/influxdb-charm:master into influxdb-charm:master

[Haw Loeung]

Allow overriding configs with extra_config

There's a lot of configuration options to tune and set, instead of
implementing all of them as charm config options, we use
extra_config. This is basically section.config=value. For example, to
change/set the below:

| [http]
| auth-enabled = true

We can use this:

| extra_config:
| - http.auth-enabled=true

[Haw Loeung (hloeung)]

$ juju config influxdb ip_address='127.0.0.1'

| ubuntu@juju-b884ff-canonistack-haw-lcy02-4:~$ diff -Naurp /etc/influxdb/{influxdb.conf.orig,influxdb.conf}
| --- /etc/influxdb/influxdb.conf.orig 2017-11-03 00:10:10.743762811 +0000
| +++ /etc/influxdb/influxdb.conf 2017-11-03 00:22:05.160094893 +0000
| @@ -201,7 +201,7 @@
| # enabled = true
|
| # The bind address used by the HTTP service.
| - # bind-address = ":8086"
| + bind-address = "127.0.0.1:8086"
|
| # Determines whether user authentication is enabled over HTTP/HTTPS.
| # auth-enabled = false
| ===

$ juju config influxdb ip_address='127.0.0.1' bind_port='8087'

| ubuntu@juju-b884ff-canonistack-haw-lcy02-4:~$ diff -Naurp /etc/influxdb/{influxdb.conf.orig,influxdb.conf}
| --- /etc/influxdb/influxdb.conf.orig 2017-11-03 00:10:10.743762811 +0000
| +++ /etc/influxdb/influxdb.conf 2017-11-03 00:24:00.227707065 +0000
| @@ -201,7 +201,7 @@
| # enabled = true
|
| # The bind address used by the HTTP service.
| - # bind-address = ":8086"
| + bind-address = "127.0.0.1:8087"
|
| # Determines whether user authentication is enabled over HTTP/HTTPS.
| # auth-enabled = false

$ juju config influxdb extra_config="- global.hostname=myhost
- http.auth-enabled=true
- data.max-series-per-database=0
"

| ubuntu@juju-b884ff-canonistack-haw-lcy02-4:~$ diff -Naurp /etc/influxdb/{influxdb.conf.orig,influxdb.conf}
| --- /etc/influxdb/influxdb.conf.orig 2017-11-03 00:10:10.743762811 +0000
| +++ /etc/influxdb/influxdb.conf 2017-11-03 00:25:45.242181284 +0000
| @@ -14,6 +14,8 @@
| # Bind address to use for the RPC service for backup and restore.
| # bind-address = "127.0.0.1:8088"
|
| +hostname = "myhost"
| +
| ###
| ### [meta]
| ###
| @@ -95,7 +97,7 @@
| # The maximum series allowed per database before writes are dropped. This limit can prevent
| # high cardinality issues at the database level. This limit can be disabled by setting it to
| # 0.
| - # max-series-per-database = 1000000
| + max-series-per-database = 0
|
| # The maximum number of tag values per tag that are allowed before writes are dropped. This limit
| # can prevent high cardinality tag values from being written to a measurement. This limit can be
| @@ -204,7 +206,7 @@
| # bind-address = ":8086"
|
| # Determines whether user authentication is enabled over HTTP/HTTPS.
| - # auth-enabled = false
| + auth-enabled = true
|
| # The default realm sent back when issuing a basic auth challenge.
| # realm = "InfluxDB"

[Paul Collins (pjdc)]

review: Approve (lgtm)

diff --git a/Makefile b/Makefile
index 5dd0577..efec4a7 100644
--- a/Makefile
+++ b/Makefile
@@ -12,7 +12,7 @@ clean:
 	rm -f $(REACTIVE_DIR)/*.pyc
 
 lint:
-	@flake8 $(REACTIVE_DIR)
+	@flake8 --max-line-length=120 $(REACTIVE_DIR)
 
 git:
 	git push $(PERSONAL_REPO)
diff --git a/config.yaml b/config.yaml
index 917e93a..840eb7b 100644
--- a/config.yaml
+++ b/config.yaml
@@ -62,7 +62,7 @@ options:
   ip_address:
     description: "Which IP to bind on"
     type: string
-    default: "0.0.0.0"
+    default: ""
   bind_port:
     description: "Which port to bind to"
     type: int
@@ -80,4 +80,13 @@ options:
     default: 1000000
     description: |
         Maximum series allowed per database before writes are dropped.
-        Specify 0 for no limit.
+        Specify 0 for no limit. (DEPRECATED) Please use extra_config instead.
+  extra_config:
+    type: string
+    default: ""
+    description: |
+        YAML-formatted list of extra configuration options to override.
+        For example:
+          - global.hostname=myhost
+          - http.auth-enabled=true
+          - data.max-series-per-database=0
diff --git a/templates/influxdb.conf b/files/influxdb.conf
index 77ed982..23c4e03 100644
--- a/templates/influxdb.conf
+++ b/files/influxdb.conf
@@ -1,8 +1,8 @@
 ### Welcome to the InfluxDB configuration file.
 
 # The values in this file override the default values used by the system if
-# a config option is not specified.  The commented out lines are the the configuration
-# field and the default value used.  Uncommentting a line and changing the value
+# a config option is not specified. The commented out lines are the configuration
+# field and the default value used. Uncommenting a line and changing the value
 # will change the value used at runtime when the process is restarted.
 
 # Once every 24 hours InfluxDB will report usage data to usage.influxdata.com
@@ -11,10 +11,8 @@
 # Change this option to true to disable reporting.
 # reporting-disabled = false
 
-# we'll try to get the hostname automatically, but if it the os returns something
-# that isn't resolvable by other servers in the cluster, use this option to
-# manually set the hostname
-# hostname = "localhost"
+# Bind address to use for the RPC service for backup and restore.
+# bind-address = "127.0.0.1:8088"
 
 ###
 ### [meta]
@@ -49,6 +47,18 @@
   # The directory where the TSM storage engine stores WAL files.
   wal-dir = "/var/lib/influxdb/wal"
 
+  # The amount of time that a write will wait before fsyncing.  A duration
+  # greater than 0 can be used to batch up multiple fsync calls.  This is useful for slower
+  # disks or when WAL write contention is seen.  A value of 0s fsyncs every write to the WAL.
+  # Values in the range of 0-100ms are recommended for non-SSD disks.
+  # wal-fsync-delay = "0s"
+
+
+  # The type of shard index to use for new shards.  The default is an in-memory index that is
+  # recreated at startup.  A value of "tsi1" will use a disk based index that supports higher
+  # cardinality datasets.
+  # index-version = "inmem"
+
   # Trace logging provides more verbose output around the tsm engine. Turning
   # this on can provide more useful output for debugging tsm engine issues.
   # trace-logging-enabled = false
@@ -77,10 +87,15 @@
   # write or delete
   # compact-full-write-cold-duration = "4h"
 
+  # The maximum number of concurrent full and level compactions that can run at one time.  A
+  # value of 0 results in runtime.GOMAXPROCS(0) used at runtime.  This setting does not apply
+  # to cache snapshotting.
+  # max-concurrent-compactions = 0
+
   # The maximum series allowed per database before writes are dropped.  This limit can prevent
   # high cardinality issues at the database level.  This limit can be disabled by setting it to
   # 0.
-  max-series-per-database = {{max_series_per_database}}
+  # max-series-per-database = 1000000
 
   # The maximum number of tag values per tag that are allowed before writes are dropped.  This limit
   # can prevent high cardinality tag values from being written to a measurement.  This limit can be
@@ -106,22 +121,20 @@
   # can help prevent run away queries.  Setting the value to 0 disables the limit.
   # query-timeout = "0s"
 
-  # The the time threshold when a query will be logged as a slow query.  This limit can be set to help
+  # The time threshold when a query will be logged as a slow query.  This limit can be set to help
   # discover slow or resource intensive queries.  Setting the value to 0 disables the slow query logging.
   # log-queries-after = "0s"
 
-  # The maximum number of points a SELECT can process.  A value of 0 will make the maximum
-  # point count unlimited.
+  # The maximum number of points a SELECT can process.  A value of 0 will make
+  # the maximum point count unlimited.  This will only be checked every 10 seconds so queries will not
+  # be aborted immediately when hitting the limit.
   # max-select-point = 0
 
   # The maximum number of series a SELECT can run.  A value of 0 will make the maximum series
   # count unlimited.
-
-  # The maximum number of series a SELECT can run. A value of zero will make the maximum series
-  # count unlimited.
   # max-select-series = 0
 
-  # The maxium number of group by time bucket a SELECt can create.  A value of zero will max the maximum
+  # The maxium number of group by time bucket a SELECT can create.  A value of zero will max the maximum
   # number of buckets unlimited.
   # max-select-buckets = 0
 
@@ -132,7 +145,7 @@
 ###
 
 [retention]
-  # Determines whether retention policy enforcment enabled.
+  # Determines whether retention policy enforcement enabled.
   # enabled = true
 
   # The interval of time when retention policy enforcement checks run.
@@ -177,27 +190,6 @@
   # store-interval = "10s"
 
 ###
-### [admin]
-###
-### Controls the availability of the built-in, web-based admin interface. If HTTPS is
-### enabled for the admin interface, HTTPS must also be enabled on the [http] service.
-###
-### NOTE: This interface is deprecated as of 1.1.0 and will be removed in a future release.
-
-[admin]
-  # Determines whether the admin service is enabled.
-  enabled = {{ admin }}
-
-  # The default bind address used by the admin service.
-  # bind-address = ":8083"
-
-  # Whether the admin service should use HTTPS.
-  # https-enabled = false
-
-  # The SSL certificate used when HTTPS is enabled.
-  # https-certificate = "/etc/ssl/influxdb.pem"
-
-###
 ### [http]
 ###
 ### Controls how the HTTP endpoints are configured. These are the primary
@@ -209,15 +201,15 @@
   # enabled = true
 
   # The bind address used by the HTTP service.
-  bind-address = "{{ip_address}}:{{bind_port}}"
+  # bind-address = ":8086"
 
-  # Determines whether HTTP authentication is enabled.
+  # Determines whether user authentication is enabled over HTTP/HTTPS.
   # auth-enabled = false
 
   # The default realm sent back when issuing a basic auth challenge.
   # realm = "InfluxDB"
 
-  # Determines whether HTTP request logging is enable.d
+  # Determines whether HTTP request logging is enabled.
   # log-enabled = true
 
   # Determines whether detailed write logging is enabled.
@@ -429,7 +421,7 @@
 ###
 
 [continuous_queries]
-  # Determiens whether the continuous query service is enabled.
+  # Determines whether the continuous query service is enabled.
   # enabled = true
 
   # Controls whether queries are logged when executed by the CQ service.
diff --git a/reactive/influxdb.py b/reactive/influxdb.py
index 856c9bb..a8394de 100644
--- a/reactive/influxdb.py
+++ b/reactive/influxdb.py
@@ -1,11 +1,13 @@
 from charmhelpers.contrib.charmsupport import nrpe
-from charmhelpers.core.host import service_start, service_stop
 from charmhelpers.core import hookenv
+from charmhelpers.core import host
 from charmhelpers.core.templating import render
-from charms.reactive import hook, set_state, when, when_any, when_not
+from charms.reactive import hook, set_state, remove_state, when, when_all, when_not
 
+import re
 import os
 import shutil
+import yaml
 
 CRONFILE = '/etc/cron.daily/influxdb-charm-backup'
 CHECK_SRC = 'files/check_influxdb.py'
@@ -28,29 +30,97 @@ def install_influx():
     hookenv.open_port(8083)
     hookenv.open_port(port)
     config_changed()
-    service_start('influxdb')
+    host.service_start('influxdb')
     hookenv.status_set('active', '')
     set_state('influxdb.configured')
-
-
-@when_any('config.changed.bind_port', 'config.changed.ip_address', 'config.changed.max_series_per_database')
+    set_state('config.changed')
+
+
+def apply_configs(fh, config):
+    new_config = []
+
+    section = 'global'
+    indent = ''
+
+    section_keys_found = {
+        'global': {},
+    }
+    initial_comments = True
+    for line in fh:
+        # Start after initial comments "Welcome to the InfluxDB configur..."
+        if initial_comments and not line.startswith('##'):
+            initial_comments = False
+        if line.startswith('[') or (not initial_comments and line.startswith('##')):
+            if section in config:
+                # If the configs weren't found in this section let's add them.
+                for key in config[section].keys():
+                    if key in section_keys_found[section]:
+                        break
+                    section_keys_found[section][key] = ''
+                    new_config.append("{}{} = {}\n\n".format(indent, key, config[section][key]))
+            new_config.append(line)
+            if line.startswith('['):
+                section = line[1:][:-2]
+                # Apart from global config options, indent the rest.
+                indent = '  '
+            continue
+        # Format is 'key = value' or '# key = value'
+        match = re.match('^([\s#]*)(\S+)(\s*=.*)', line)
+        if not match:
+            new_config.append(line)
+            continue
+        key = match.group(2)
+        value = match.group(3)
+        # Store all config options we found per section.
+        if section not in section_keys_found:
+            section_keys_found[section] = {}
+        section_keys_found[section][key] = ''
+        if section not in config or key not in config[section]:
+            # We want to keep {meta,data}.{dir,wal-dir} always.
+            if section in ['meta', 'data'] and key in ['dir', 'wal-dir']:
+                new_config.append(line)
+            else:
+                new_config.append("{}# {}{}\n".format(indent, key, value))
+            continue
+        new_config.append("{}{} = {}\n".format(indent, key, config[section][key]))
+
+    return ''.join(new_config)
+
+
+@when_all('influxdb.configured', 'config.changed')
 def config_changed():
     config = hookenv.config()
-    port = config.get('bind_port')
-    ip_address = config.get('ip_address')
-    max_series_per_database = config.get('max_series_per_database')
-    render(source='influxdb.conf',
-           target='/etc/influxdb/influxdb.conf',
-           owner='root',
-           perms=0o644,
-           context={
-               'admin': 'true',
-               'bind_port': port,
-               'ip_address': ip_address,
-               'max_series_per_database': max_series_per_database,
-           })
-    service_stop('influxdb')
-    service_start('influxdb')
+    conf = {
+        'global': {},
+    }
+
+    # Only write out the bind-address if it differs from the default.
+    if config.get('ip_address') or config.get('bind_port') != 8086:
+        conf['http'] = {}
+        conf['http']['bind-address'] = '"{}:{}"'.format(config.get('ip_address', ''),
+                                                        config.get('bind_port'))
+
+    # Construct our dict of configs from the extra_config charm option.
+    if config.get('extra_config', None):
+        for c in yaml.safe_load(config.get('extra_config', '[]')):
+            k, value = c.split('=', 1)
+            section, key = k.split('.', 1)
+            if section not in conf:
+                conf[section] = {}
+            if str(value) in ('true', 'false') or value.isdigit():
+                conf[section][key] = value
+            else:
+                conf[section][key] = '"{}"'.format(value)
+
+    target = '/etc/influxdb/influxdb.conf'
+    if not os.path.exists(target):
+        shutil.copy2('files/influxdb.conf', target)
+    with open(target, 'r') as f:
+        updated_config = apply_configs(f, conf)
+    host.write_file(target, updated_config.encode("utf-8"), perms=0o644)
+
+    host.service_stop('influxdb')
+    host.service_start('influxdb')
 
 
 @when('config.changed.backup_dir')
@@ -77,10 +147,8 @@ def create_backup_job():
 
 @hook('upgrade-charm')
 def upgrade_charm():
-    # Force config changed on upgrade-charm in case one of the templates changed.
+    remove_state('influxdb.configured')
     set_state('config.changed.backup_dir')
-    set_state('config.changed.bind_port')
-    set_state('config.changed.ip_address')
     # If we have an NRPE relation, force run of that as well
     if hookenv.relation_ids('nrpe-exteral-master'):
         set_state('nrpe-external-master.available')