Merge lp:~verterok/charms/trusty/telegraf/proper-templates into lp:~ubuntuone-hackers/charms/trusty/telegraf/trunk

[Guillermo Gonzalez]

Support telegraf updates, use jinja templates for plugin templates and add extra_options config to be able to fine tune each plugin

- support telegraf updates
- use jinja templates for plugin templates
- add extra_options config, to be able to fine tune each plugin

[Bret Barker (noise)]

wording nitpick below, otherwise good job!

review: Approve

[lp:~verterok/charms/trusty/telegraf/proper-templates]

17. By Guillermo Gonzalez on 2016-03-30

fix README wording

=== modified file 'README.md'
--- README.md	2016-02-01 15:13:24 +0000
+++ README.md	2016-03-30 20:53:04 +0000
@@ -24,6 +24,29 @@
 
 The only output plugin supported via relation is influxdb, any other output plugin needs to be configured manually (via juju set)
 
+To configure any of the (default or via relation) plugins, the extra_options charm config can be used. It's string in yaml format, for example: 
+
+    inputs:
+        cpu:
+            percpu: 'false'
+            fielddrop: '["time_*"]'
+        disk:
+            mount_points: '["/"]'
+            ignore_fs: '["tmpfs", "devtmpfs"]'
+        elasticsearch:
+            local: 'false' 
+            cluster_health: 'true'
+        postgresql:
+            databases: '["foo", "bar"]'
+            tagpass: 
+                db: '["template", "postgres"]'
+    outputs:
+        influxdb:
+            precision: "ms" 
+
+This extra options will only be applied to plugins defined in templates/base_inputs.conf and any other plugins configured via relations.
+
+
 To use a different metrics storage, e.g: graphite. the plugin configuration needs to be set as a base64 string in outputs_config configuration.
 
 For exmaple, save the following config to a file: 

=== modified file 'config.yaml'
--- config.yaml	2016-02-25 18:36:59 +0000
+++ config.yaml	2016-03-30 20:53:04 +0000
@@ -66,7 +66,7 @@
     default: ""
     description: "[outputs.xxx] sections as base64 string"
   package_name:
-    default: "telegraf_0.10.1-1_amd64.deb"
+    default: "telegraf_0.10.1-1_amd64.deb" 
     type: string
     description: |
         Filename of telegraf deb package.  If this matches the
@@ -81,4 +81,28 @@
     default: ""
     type: string
     description: "GPG key for apt_repository"
+  extra_options:
+    default: ""
+    type: string 
+    description: |
+        YAML with extra options for out|inputs managed by relations or in the default config.
+        Values must be strings.
+        example:
+          inputs:
+              cpu:
+                  percpu: 'false'
+                  fielddrop: '["time_*"]'
+              disk:
+                  mount_points: '["/"]'
+                  ignore_fs: '["tmpfs", "devtmpfs"]'
+              elasticsearch:
+                  local: 'false' 
+                  cluster_health: 'true'
+              postgresql:
+                  databases: '["foo", "bar"]'
+                  tagpass: 
+                      db: '["template", "postgres"]'
+          outputs:
+              influxdb:
+                  precision: "ms" 
 

=== modified file 'hooks/actions.py'
--- hooks/actions.py	2016-03-11 15:36:40 +0000
+++ hooks/actions.py	2016-03-30 20:53:04 +0000
@@ -4,11 +4,15 @@
 import subprocess
 import sys
 
+import yaml
+
 from charmhelpers.core import hookenv, host
 from charmhelpers.core.templating import render
 from charmhelpers.core.services.base import ManagerCallback
 from charmhelpers.fetch import apt_install, apt_update, add_source
 
+from jinja2 import Template
+
 
 CONFIG_FILE = '/etc/telegraf/telegraf.conf'
 
@@ -32,6 +36,8 @@
         return old
 
     def __call__(self, manager, service_name, event_name):
+        if hookenv.hook_name() in ('update-status',):
+            return
         new_hash = host.file_hash(CONFIG_FILE)
         old_hash = self._update_persisted_data('config_hash', new_hash)
         if new_hash != old_hash:
@@ -59,7 +65,9 @@
         pkg_file = os.path.join(hookenv.charm_dir(), "files",
                                 config['package_name'])
         if os.path.exists(pkg_file):
-            if subprocess.call(["dpkg", "-i", pkg_file]) != 0:
+            env = os.environ.copy()
+            env['DEBIAN_FRONTEND'] = 'noninteractive'
+            if subprocess.call(["dpkg", "-i", "--force-confold", pkg_file], env=env) != 0:
                 hookenv.log("CHARM: Error installing telegraf package")
                 sys.exit(1)
         else:
@@ -70,12 +78,22 @@
             apt_install(config['package_name'], options=['--force-yes'])
 
 
+def update_telegraf(service_name):
+    config = hookenv.config()
+    if hookenv.hook_name() == 'config-changed' \
+            and config.changed('package_name'):
+        install()
+    elif hookenv.hook_name() == 'upgrade-charm':
+        install()
+
+
 def log_start(service_name):
     hookenv.log('telegraf starting')
 
 
 def render_config(service_name):
     config = hookenv.config()
+    extra_options = get_extra_options()
     context = config.copy()
     inputs = base64.b64decode(config['inputs_config'])
     outputs = base64.b64decode(config['outputs_config'])
@@ -90,7 +108,9 @@
     else:
         # use base inputs from charm templates
         with open('templates/base_inputs.conf', 'r') as fd:
-            context["inputs"] = fd.read()
+            context["inputs"] = render_template(
+                fd.read(),
+                {'extra_options': extra_options['inputs']})
     if outputs:
         context["outputs"] = outputs
     else:
@@ -119,12 +139,45 @@
             return rels[0]['__unit__']
 
 
+def get_extra_options():
+    extra_options = {'inputs': {}, 'outpus': {}}
+    extra_options_raw = hookenv.config()['extra_options']
+    extra_opts = yaml.load(extra_options_raw) or {}
+    extra_options.update(extra_opts)
+    return extra_options
+
+
+def render_extra_options(kind, name):
+    template = """
+  {% if extra_options %}
+  {% for key, value in extra_options.items() %}
+  {% if key == 'tagpass' or key == 'tagdrop' %}
+  [{{ kind }}.{{ name }}.{{key}}]
+    {% for tag, tagvalue in value.items() %}
+    {{ tag }} = {{ tagvalue }}
+    {% endfor %}
+  {% else %}
+  {{ key }} = {{ value }}
+  {% endif %}
+  {% endfor %}
+  {% endif %}
+  """
+    extra_options = get_extra_options()
+    context = {"extra_options": extra_options[kind].get(name, {}),
+               "kind": kind,
+               "name": name}
+    return render_template(template, context)
+
+
+def render_template(template, context):
+    tmpl = Template(template, lstrip_blocks=True, trim_blocks=True)
+    return tmpl.render(**context)
+
+
 def elasticsearch_input(service_name):
     template = """
-[elasticsearch]
-  servers = {}
-  local = true
-  cluster_health = true
+[[inputs.elasticsearch]]
+  servers = {{ servers }}
 """
     rels = hookenv.relations_of_type('elasticsearch')
     hosts = []
@@ -138,7 +191,9 @@
         hosts.append("http://{}:{}".format(es_host, port))
     config_path = '{}/{}.conf'.format(CONFIG_DIR, 'elasticsearch')
     if hosts:
-        inputs.append(template.format(json.dumps(hosts)))
+        context = {"servers": json.dumps(hosts)}
+        inputs.append(render_template(template, context) + \
+                      render_extra_options("inputs", "elasticsearch"))
         hookenv.log("Updating {} plugin config file".format('elasticsearch'))
         host.write_file(config_path, '\n'.join(inputs))
     elif os.path.exists(config_path):
@@ -148,7 +203,7 @@
 def mongodb_input(service_name):
     template = """
 [[inputs.mongodb]]
-  servers = {}
+  servers = {{ servers }}
 """
     rels = hookenv.relations_of_type('mongodb')
     mongo_addresses = []
@@ -163,7 +218,10 @@
         mongo_addresses.append(mongo_address)
     config_path = '{}/{}.conf'.format(CONFIG_DIR, 'mongodb')
     if mongo_addresses:
-        inputs.append(template.format(json.dumps(mongo_addresses)))
+        extra_options = get_extra_options()
+        context = {"servers": json.dumps(mongo_addresses)}
+        inputs.append(render_template(template, context) + \
+                      render_extra_options("inputs", "mongodb"))
         hookenv.log("Updating {} plugin config file".format('mongodb'))
         host.write_file(config_path, '\n'.join(inputs))
     elif os.path.exists(config_path):
@@ -173,15 +231,18 @@
 def postgresql_input(service_name):
     template = """
 [[inputs.postgresql]]
-  address = "host={host} user={user} password={password} dbname={database}"
+  address = "host={{host}} user={{user}} password={{password}} dbname={{database}}"
 """
     required_keys = ['host', 'user', 'password', 'database']
     rels = hookenv.relations_of_type('postgresql')
+    extra_options = get_extra_options()
     inputs = []
     for rel in rels:
         if all([rel.get(key) for key in required_keys]) \
                 and hookenv.local_unit() in rel.get('allowed-units'):
-            inputs.append(template.format(**rel))
+            context = rel.copy()
+            inputs.append(render_template(template, context) + \
+                          render_extra_options("inputs", "postgresql"))
     config_path = '{}/{}.conf'.format(CONFIG_DIR, 'postgresql')
     if inputs:
         hookenv.log("Updating {} plugin config file".format('postgresql'))
@@ -193,7 +254,7 @@
 def haproxy_input(service_name):
     template = """
 [[inputs.haproxy]]
-  servers = {}
+  servers = {{ servers }}
 """
     rels = hookenv.relations_of_type('haproxy')
     haproxy_addresses = []
@@ -220,7 +281,8 @@
         haproxy_addresses.append(haproxy_address)
     config_path = '{}/{}.conf'.format(CONFIG_DIR, 'haproxy')
     if haproxy_addresses:
-        inputs.append(template.format(json.dumps(haproxy_addresses)))
+        inputs.append(render_template(template, {"servers": json.dumps(haproxy_addresses)}) + \
+                      render_extra_options("inputs", "haproxy"))
         hookenv.log("Updating {} plugin config file".format('haproxy'))
         host.write_file(config_path, '\n'.join(inputs))
     elif os.path.exists(config_path):

=== modified file 'hooks/services.py'
--- hooks/services.py	2016-03-03 19:44:28 +0000
+++ hooks/services.py	2016-03-30 20:53:04 +0000
@@ -15,6 +15,7 @@
             'required_data': [
             ],
             'data_ready': [
+                actions.update_telegraf,
                 actions.render_config,
                 actions.elasticsearch_input,
                 actions.mongodb_input,

=== modified file 'templates/base_inputs.conf'
--- templates/base_inputs.conf	2016-02-25 21:08:33 +0000
+++ templates/base_inputs.conf	2016-03-30 20:53:04 +0000
@@ -1,17 +1,37 @@
+{% macro render_options(name, options) %}
+  {% if options[name] %}
+  {% for key, value in options[name].items() %}
+  {% if key == 'tagpass' or key == 'tagdrop' %}
+  [inputs.{{ name }}.{{key}}]
+    {% for tag, tagvalue in value.items() %}
+    {{ tag }} = {{ tagvalue }}
+    {% endfor %}
+  {% else %}
+  {{ key }} = {{ value }}
+  {% endif %}
+  {% endfor %}
+  {% endif %}
+{% endmacro %}
+
 # Read metrics about cpu usage
 [[inputs.cpu]]
+  {% if extra_options['cpu'] %}
+{{ render_options('cpu', extra_options) }}
+  {% else %}
   # Whether to report per-cpu stats or not
   percpu = true
   # Whether to report total system cpu stats or not
   totalcpu = true
   # Comment this line if you want the raw CPU time metrics
   drop = ["time_*"]
+  {% endif %}
 
 # Read metrics about disk usage by mount point
 [[inputs.disk]]
   # By default, telegraf gather stats for all mountpoints.
   # Setting mountpoints will restrict the stats to the specified mountpoints.
   # mount_points=["/"]
+{{ render_options('disk', extra_options) }}
 
 # Read metrics about disk IO by device
 [[inputs.diskio]]
@@ -21,6 +41,7 @@
   # devices = ["sda", "sdb"]
   # Uncomment the following line if you do not need disk serial numbers.
   # skip_serial_number = true
+{{ render_options('diskio', extra_options) }}
 
 # Read metrics about memory usage
 [[inputs.mem]]
@@ -33,6 +54,7 @@
   # regardless of status.
   #
   # interfaces = ["eth0", ... ]
+{{ render_options('net', extra_options) }}
 
 # Read metrics about TCP status such as established, time wait etc and UDP sockets counts.
 [[inputs.netstat]]