Merge ~jacekn/prometheus-charm:snmp into prometheus-charm:master

[Jacek Nykis]

Add snmp-exporter relation support

Add scrape-jobs config option to allow arbitrary SNMP jobs to be added. This is needed because SNMP config is non trivial. Targets can belong to one of a few different "modules". For each module or group of targets we likely need different scrape interval and timeout options.

We could try to do it like this:
juju set prometheus snmp_targets='["target1[module_name1][30s][30s]", "target2[module_name2][90s][90s]"'

but this can become unmanageable quickly and would require logic to programmatically split targets into separate scrape jobs.

[Jamon Camisso (jamon)]

+1

review: Approve

diff --git a/config.yaml b/config.yaml
index 8a2007b..a8497c0 100644
--- a/config.yaml
+++ b/config.yaml
@@ -37,6 +37,37 @@ options:
            - 'my-production-site:443'
          label: production
     default: ""
+  scrape-jobs:
+    type: string
+    default: ""
+    description: |
+      Can be used to add custom scrape jobs, for example SNMP exporter config.
+      This will be parsed as YAML, format is a list of dicts as shown in the example.
+      For convenience "{{ snmp_exporter }}" will be replaced with SNMP exporter address.
+      Example:
+       - job_name: 'snmp'
+         scrape_interval: 1m
+         scrape_timeout: 1m
+         static_configs:
+           - targets:
+             - device1.example.com
+             - device2.example.com
+         params:
+           module: [default]
+         relabel_configs:
+           - source_labels: [__address__]
+             target_label: __param_target
+           - source_labels: [__param_target]
+             target_label: instance
+           - target_label: __address__
+             replacement: "{{ snmp_exporter }}" # SNMP exporter.
+       - job_name: 'my-job'
+         scrape_interval: 30s
+         scrape_timeout: 30s
+         static_configs:
+           - targets:
+             - server1.example.com
+             - server2.example.com
   monitor-name:
     type: string
     description: |
diff --git a/metadata.yaml b/metadata.yaml
index 2dec74f..1eba152 100644
--- a/metadata.yaml
+++ b/metadata.yaml
@@ -11,8 +11,6 @@ series:
   - xenial
 tags:
   - monitoring
-categories:
-  - misc
 subordinate: false
 provides:
   grafana-source:
@@ -27,6 +25,8 @@ requires:
     interface: prometheus
   alertmanager-service:
     interface: http
+  snmp-exporter:
+    interface: http
 storage:
   metrics-filesystem:
     type: filesystem
diff --git a/reactive/prometheus.py b/reactive/prometheus.py
index 917b575..021c9f4 100644
--- a/reactive/prometheus.py
+++ b/reactive/prometheus.py
@@ -3,6 +3,7 @@ import pwd
 import subprocess
 import yaml
 import re
+from jinja2 import Template
 
 from charmhelpers import fetch
 from charmhelpers.core import host, hookenv, unitdata
@@ -133,6 +134,20 @@ def get_version():
         return (1, 0, 0)
 
 
+def get_manual_jobs():
+    config = hookenv.config()
+    if not config.get('scrape-jobs', False):
+        return None
+    try:
+        yaml.safe_load(config.get('scrape-jobs'))
+    except:
+        return None
+    else:
+        snmp_exporter = unitdata.kv().get('snmp_exporter_address', '127.0.0.1:9116')
+        t = Template(config.get('scrape-jobs'))
+        return t.render(snmp_exporter=snmp_exporter)
+
+
 @when('prometheus.do-reconfig-yml')
 def write_prometheus_config_yml():
     hookenv.status_set('maintenance', 'Updating configuration')
@@ -141,6 +156,7 @@ def write_prometheus_config_yml():
     ver_major, ver_minor, ver_patch = get_version()
     target_jobs = unitdata.kv().get('target_jobs', [])
     scrape_jobs = unitdata.kv().get('scrape_jobs', [])
+    manual_jobs = get_manual_jobs()
 
     # transform eg. 'h1:p1 ,  h2:p2' (string), to ['h1:p1', 'h2:p2'] (list)
     static_targets = None
@@ -162,6 +178,7 @@ def write_prometheus_config_yml():
         'jobs': target_jobs,
         'federate': federate_config,
         'scrape_jobs': scrape_jobs,
+        'manual_jobs': manual_jobs,
         'ver_major': ver_major,
         'ver_minor': ver_minor,
         'ver_patch': ver_patch,
@@ -371,6 +388,28 @@ def update_prometheus_alertmanager(alertmanager):
 
 
 @when('prometheus.started')
+@when_not('snmp-exporter.available')
+def update_prometheus_no_snmp_exporter():
+    unitdata.kv().unset('snmp_exporter_address')
+    set_state('prometheus.do-check-reconfig')
+
+
+@when('prometheus.started')
+@when('snmp-exporter.available')
+def update_prometheus_snmp_exporter(snmp_exporter):
+    services = snmp_exporter.services()
+    if not (data_changed('snmp-exporter.related_services', services)):
+        return
+    # At this point we only support single snmp exporter target
+    for service in services:
+        for unit in service['hosts']:
+            hookenv.log('{} has a unit {}:{}'.format(
+                        service['service_name'], unit['hostname'], unit['port']))
+            unitdata.kv().set('snmp_exporter_address', '{hostname}:{port}'.format(**unit))
+    set_state('prometheus.do-check-reconfig')
+
+
+@when('prometheus.started')
 @when('nrpe-external-master.available')
 def update_nrpe_config(svc):
     # python-dbus is used by check_upstart_job
diff --git a/templates/prometheus.yml.j2 b/templates/prometheus.yml.j2
index d2ba919..0e39110 100644
--- a/templates/prometheus.yml.j2
+++ b/templates/prometheus.yml.j2
@@ -1,7 +1,9 @@
 {%- if ver_major == 0 and ver_minor < 20 %}
 {%- set static_config_option_text = "target_groups" %}
+{%- set file_sd_configs_option_text = "names" %}
 {%- else %}
 {%- set static_config_option_text = "static_configs" %}
+{%- set file_sd_configs_option_text = "files" %}
 {%- endif %}
 # my global config
 global:
@@ -67,13 +69,17 @@ scrape_configs:
           group: 'promoagents-juju'
 {%- endfor %}
 
+{% if manual_jobs %}
+# Manually configured scrape jobs eg. SNMP exporter
+# We need to paste raw because we don't know all future option names
+{{ manual_jobs|indent(2, True) }}
+{% endif %}
+
 {%- if promreg %}
 # Prometheus registration
   - job_name: 'prometheus-registration'
     file_sd_configs:
-# After version 0.2.0 files is correct not names
-#      - files: ['/etc/prometheus/promreg/targets.yaml']
-      - names: ['/etc/prometheus/promreg/targets.yaml']
+      - {{ file_sd_configs_option_text }}: ['/etc/prometheus/promreg/targets.yaml']
 {%- endif %}
 
 # federate jobs