Merge ~afreiberger/charm-grafana:lp#1858490 into charm-grafana:master

[Drew Freiberger]

Check dashboards before uploading new revisions

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[Joe Guo (guoqiao)]

Except pep8 style, looks good to me.

review: Approve

[Alvaro Uria (aluria)]

Please find comments inline. I think the json_diff_result condition is the opposite of what it should be.

review: Needs Fixing

[Drew Freiberger (afreiberger)]

merged fixes, added .gitignore to cover tox coverage/report files. Thanks, Alvaro!

review: Needs Resubmitting

[Alvaro Uria (aluria)]

lgtm

review: Approve

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change successfully merged at revision 220f3b5b72fffbf3dae8f151f32836b5eaa0d18d

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..88f01bf
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,13 @@
+*.swp
+*~
+.idea
+.tox
+.coverage
+.vscode
+/builds/
+/deb_dist
+/dist
+/repo-info
+__pycache__
+/report/
+
diff --git a/reactive/grafana.py b/reactive/grafana.py
index e1dc444..713aa8f 100644
--- a/reactive/grafana.py
+++ b/reactive/grafana.py
@@ -9,6 +9,7 @@ import shutil
 import six
 import subprocess
 import time
+from jsondiff import diff
 
 from charmhelpers.contrib.charmsupport import nrpe
 from charmhelpers.core import (
@@ -523,20 +524,125 @@ def render_custom(source, context, **parameters):
     return template.render(context)
 
 
+def get_current_dashboards(port, passwd):
+    """Returns list of available dashboards.
+
+    https://grafana.com/docs/grafana/latest/http_api/folder_dashboard_search/
+    """
+    dash_req = requests.get(
+        "http://127.0.0.1:{}/api/search?type=dash-db".format(port),
+        auth=('admin', passwd),
+    )
+    return dash_req.json() if dash_req.status_code == 200 else []
+
+
+def get_current_dashboard_json(uid, port, passwd):
+    """Returns the dashboard revision information (fake rev if not found).
+
+    https://grafana.com/docs/grafana/latest/http_api/dashboard/#get-dashboard-by-uid
+    """
+    default_dashboard = json.loads('{"version": 0}')
+    if not uid:
+        return default_dashboard
+
+    dash_req = requests.get(
+        "http://127.0.0.1:{}/api/dashboards/uid/{}".format(port, uid),
+        auth=('admin', passwd),
+    )
+    if dash_req.status_code != 200:
+        return default_dashboard
+
+    try:
+        return json.loads(dash_req.text)["dashboard"]
+    except json.decoder.JSONDecodeError:
+        return default_dashboard
+
+
+def check_and_add_dashboard(
+    filename, context, prom_metrics, dash_to_uid, port, gf_adminpasswd
+):
+    """Verifies if the rendered dashboard from template is new or do nothing.
+
+    * Renders a template with context gathered from
+    generate_prometheus_dashboards
+    * Checks that metrics in the rendered dashboard have already been retrieved
+    by Prometheus
+
+    https://grafana.com/docs/grafana/latest/features/datasources/prometheus/#query-editor
+    """
+    dashboard_str = render_custom(
+        source=filename,
+        context=context,
+        variable_start_string="<<",
+        variable_end_string=">>",
+    )
+    hookenv.log("Checking Dashboard Template: {}".format(filename))
+    expr = str(re.findall('"expr":(.*),', dashboard_str))
+    metrics = set(re.findall("[a-zA-Z0-9]*_[a-zA-Z0-9_]*", expr))
+    if not metrics:
+        hookenv.log(
+            "Skipping Dashboard Template: {} no metrics in template"
+            " {}".format(filename, metrics)
+        )
+        return
+
+    missing_metrics = set([x for x in metrics if x not in prom_metrics])
+    if missing_metrics:
+        hookenv.log(
+            "Skipping Dashboard Template: {} missing {} metrics."
+            "Missing: {}".format(
+                filename, len(missing_metrics), ', '.join(missing_metrics)
+            ),
+            hookenv.DEBUG,
+        )
+        return
+
+    dashboard_json = json.loads(dashboard_str)
+    # before uploading the dashboard, we should check that it doesn't already exist with same data lp#1858490
+    new = dashboard_json["dashboard"]
+    curr = get_current_dashboard_json(
+        dash_to_uid.get(dashboard_json["dashboard"]["title"], None),
+        port,
+        gf_adminpasswd,
+    )
+    # must remove the versions as they will likely be different
+    del new["version"]
+    del curr["version"]
+    dashboard_changed = diff(new, curr)
+    if not dashboard_changed:
+        hookenv.log(
+            "Skipping Dashboard Template: already up to date: {}".format(filename)
+        )
+        return
+
+    hookenv.log("Using Dashboard Template: {}".format(filename))
+    post_req = "http://127.0.0.1:{}/api/dashboards/db".format(port)
+    r = requests.post(post_req, json=dashboard_json, auth=("admin", gf_adminpasswd))
+
+    if r.status_code != 200:
+        hookenv.log(
+            "Posting template {} failed with error: {}".format(filename, r.text),
+            hookenv.ERROR,
+        )
+
+
 def generate_prometheus_dashboards(gf_adminpasswd, ds):
     # prometheus_host = ds
     ds_name = '{} - {}'.format(ds['service_name'], ds['description'])
     config = hookenv.config()
 
+    # https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values
     response = requests.get('{}/api/v1/label/__name__/values'.format(ds['url']))
     if response.status_code != 200:
         hookenv.log('Could not reach prometheus API status code: '
                     ' {}'.format(response.status_code), 'ERROR')
         return
 
+    current_dashboards = get_current_dashboards(config["port"], gf_adminpasswd)
+    dash_to_uid = {dash['title']: dash['uid'] for dash in current_dashboards if 'title' in dash and 'uid' in dash}
+
     prom_metrics = response.json()['data']
     templates_dir = 'templates/dashboards/prometheus'
-    post_req = 'http://127.0.0.1:{}/api/dashboards/db'.format(config['port'])
     context = {'datasource': ds_name,
                'external_network': config['external_network'],
                'bcache_enabled': "bcache_cache_hit_ratio" in prom_metrics,
@@ -551,33 +657,16 @@ def generate_prometheus_dashboards(gf_adminpasswd, ds):
                       'ip_status', 'neutron_net', config['external_network'],
                       'neutron_public_ip_usage']
     prom_metrics.extend(ignore_metrics)
-    for filename in os.listdir(templates_dir):
-        dashboard_str = render_custom(source=filename,
-                                      context=context,
-                                      variable_start_string="<<",
-                                      variable_end_string=">>",
-                                      )
-        hookenv.log("Checking Dashboard Template: {}".format(filename))
-        expr = str(re.findall('"expr":(.*),', dashboard_str))
-        metrics = set(re.findall('[a-zA-Z0-9]*_[a-zA-Z0-9_]*', expr))
-        if not metrics:
-            hookenv.log("Skipping Dashboard Template: {} no metrics in template"
-                        " {}".format(filename, metrics))
-            continue
-        missing_metrics = set([x for x in metrics if x not in prom_metrics])
-        if missing_metrics:
-            hookenv.log("Skipping Dashboard Template: {} missing {} metrics."
-                        "Missing: {}".format(filename, len(missing_metrics),
-                                             ', '.join(missing_metrics)
-                                             ), hookenv.DEBUG)
-        else:
-            hookenv.log("Using Dashboard Template: {}".format(filename))
-            dashboard_json = json.loads(dashboard_str)
-            r = requests.post(post_req, json=dashboard_json, auth=('admin', gf_adminpasswd))
 
-            if r.status_code != 200:
-                hookenv.log("Posting template {} failed with error:"
-                            " {}".format(filename, r.text), 'ERROR')
+    for filename in os.listdir(templates_dir):
+        check_and_add_dashboard(
+            filename,
+            context,
+            prom_metrics,
+            dash_to_uid,
+            config["port"],
+            gf_adminpasswd,
+        )
 
 
 def generate_query(ds, is_default, id=None):
diff --git a/wheelhouse.txt b/wheelhouse.txt
index f229360..ffdadef 100644
--- a/wheelhouse.txt
+++ b/wheelhouse.txt
@@ -1 +1,2 @@
 requests
+jsondiff