Merge charm-grafana:dev/fix-default-data-source-values into charm-grafana:master

[David O Neill]

This fixes
https://bugs.launchpad.net/charm-grafana/+bug/1897577

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

3909788... by David O Neill on 2020-09-29

fix missing default

[Alvaro Uria (aluria)]

"make test" fails due to linting (seems unrelated to this change, but it would be good to fix):
"""
lint run-test-pre: PYTHONHASHSEED='1443983391'
lint run-test: commands[0] | flake8
./reactive/grafana.py:987:60: W291 trailing whitespace
ERROR: InvocationError for command /home/circleci/project/charm-grafana/src/.tox/lint/bin/flake8 (exited with code 1)
___________________________________ summary ____________________________________
ERROR: lint: commands failed
Makefile:57: recipe for target 'lint' failed
make: *** [lint] Error 1
"""

Have we tested if the change works on other versions of the grafana snap? 6/stable and latest/stable point to v6.7.4, while 7/stable points to v7.0.3. The APT package is on v7, too, and this change should be tested on both v6 (only available via snap) and v7.
Note: Func tests already test 6/stable from the snap, as well as changes to 7/stable; Non-snap func tests (aka via apt) already test v7 directly.

Marked as "Needs fixing" so "make test" can pass.

review: Needs Fixing

faee20b... by David O Neill on 2020-09-30

Rework values for version 7 of snap and fix linting issues

[Adam Dyess (addyess)]

Thanks for fixing the linting + black here. LGTM although this .append(...) stuff isn't super clear

review: Approve

Unmerged commits

faee20b... by David O Neill on 2020-09-30

Rework values for version 7 of snap and fix linting issues

3909788... by David O Neill on 2020-09-29

fix missing default

diff --git a/src/lib/charms/layer/grafana.py b/src/lib/charms/layer/grafana.py
index d48c5bd..ecd8c69 100644
--- a/src/lib/charms/layer/grafana.py
+++ b/src/lib/charms/layer/grafana.py
@@ -118,7 +118,10 @@ def post_dashboard(name, dashboard):
         return (False, "Unable to retrieve grafana password.")
     api_auth = ("admin", passwd)
     r = requests.post(
-        import_url, auth=api_auth, headers=headers, data=json.dumps(dashboard),
+        import_url,
+        auth=api_auth,
+        headers=headers,
+        data=json.dumps(dashboard),
     )
     if r.status_code == 200:
         return (True, None)
diff --git a/src/reactive/grafana.py b/src/reactive/grafana.py
index ff26b3e..88e7eea 100644
--- a/src/reactive/grafana.py
+++ b/src/reactive/grafana.py
@@ -188,7 +188,8 @@ def install_packages():
 
         if config.changed("install_file") and config.get("install_file"):
             hookenv.status_set(
-                "maintenance", "Installing deb pkgs since install_file changed",
+                "maintenance",
+                "Installing deb pkgs since install_file changed",
             )
 
             url = config.get("install_file")
@@ -286,7 +287,8 @@ def install_plugins():
 def upgrade_charm():
     """Run upgrade charm hook."""
     hookenv.status_set(
-        "maintenance", "Forcing package update and reconfiguration on upgrade-charm",
+        "maintenance",
+        "Forcing package update and reconfiguration on upgrade-charm",
     )
     remove_state("grafana.installed")
     remove_state("grafana.backup.configured")
@@ -318,7 +320,9 @@ def config_changed():
         else:
             # Install from requested channel
             snap.install(
-                SNAP_NAME, channel=config["snap_channel"], force_dangerous=False,
+                SNAP_NAME,
+                channel=config["snap_channel"],
+                force_dangerous=False,
             )
     remove_state("grafana.configured")
     if config.changed("dashboards_backup_schedule") or config.changed(
@@ -401,7 +405,8 @@ def add_backup_api_keys():
     for i in select_query("SELECT id FROM org"):
         org_id = i[0]
         if select_query(
-            "SELECT id FROM api_key WHERE org_id=? AND name=?", [org_id, name],
+            "SELECT id FROM api_key WHERE org_id=? AND name=?",
+            [org_id, name],
         ):
             hookenv.log(
                 "API key {} in org {} already exists, skipping".format(name, org_id)
@@ -475,7 +480,8 @@ def setup_backup_shedule():
         hookenv.status_set("maintenance", "Configuring grafana dashboard backup")
         hookenv.log("Setting up dashboards backup job...")
         host.rsync(
-            "files/dashboards_backup", "/usr/local/bin/dashboards_backup",
+            "files/dashboards_backup",
+            "/usr/local/bin/dashboards_backup",
         )
         host.mkdir(config.get("dashboards_backup_dir"))
         settings = {
@@ -604,7 +610,8 @@ def sources_gone(relation):
         conn = sqlite3.connect("{}/grafana.db".format(data_dir), timeout=30)
         cur = conn.cursor()
         cur.execute(
-            "DELETE FROM DATA_SOURCE WHERE  type=? AND url=?", (ds["type"], ds["url"]),
+            "DELETE FROM DATA_SOURCE WHERE  type=? AND url=?",
+            (ds["type"], ds["url"]),
         )
         conn.commit()
         cur.close()
@@ -784,7 +791,9 @@ def check_and_add_dashboard(
         hookenv.log(
             "Skipping Dashboard Template: {} missing {} metrics."
             "Missing: {}".format(
-                filename, len(missing_metrics), ", ".join(missing_metrics),
+                filename,
+                len(missing_metrics),
+                ", ".join(missing_metrics),
             ),
             hookenv.DEBUG,
         )
@@ -820,7 +829,8 @@ def check_and_add_dashboard(
 
     hookenv.log(
         "dashboard differences[{}], dashboard title[{}]".format(
-            dashboard_changed, dashboard_json["dashboard"]["title"],
+            dashboard_changed,
+            dashboard_json["dashboard"]["title"],
         ),
         hookenv.DEBUG,
     )
@@ -955,6 +965,14 @@ def generate_query(ds, is_default, id=None, org_id=1):
             # The 32 bit mask is necessary because Python's 64 bit ints are too
             # large for SQLite3.
             values.append(hash(ds["url"]) & 0xFFFFFFFF)
+            # latest grafana will fail to accept the datasource
+            # unless these fields are default as follows
+            fields.append("json_data")
+            values.append("{}")
+            fields.append("secure_json_data")
+            values.append("{}")
+            fields.append("read_only")
+            values.append(0)
 
         # Used in connecting a mysql source
         if "database" in ds:
@@ -1020,7 +1038,8 @@ def get_orgs(port, passwd):
     https://grafana.com/docs/grafana/latest/http_api/org/
     """
     req = requests.get(
-        "http://127.0.0.1:{}/api/orgs".format(port), auth=("admin", passwd),
+        "http://127.0.0.1:{}/api/orgs".format(port),
+        auth=("admin", passwd),
     )
     return req.json() if req.status_code == 200 else []
 
diff --git a/src/tests/functional/tests/test_grafana.py b/src/tests/functional/tests/test_grafana.py
index f848077..beeddd2 100644
--- a/src/tests/functional/tests/test_grafana.py
+++ b/src/tests/functional/tests/test_grafana.py
@@ -41,7 +41,9 @@ class BaseGrafanaTest(unittest.TestCase):
         """Get grafana admin password."""
         if self._admin_pass is None:
             action = model.run_action(
-                self.lead_unit_name, "get-admin-password", raise_on_failure=True,
+                self.lead_unit_name,
+                "get-admin-password",
+                raise_on_failure=True,
             )
             self._admin_pass = action.data["results"]["password"]
         return self._admin_pass
@@ -84,7 +86,9 @@ class CharmOperationTest(BaseGrafanaTest):
         We'll retry until the TEST_TIMEOUT.
         """
         test_command = "{} -I 127.0.0.1 -p {} -u {}".format(
-            "/usr/lib/nagios/plugins/check_http", DEFAULT_API_PORT, DEFAULT_API_URL,
+            "/usr/lib/nagios/plugins/check_http",
+            DEFAULT_API_PORT,
+            DEFAULT_API_URL,
         )
         timeout = time.time() + TEST_TIMEOUT
         while time.time() < timeout:
@@ -174,7 +178,9 @@ class CharmOperationTest(BaseGrafanaTest):
             "role": "Viewer",
         }
         action = model.run_action(
-            self.lead_unit_name, "create-user", action_params=params,
+            self.lead_unit_name,
+            "create-user",
+            action_params=params,
         )
         self.assertTrue(action.data["results"]["Code"] == "0")
         time.sleep(30)  # Dirty hack to overcome race condition
diff --git a/src/tests/unit/test_grafana.py b/src/tests/unit/test_grafana.py
index b9c75a9..c2386e4 100644
--- a/src/tests/unit/test_grafana.py
+++ b/src/tests/unit/test_grafana.py
@@ -109,7 +109,9 @@ class GrafanaTestCase(unittest.TestCase):
 
     @mock.patch.object(grafana, "model_name")
     @mock.patch.object(
-        grafana, "get_folders", return_value=[{"id": 7, "title": "[juju-foo-model]"}],
+        grafana,
+        "get_folders",
+        return_value=[{"id": 7, "title": "[juju-foo-model]"}],
     )
     @mock.patch.object(grafana, "create_folder")
     def test_ensure_and_get_dash_folder_folder_exists(