Merge ~tcuthbert/turku/+git/turku-api:main into ~turku/turku/+git/turku-api:main

[Thomas Cuthbert]

[Barry Price (barryprice)]

LGTM +1

review: Approve

Unmerged commits

147397e... by Thomas Cuthbert on 2021-11-16

Import from Bazaar to Git

f249756... by Thomas Cuthbert on 2021-02-04

Refactor the metrics based on promtheus querying.

Reviewed-on: https://code.launchpad.net/~tcuthbert/turku/turku-api/+merge/397467
Reviewed-by: Junien Fridrick <email address hidden>

b1cdb06... by Thomas Cuthbert on 2021-02-04

Refactor the metrics based on findings with prometheus querying.

fc68957... by Thomas Cuthbert on 2021-02-04

Add unhealthy turku backup sources monitoring script

Reviewed-on: https://code.launchpad.net/~tcuthbert/turku/turku-api/+merge/397366
Reviewed-by: Haw Loeung <email address hidden>

2de52a4... by Thomas Cuthbert on 2021-02-04

Reduce indentation level

03b53d0... by Thomas Cuthbert on 2021-02-04

Add unhealthy turku backup sources monitoring script

diff --git a/scripts/turku_sick_sources b/scripts/turku_sick_sources
new file mode 100755
index 0000000..fc44712
--- /dev/null
+++ b/scripts/turku_sick_sources
@@ -0,0 +1,127 @@
+#!/usr/bin/env python2
+
+# Turku backups
+# Copyright 2021 Canonical Ltd.
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 3, as published by
+# the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranties of MERCHANTABILITY,
+# SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import argparse
+import os
+import sys
+
+from time import mktime
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(BASE_DIR)
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "turku_api.settings")
+
+from turku_api.models import Source  # NOQA: E402
+
+
+MEASUREMENT_NAME = "turku_sick"
+
+
+def options():
+    parser = argparse.ArgumentParser(
+        description=SickSources.__doc__, formatter_class=argparse.RawTextHelpFormatter
+    )
+    parser.add_argument(
+        "--metrics_dir", help="directory where metrics will be dumped", type=str
+    )
+    return parser.parse_args()
+
+
+class SickSources:
+    """
+    turku_sick_sources dumps metrics in the telegraf line format for consumption by telegraf.
+
+    The tool emits metrics for turku backup sources that are active and published, and failing their backups.
+
+    The tool has a --metrics_dir option that can be used for writing metrics to a file. This is useful when
+    you want to use cron + the telegraf file input rather than exec. The metrics will be written to a file named
+    sick_sources.out
+    """
+
+    # Telegraf doesn't like boolean metrics so use this primitive enum-like object
+    # for readability.
+    _health_enum = {0: "no", 1: "yes"}
+
+    def run(self, options):
+        sources_total = 0
+        sources_sick = 0
+        sick_objects = []
+        data = []
+
+        for source in Source.objects.filter(
+            machine__active=True, machine__published=True, active=True, published=True,
+        ):
+            sources_total += 1
+            if not source.healthy():
+                sources_sick += 1
+            # Rather than mixing types, initialise timestamp as epoch at default precision.
+            date_last_backed_up = 0.0
+            if source.date_last_backed_up is not None:
+                date_last_backed_up = mktime(source.date_last_backed_up.timetuple())
+            source = (
+                source.machine.unit_name,
+                source.name,
+                self._health_enum[int(source.healthy())],
+                date_last_backed_up,
+            )
+            sick_objects.append(source)
+
+        if sources_sick == 0:
+            sys.stdout.write("Nothing to do, all turku sources are healthy\n")
+            return
+
+        totals = (
+            "{measurement} sources_unhealthy={sources_sick},sources_total={sources_total}\n"
+        ).format(
+            measurement=MEASUREMENT_NAME,
+            sources_sick=sources_sick,
+            sources_total=sources_total,
+        )
+        data.append(totals)
+
+        sys.stdout.write(totals)
+
+        for source in sick_objects:
+            machine_unit_name, name, health, date_last_backed_up = source
+            # The metrics we are exporting is the unix timestamp of date last backed up. Include a healthy label
+            # for concise query filtering.
+            metric = (
+                "{measurement},source_machine_name={machine_unit_name},source_name={name},healthy={health} "
+                "date_last_backed_up={date_last_backed_up}\n"
+            ).format(
+                measurement=MEASUREMENT_NAME,
+                machine_unit_name=machine_unit_name,
+                name=name,
+                health=health,
+                date_last_backed_up=date_last_backed_up,
+            )
+            data.append(metric)
+
+            sys.stdout.write(metric)
+
+        if os.path.isdir(options.metrics_dir):
+            file_p = os.path.join(options.metrics_dir, "sick_sources.out")
+            with open(file_p, "w") as f:
+                f.writelines(data)
+
+        sys.stdout.write(
+            "Sick turku sources successfully dumped to: {}\n".format(file_p)
+        )
+
+
+if __name__ == "__main__":
+    sys.exit(SickSources().run(options()))