Merge ~lihuiguo/charm-nrpe:bug/1901260 into charm-nrpe:master

[Linda Guo]

    Added snap refresh check

    check_snap_refresh.py to check if snap refresh is successful
    Alerts if snap refresh failed.

    LP#1901260

Added snap refresh check

[Xav Paice (xavpaice)]

1 nit

405910a... by Linda Guo on 2021-07-05

fixed typo

79e6f9e... by Linda Guo on 2021-07-05

skip if last refresh time is n/a

last refresh time unavailable on newly deployed machine,
skip snap refresh checking

5a9a4b9... by Linda Guo on 2021-07-05

 trigger a snap refresh on installing the check

run snap refresh on install the checking
to avoid newly deployed machines start off
with warning

[Joe Guo (guoqiao)]

2 inline comments, only the `/` vs `//` matters.

review: Needs Fixing

950d4fc... by Linda Guo on 2021-07-08

Updated according to Joe's comment

[Linda Guo (lihuiguo)]

Fix's Joe's comments

[James Troup (elmo)]

I think we need to be more careful in parsing the timer output; I just checked a random etcd unit and saw this:

ubuntu@juju-647be6-10-lxd-0:~$ snap refresh --abs-time --time
timer: wed2
last: 2021-06-09T00:02:00Z
next: 2021-07-14T00:00:00Z
ubuntu@juju-647be6-10-lxd-0:~$

See also: https://snapcraft.io/docs/timer-string-format

review: Needs Fixing

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

Unmerged commits

950d4fc... by Linda Guo on 2021-07-08

Updated according to Joe's comment

5a9a4b9... by Linda Guo on 2021-07-05

 trigger a snap refresh on installing the check

run snap refresh on install the checking
to avoid newly deployed machines start off
with warning

79e6f9e... by Linda Guo on 2021-07-05

skip if last refresh time is n/a

last refresh time unavailable on newly deployed machine,
skip snap refresh checking

405910a... by Linda Guo on 2021-07-05

fixed typo

4750431... by Linda Guo on 2021-07-05

fixed typo

c25bd0d... by Linda Guo on 2021-07-05

Merge branch 'master' of git+ssh://git.launchpad.net/charm-nrpe into bug/1901260

ab631fc... by Linda Guo on 2021-05-21

Added snap refresh check

check_snap_refresh.py to check if snap refresh is successful
Alerts if snap refresh failed.

LP#1901260

diff --git a/config.yaml b/config.yaml
index 58ff320..a4e1844 100644
--- a/config.yaml
+++ b/config.yaml
@@ -193,3 +193,8 @@ options:
       beginning with the string /sys.
       The check is disabled on all LXD units, and also for non-container units if this parameter is
       set to ''.
+  snap_refresh_check:
+    default: False
+    type: boolean
+    description: |
+        To enable or disable snap refresh failure check.
\ No newline at end of file
diff --git a/files/plugins/check_snap_refresh.py b/files/plugins/check_snap_refresh.py
new file mode 100755
index 0000000..348cc60
--- /dev/null
+++ b/files/plugins/check_snap_refresh.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""Check snap refresh failure and alert."""
+#
+# Copyright 2021 Canonical Ltd
+#
+#
+# Check snap refresh and alert.
+#
+import datetime
+import os
+import subprocess
+import sys
+
+
+def get_interval(timer):
+    """Get snap refresh interval."""
+    try:
+        frequency = timer.split("/")[-1]
+        interval = 24 // int(frequency)
+    except Exception:
+        # default 6 hours
+        interval = 6
+    return interval
+
+
+def main():
+    """Check snap refresh is successful or not."""
+    try:
+        # check if snap is installed. If not, then skip.
+        if not os.path.exists('/usr/bin/snap'):
+            print("OK: snap isn't installed")
+            sys.exit(0)
+        refresh_time = {}
+        # snap refresh output looks like
+        #  $snap refresh --abs-time --time
+        #  timer: 00:00~24:00/4
+        #  last: 2021-03-29T15:34:00+13:00
+        #  next: 2021-03-29T22:24:00+13:00
+        lines = subprocess.getoutput("snap refresh --abs-time --time").splitlines()
+        if len(lines) < 3:
+            print("WARNING: snap command returned unexpected result")
+            sys.exit(2)
+        for line in lines:
+            k, v = line.split(":", 1)
+            refresh_time[k] = v
+        interval = get_interval(refresh_time['timer'])
+        last_refresh = refresh_time['last'].split('+')[0].strip().replace('T', ' ')
+        # last refresh time unavailable, maybe the machine was just installed
+        # charm would run 'snap refresh' when installing this check, ignore here
+        if last_refresh == "n/a":
+            print("OK: last snap refresh time is unavailable.")
+            sys.exit(0)
+        last_refresh_dtime = datetime.datetime.strptime(last_refresh,
+                                                        '%Y-%m-%d %H:%M:%S')
+        expect_refresh_dtime = last_refresh_dtime + datetime.timedelta(hours=interval)
+        if datetime.datetime.now() > expect_refresh_dtime:
+            print("CRITICAL: snap hasn't been refreshed for more than {} hours!"
+                  .format(interval))
+            sys.exit(2)
+        else:
+            print("OK: next snap refresh time {}".format(refresh_time['next']))
+            sys.exit(0)
+    except Exception as e:
+        print("WARNING: {}".format(str(e)))
+        sys.exit(2)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/hooks/nrpe_helpers.py b/hooks/nrpe_helpers.py
index e4b51fd..240925b 100644
--- a/hooks/nrpe_helpers.py
+++ b/hooks/nrpe_helpers.py
@@ -507,6 +507,19 @@ class SubordinateCheckDefinitions(dict):
                 }
             checks.append(check_ro_filesystem)
 
+        if hookenv.config("snap_refresh_check"):
+            snap_refresh_check = {
+                "description": "Snap refresh check",
+                "cmd_name": "check_snap_refresh",
+                "cmd_exec": local_plugin_dir + "check_snap_refresh.py",
+                "cmd_params": " ",
+                }
+            checks.append(snap_refresh_check)
+            try:
+                subprocess.check_call(["sudo", "snap", "refresh"])
+            except subprocess.CalledProcessError as err:
+                hookenv.log(err.output)
+
         if hookenv.config("lacp_bonds").strip():
             for bond_iface in hookenv.config("lacp_bonds").strip().split():
                 if os.path.exists("/sys/class/net/{}".format(bond_iface)):
diff --git a/tests/functional/tests/nrpe_tests.py b/tests/functional/tests/nrpe_tests.py
index f887fdb..5f09f42 100644
--- a/tests/functional/tests/nrpe_tests.py
+++ b/tests/functional/tests/nrpe_tests.py
@@ -224,3 +224,24 @@ class TestNrpe(TestBase):
             raise model.CommandRunFailed(cmd, result)
         content = result.get("Stdout")
         self.assertTrue(line in content)
+
+    def test_06_snap_refresh(self):
+        """Check snap refresh checks are applied."""
+        model.set_application_config(self.application_name,
+                                     {"snap_refresh_check": "True"})
+        model.block_until_all_units_idle()
+        cmd = "cat /etc/nagios/nrpe.d/check_snap_refresh.cfg"
+        line = (
+            "command[check_snap_refresh]=/usr/local/lib/nagios/plugins/"
+            "check_snap_refresh.py"
+        )
+        result = model.run_on_unit(self.lead_unit_name, cmd)
+        code = result.get("Code")
+        if code != "0":
+            logging.warning(
+                "Unable to find nrpe check at "
+                "/etc/nagios/nrpe.d/check_snap_refresh.cfg"
+            )
+            raise model.CommandRunFailed(cmd, result)
+        content = result.get("Stdout")
+        self.assertTrue(line in content)