Merge ~jillrouleau/charm-nrpe:xfs_checks into ~nrpe-charmers/charm-nrpe:master

[Jill Rouleau]

[Xav Paice (xavpaice)]

LGTM

review: Approve

diff --git a/config.yaml b/config.yaml
index 3a55f70..b679aae 100644
--- a/config.yaml
+++ b/config.yaml
@@ -143,3 +143,10 @@ options:
     description: |
         A string to be appended onto all the nrpe checks created by this charm
         to avoid potential clashes with existing checks
+  xfs_errors:
+    default: "120"
+    type: string
+    description: |
+        dmesg history length to check for xfs errors, in minutes
+        .
+        Set to '' in order to disable this check.
diff --git a/files/plugins/check_xfs_errors.py b/files/plugins/check_xfs_errors.py
new file mode 100755
index 0000000..c031336
--- /dev/null
+++ b/files/plugins/check_xfs_errors.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+#
+# Copyright 2017 Canonical Ltd
+#
+# Author: Jill Rouleau <[e-mail]>
+#
+# Check for xfs errors and alert
+#
+
+import sys
+import re
+import datetime
+import subprocess
+
+# error messages commonly seen in dmesg on xfs errors
+raw_xfs_errors = ['XFS_WANT_CORRUPTED_',
+                  'xfs_error_report',
+                  'corruption detected at xfs_',
+                  'Unmount and run xfs_repair']
+
+xfs_regex = [re.compile(i) for i in raw_xfs_errors]
+
+# nagios can't read from kern.log, so we look at dmesg - this does present
+# a known limitation if a node is rebooted or dmesg is otherwise cleared.
+log_lines = [line for line in subprocess.getoutput(['dmesg -T']).split('\n')]
+
+err_results = [line for line in log_lines for rgx in xfs_regex if
+               re.search(rgx, line)]
+
+# Look for errors within the last N minutes, specified in the check definition
+check_delta = int(sys.argv[1])
+
+# dmesg -T formatted timestamps are inside [], so we need to add them
+datetime_delta = '['+(datetime.datetime.now() -
+                      datetime.timedelta(minutes=check_delta)
+                      ).strftime('%c')+']'
+
+recent_logs = [i for i in err_results if i >= datetime_delta]
+
+if recent_logs:
+    print('CRITCAL: Recent XFS errors in kern.log.'+'\n'+'{}'.format(
+        recent_logs))
+    sys.exit(2)
+else:
+    print('OK')
+    sys.exit(0)
diff --git a/hooks/nrpe_helpers.py b/hooks/nrpe_helpers.py
index 60008f9..cfcb348 100644
--- a/hooks/nrpe_helpers.py
+++ b/hooks/nrpe_helpers.py
@@ -337,6 +337,12 @@ class SubordinateCheckDefinitions(dict):
                 'cmd_exec': local_plugin_dir + 'check_conntrack.sh',
                 'cmd_params': hookenv.config('conntrack'),
             },
+            {
+                'description': 'XFS Errors',
+                'cmd_name': 'check_xfs_errors',
+                'cmd_exec': local_plugin_dir + 'check_xfs_errors.py',
+                'cmd_params': hookenv.config('xfs_errors'),
+            },
         ]
         self['checks'] = []
         sub_postfix = str(hookenv.config("sub_postfix"))