Merge ~jk0ne/charm-nrpe:check_swap_activity into ~nrpe-charmers/charm-nrpe:master

[Jay Kuri]

Add swap_activity check and bump default swap usage limits to more accurately reflect actionable resource-starvation probable states.

diff --git a/config.yaml b/config.yaml
index 680397c..1086205 100644
--- a/config.yaml
+++ b/config.yaml
@@ -78,9 +78,13 @@ options:
     type: string
     description: Users check
   swap:
-    default: "-w 90% -c 75%"
+    default: "-w 40% -c 25%"
     type: string
     description: Swap check
+  swap_activity:
+    default: "-i 5 -w 100 -c 500"
+    type: string
+    description: Swap activity check
   mem:
     default: "-C -u -w 85 -c 90"
     type: string
diff --git a/files/plugins/check_swap_activity b/files/plugins/check_swap_activity
new file mode 100755
index 0000000..3fc49d4
--- /dev/null
+++ b/files/plugins/check_swap_activity
@@ -0,0 +1,78 @@
+#!/bin/bash
+
+# This script checks swap pageouts and reports number of kbytes moved 
+# from physical ram to swap space in a given number of seconds
+#
+# Usage: "check_swap_activity -i interval  -w warning_kbyts  -c critical_kbytes
+#
+#
+
+set -eu
+
+. /usr/lib/nagios/plugins/utils.sh
+
+
+help() {
+cat << EOH
+usage: $0 [ -i ## ] -w ## -c ## 
+
+Measures page-outs to swap over a given interval, by default 5 seconds.
+
+   -i time in seconds to monitor (defaults to 5 seconds)
+   -w warning Level in kbytes
+   -c critical Level in kbytes
+
+EOH
+}
+
+TIMEWORD=seconds
+WARN_LVL=
+CRIT_LVL=
+INTERVAL=5
+## FETCH ARGUMENTS
+while getopts "i:w:c:" OPTION; do
+        case "${OPTION}" in
+                i)
+                        INTERVAL=${OPTARG}
+                        if [ $INTERVAL -eq 1 ]; then
+                            TIMEWORD=second
+                        fi
+                        ;;
+                w)
+                        WARN_LVL=${OPTARG}
+                        ;;
+                c)
+                        CRIT_LVL=${OPTARG}
+                        ;;
+                ?)
+                        help
+                        exit 3
+                        ;;
+        esac
+done
+
+if [ -z ${WARN_LVL} ] || [ -z ${CRIT_LVL} ]  ; then
+        help
+        exit 3
+fi
+
+## Get swap pageouts over $INTERVAL
+PAGEOUTS=$(vmstat -w ${INTERVAL} 2 | tail -n 1 | awk '{print $8}')
+
+SUMMARY="| swapout_size=${PAGEOUTS}KB;${WARN_LVL};${CRIT_LVL};"
+if [ ${PAGEOUTS} -lt ${WARN_LVL} ]; then
+        # pageouts are below threshold
+        echo "OK - ${PAGEOUTS} kb swapped out in last ${INTERVAL} ${TIMEWORD} $SUMMARY"
+        exit $STATE_OK
+elif [ ${PAGEOUTS} -ge ${CRIT_LVL} ]; then
+        ## SWAP IS IN CRITICAL STATE
+        echo "CRITICAL - ${PAGEOUTS} kb swapped out in last ${INTERVAL} ${TIMEWORD} $SUMMARY" 
+        exit $STATE_CRITICAL
+elif [ ${PAGEOUTS} -ge ${WARN_LVL} ] && [ ${PAGEOUTS} -lt ${CRIT_LVL} ]; then
+        ## SWAP IS IN WARNING STATE
+        echo "WARNING - ${PAGEOUTS} kb swapped out in last ${INTERVAL} ${TIMEWORD} $SUMMARY"
+        exit $STATE_WARNING
+else
+        echo "CRITICAL: Failure to process pageout information $SUMMARY"
+        exit $STATE_UNKNOWN
+fi
diff --git a/hooks/nrpe_helpers.py b/hooks/nrpe_helpers.py
index 1b03b0f..6768e99 100644
--- a/hooks/nrpe_helpers.py
+++ b/hooks/nrpe_helpers.py
@@ -307,6 +307,12 @@ class SubordinateCheckDefinitions(dict):
                 'cmd_params': hookenv.config('swap'),
             },
             {
+                'description': 'Swap Activity',
+                'cmd_name': 'check_swap_activity',
+                'cmd_exec': local_plugin_dir + 'check_swap_activity',
+                'cmd_params': hookenv.config('swap_activity'),
+            },
+            {
                 'description': 'Memory',
                 'cmd_name': 'check_mem',
                 'cmd_exec': local_plugin_dir + 'check_mem.pl',
diff --git a/tests/10-tests b/tests/10-tests
index df455fa..29f2e58 100755
--- a/tests/10-tests
+++ b/tests/10-tests
@@ -71,6 +71,7 @@ class TestDeployment(unittest.TestCase):
             'check_zombie_procs_sub',
             'check_total_procs_sub',
             'check_conntrack_sub',
+            'check_swap_activity_sub',
         ]
         for check in checks:
             if check not in monitors['monitors']['remote']['nrpe'].keys():
@@ -336,6 +337,15 @@ class TestDeployment(unittest.TestCase):
         }
         self.check_nrpe_setting(**test_config)
 
+    def test_custom_swap_activity_check_params(self):
+        chk_key = 'command[check_swap_activity]=/usr/local/lib/nagios/plugins/check_swap_activity'
+        test_config = {
+            'filename': '/etc/nagios/nrpe.d/check_swap_activity_sub.cfg',
+            'expected_settings': {chk_key: '-w 20 -c 700'},
+            'juju_kv': {'swap_activity': '-w 20 -c 700'}
+        }
+        self.check_nrpe_setting(**test_config)
+
     def test_custom_conntrack_check_params(self):
         chk_key = ('command[check_conntrack]=/usr/local/lib/nagios/plugins/'
                    'check_conntrack.sh')