Merge lp:~ben-kaehne/charms/trusty/nagios-extra-scripts/check_interface into lp:~jillrouleau/charms/trusty/nagios-extra-scripts/refactored-checks

[Benjamin Kaehne]

I have added in the "check_interfaces" check.

Unmerged revisions

19. By Benjamin Kaehne on 2016-04-07

nagios check to check interfaces now installed into this repo with some code tidy.

Check willd default to 10g links; as per our cloud installs

18. By Benjamin Kaehne on 2016-04-06

fix

17. By Benjamin Kaehne on 2016-04-06

Pushing to test in staging. Part 1

=== added directory 'files/cron'
=== added file 'files/cron/checknet-ethtooldata'
--- files/cron/checknet-ethtooldata	1970-01-01 00:00:00 +0000
+++ files/cron/checknet-ethtooldata	2016-04-07 05:19:48 +0000
@@ -0,0 +1,1 @@
+23 */3 * * * root /usr/local/sbin/cat-check-network

=== added directory 'files/localbin'
=== added file 'files/localbin/cat-check-network'
--- files/localbin/cat-check-network	1970-01-01 00:00:00 +0000
+++ files/localbin/cat-check-network	2016-04-07 05:19:48 +0000
@@ -0,0 +1,22 @@
+#!/bin/sh
+#                                       m   
+#  mmmm   m   m  mmmm   mmmm    mmm   mm#mm 
+#  #" "#  #   #  #" "#  #" "#  #"  #    #   
+#  #   #  #   #  #   #  #   #  #""""    #   
+#  ##m#"  "mm"#  ##m#"  ##m#"  "#mm"    "mm 
+#  #             #      #                   
+#  "             "      "                   
+# This file is managed by puppet.  Do not make local changes.
+
+# cache ethtool results so nagios can check the interfaces.
+
+PATH='/sbin:/usr/sbin'
+
+# Remove existing files, otherwise nagios alerts on removed network interfaces
+/bin/rm -f /var/lib/nagios/ethtool.*
+
+# most boxes have eth?, arm buildds have usb?.  add other funky interface types
+# as we become aware of them.
+for dev in $(/sbin/ifconfig -s | /usr/bin/awk '/^p?eth|usb[0-9]+[^:]/ {print $1}'); do
+   ethtool ${dev} > /var/lib/nagios/ethtool.${dev}; 
+done

=== added file 'files/nrpe/interfaces_status.py'
--- files/nrpe/interfaces_status.py	1970-01-01 00:00:00 +0000
+++ files/nrpe/interfaces_status.py	2016-04-07 05:19:48 +0000
@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+#                                       m
+#  mmmm   m   m  mmmm   mmmm    mmm   mm#mm
+#  #" "#  #   #  #" "#  #" "#  #"  #    #
+#  #   #  #   #  #   #  #   #  #""""    #
+#  ##m#"  "mm"#  ##m#"  ##m#"  "#mm"    "mm
+#  #             #      #
+#  "             "      "
+# This file is managed by puppet.  Do not make local changes.
+
+# Copyright (C) 2009, 2012  James Troup <[e-mail]>
+
+################################################################################
+
+import nagios_plugin
+import glob
+from optparse import OptionParser
+
+
+###########################################################################
+
+def compare_expected(line, expected, type, interface):
+    expected = expected.strip().lower()
+    value = line.split(":")[1].strip().lower()
+    if value != expected:
+        raise nagios_plugin.CriticalError(
+            "%s: %s is '%s' not '%s'" % (interface, type, value, expected))
+
+
+################################################################################
+
+## Example ethtool output:
+
+# Settings for eth0:
+#         Supported ports: [ TP ]
+#         Supported link modes:   10baseT/Half 10baseT/Full
+#                                 100baseT/Half 100baseT/Full
+#                                 1000baseT/Half 1000baseT/Full
+#         Supports auto-negotiation: Yes
+#         Advertised link modes:  10baseT/Half 10baseT/Full
+#                                 100baseT/Half 100baseT/Full
+#                                 1000baseT/Half 1000baseT/Full
+#         Advertised auto-negotiation: Yes
+#         Speed: 1000Mb/s
+#         Duplex: Full
+#         Port: Twisted Pair
+#         PHYAD: 1
+#         Transceiver: internal
+#         Auto-negotiation: on
+#         Supports Wake-on: g
+#         Wake-on: g
+#         Current message level: 0x000000ff (255)
+#         Link detected: yes
+
+def check_interface(filename, interface, expected_speed, half_duplex):
+    nagios_plugin.check_file_freshness(filename, 2 * 3 * 60 * 60)
+    input_file = open(filename)
+    link_status = False
+    for line in input_file.readlines():
+        line = line.lstrip()
+        if line.startswith("Speed: "):
+            compare_expected(line, expected_speed, "speed", interface)
+        if line.startswith("Duplex: ") and interface not in half_duplex:
+            compare_expected(line, "Full", "duplex", interface)
+        if line.startswith("Link detected: "):
+            compare_expected(line, "Yes", "link detected", interface)
+            link_status = True
+    # We can't error out on lack of speed/duplex as Xen guest or
+    # bridge interfaces don't have them.
+
+    # On the other hand, we can error out if we don't find any 'link
+    # detected' line.  This check should simply be disabled on any
+    # hardware (e.g. hppa) which is so pre-historic it doesn't even
+    # support ethtool.
+    if not link_status:
+        raise nagios_plugin.CriticalError(
+            "%s: did not find link detected information" % (interface))
+
+
+################################################################################
+
+def main():
+    parser = OptionParser()
+    parser.add_option("--speed", dest="expected_speed",
+                      help="expected speed of nics [default=%default]",
+                      default="1000MB/s")
+    parser.add_option("--eth0", dest="eth0",
+                      help="expected speed of eth0 [default=global]")
+    parser.add_option("--eth1", dest="eth1",
+                      help="expected speed of eth1 [default=global]")
+    parser.add_option("--eth2", dest="eth2",
+                      help="expected speed of eth2 [default=global]")
+    parser.add_option("--half", dest="half_duplex", action="append",
+                      default=[], help="list of interfaces that are "
+                      " expected to be half duplex")
+    (opts, args) = parser.parse_args()
+    interfaces = []
+    etoutputs = glob.glob("/var/lib/nagios/ethtool.*")
+    # glob can return an empty list if there are no files.  check result
+    # should then be unknown.
+    if not etoutputs:
+        raise nagios_plugin.UnknownError("No ethtool output in /var/lib/nagios")
+    for filename in etoutputs:
+        # only check non-vlan interfaces
+        # vlan interfaces have a filename ending in "ethtool.eth2.2590" or such
+        # while non-vlan will end with "ethtool.eth0"
+        # Ignore anything with multiple dots in the name, since ethtool
+        # will simply say it has no info for the vlans
+        # also ignore aliases on an interface, on the assumption that we
+        # always have the non-alias interface as well
+        if filename.count(".") == 1 and filename.count(":") == 0:
+            interface = filename.split(".")[-1]
+            expected_speed = opts.__dict__.get(interface, None)
+            if not expected_speed:
+                expected_speed = opts.expected_speed
+            check_interface(filename, interface, expected_speed,
+                            half_duplex=opts.half_duplex)
+            interfaces.append(interface)
+    print "All OK on: %s" % (", ".join(interfaces))
+
+if __name__ == "__main__":
+    nagios_plugin.try_check(main)

=== modified file 'hooks/hooks.py'
--- hooks/hooks.py	2015-11-24 01:00:30 +0000
+++ hooks/hooks.py	2016-04-07 05:19:48 +0000
@@ -14,6 +14,11 @@
 
 NAGIOS_EXPORT = "/var/lib/nagios/export/"
 NAGIOS_PLUGINS = '/usr/lib/nagios/plugins/'
+SCRIPT_DEST = { 'nrpe': NAGIOS_PLUGINS,
+                'localbin': '/usr/local/sbin/',
+                'cron': '/etc/cron.d/' }
+
+
 hooks = hookenv.Hooks()
 config = hookenv.config()
 
@@ -24,13 +29,11 @@
     ContainerType = "metal"
 
 
-def install_nrpe_scripts(script):
-    # We want to be able to specify which scripts to copy
-    # Not all scripts in CHARM_DIR/files will go on all units
-    nrpe_files_dir = os.path.join(os.environ["CHARM_DIR"], "files", "nrpe")
-    for fname in glob.glob(os.path.join(nrpe_files_dir, script)):
+def install_scripts(file, type):
+    nrpe_files_dir = os.path.join(os.environ["CHARM_DIR"], "files", type)
+    for fname in glob.glob(os.path.join(nrpe_files_dir, file)):
         shutil.copy2(fname,
-                     os.path.join(NAGIOS_PLUGINS, os.path.basename(fname)))
+                     os.path.join(SCRIPT_DEST[type], os.path.basename(fname)))
 
 
 @hooks.hook('install', 'start')
@@ -94,16 +97,23 @@
 
     # If this is a bare metal host, check conntrack
     if ContainerType is "lxc":
-        install_nrpe_scripts("conntrack_status.py")
+        install_scripts('conntrack_status.py', 'nrpe')
         nrpe_setup.add_check('conntrack_max', 'Check Maximum Conntrack',
                               'conntrack_status.py conntrack_max'),
         nrpe_setup.add_check('conntrack_used', 'Check Used Conntrack',
                               'conntrack_status.py conntrack_used'),
 
     # In all cases, check ksplice status.
-    install_nrpe_scripts("ksplice_status.py"),
+    install_scripts('ksplice_status.py', 'nrpe')
     nrpe_setup.add_check('ksplice_status', 'Check KSplice Status',
-                          'ksplice_status.py'),
+                          'ksplice_status.py')
+
+    install_scripts('interfaces_status.py', 'nrpe')
+    install_scripts('cat-check-network', 'localbin')
+    install_scripts('checknet-ethtooldata', 'cron')
+
+    nrpe_setup.add_check('interfaces_status', 'Check Interface',
+                          'interfaces_status.py')
 
     nrpe_setup.write()
 

=== modified file 'hooks/services.py'
--- hooks/services.py	2015-10-10 00:36:15 +0000
+++ hooks/services.py	2016-04-07 05:19:48 +0000
@@ -60,11 +60,6 @@
             ],
             'data_ready': [
                 helpers.render_template(
-                    source='conntrack_status.py',
-                    target='/usr/lib/nagios/plugins/conntrack_status.py',
-                    perms=0o755
-                ),
-                helpers.render_template(
                     source='conntrack_nrpe.cfg.j2',
                     target='/etc/nagios/nrpe.d/check_conntrack.cfg',
                     perms=0o644
@@ -79,9 +74,18 @@
                     )
                 ),
                 helpers.render_template(
-                    source='ksplice_status.py',
-                    target='/usr/lib/nagios/plugins/ksplice_status.py',
-                    perms=0o755
+                    source='interfaces_nrpe.cfg.j2',
+                    target='/etc/nagios/nrpe.d/check_interfaces.cfg',
+                    perms=0o644
+                ),
+                helpers.render_template(
+                    source='interfaces_nagios.cfg.j2',
+                    target=EXPORTDIR+'service__{}-{}_{}.cfg'.format(
+                        config['nagios_context'],
+                        nagios_helpers.parent_unit,
+                        "interfaces_status",
+                        perms=0o644
+                    )
                 ),
                 helpers.render_template(
                     source='ksplice_nrpe.cfg.j2',

=== modified file 'metadata.yaml'
--- metadata.yaml	2015-11-23 18:55:44 +0000
+++ metadata.yaml	2016-04-07 05:19:48 +0000
@@ -9,11 +9,11 @@
   - ops
 subordinate: true
 requires:
-#  general-info:
-#    interface: juju-info
-#    scope: container
-#provides:
-#
+  general-info:
+    interface: juju-info
+    scope: container
+provides:
+
   nrpe-external-master:
     interface: nrpe-external-master
     scope: container

=== added file 'templates/interfaces_nagios.cfg.j2'
--- templates/interfaces_nagios.cfg.j2	1970-01-01 00:00:00 +0000
+++ templates/interfaces_nagios.cfg.j2	2016-04-07 05:19:48 +0000
@@ -0,0 +1,22 @@
+#---------------------------------------------------
+# This file is Juju managed
+#---------------------------------------------------
+#
+{% macro servicegroup(default='juju') -%}
+    {%- if config['nagios_servicegroups'] -%}
+{{ config['nagios_servicegroups'] }}
+    {%- elif config['nagios_context'] -%}
+{{ config['nagios_context'] }}
+    {%- else -%}
+{{ default }}
+    {%- endif -%}
+{%- endmacro %}
+{% for rel in nrpe_external_master -%}
+define service {
+    use                             active-service
+    host_name                       {{ servicegroup() }}-{{ parent_unit }}
+    service_description             {{ servicegroup() }}-{{ parent_unit }}[check_interfaces] Check Network Interfaces 
+    check_command                   check_nrpe!check_interfaces
+    servicegroups                   {{ servicegroup() }}
+}
+{%- endfor %}

=== added file 'templates/interfaces_nrpe.cfg.j2'
--- templates/interfaces_nrpe.cfg.j2	1970-01-01 00:00:00 +0000
+++ templates/interfaces_nrpe.cfg.j2	2016-04-07 05:19:48 +0000
@@ -0,0 +1,6 @@
+#---------------------------------------------------
+# This file is Juju managed
+#---------------------------------------------------
+#
+# Check interfaces
+command[check_interfaces]=/usr/lib/nagios/plugins/interfaces_status.py --speed 10000MB/s