Merge lp:~mthaddon/charms/precise/haproxy/get-upstream into lp:charms/haproxy

[Tom Haddon]

Include nrpe-external-master relation, and ability to inject pre-install hooks if necessary.

[lp:~mthaddon/charms/precise/haproxy/get-upstream]

71. By Tom Haddon on 2012-12-21

Remove illegal characters from description string

[Juan L. Negron (negronjl)]

Reviewing this now.

-Juan

[Juan L. Negron (negronjl)]

This seems to be sane.

Approving ... merging.

-Juan

review: Approve

=== modified file 'config.yaml'
--- config.yaml	2012-07-13 21:31:43 +0000
+++ config.yaml	2012-12-21 11:12:22 +0000
@@ -106,3 +106,13 @@
         before the first variable, service_name, as above. Service options is a
         comma separated list, server options will be appended as a string to
         the individual server lines for a given listen stanza.
+  nagios_context:
+    default: "juju"
+    type: string
+    description: |
+        Used by the nrpe-external-master subordinate charm.
+        A string that will be prepended to instance name to set the host name
+        in nagios. So for instance the hostname would be something like:
+            juju-postgresql-0
+        If you're running multiple environments with the same services in them
+        this allows you to differentiate between them.

=== added directory 'files'
=== added directory 'files/nrpe-external-master'
=== added file 'files/nrpe-external-master/check_haproxy.sh'
--- files/nrpe-external-master/check_haproxy.sh	1970-01-01 00:00:00 +0000
+++ files/nrpe-external-master/check_haproxy.sh	2012-12-21 11:12:22 +0000
@@ -0,0 +1,33 @@
+#!/bin/bash
+#--------------------------------------------
+# This file is managed by Juju
+#--------------------------------------------
+#                                       
+# Copyright 2009,2012 Canonical Ltd.
+# Author: Tom Haddon
+
+CRITICAL=0
+NOTACTIVE=''
+LOGFILE=/var/log/nagios/check_haproxy.log
+AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}')
+
+for appserver in $(grep '    server' /etc/haproxy/haproxy.cfg | awk '{print $2'});
+do
+    output=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 10000 --regex="class=\"active(2|3).*${appserver}" -e ' 200 OK')
+    if [ $? != 0 ]; then
+        date >> $LOGFILE
+        echo $output >> $LOGFILE
+        /usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 10000 -v | grep $appserver >> $LOGFILE 2>&1
+        CRITICAL=1
+        NOTACTIVE="${NOTACTIVE} $appserver"
+    fi
+done
+
+if [ $CRITICAL = 1 ]; then
+    echo "CRITICAL:${NOTACTIVE}"
+    exit 2
+fi
+
+echo "OK: All haproxy instances looking good"
+exit 0
+

=== added file 'files/nrpe-external-master/check_haproxy_queue_depth.sh'
--- files/nrpe-external-master/check_haproxy_queue_depth.sh	1970-01-01 00:00:00 +0000
+++ files/nrpe-external-master/check_haproxy_queue_depth.sh	2012-12-21 11:12:22 +0000
@@ -0,0 +1,30 @@
+#!/bin/bash
+#--------------------------------------------
+# This file is managed by Juju
+#--------------------------------------------
+#                                       
+# Copyright 2009,2012 Canonical Ltd.
+# Author: Tom Haddon
+
+# These should be config options at some stage
+CURRQthrsh=0
+MAXQthrsh=100
+
+AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}')
+
+HAPROXYSTATS=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 10000 -v)
+
+for BACKEND in $(echo $HAPROXYSTATS| xargs -n1 | grep BACKEND | awk -F , '{print $1}')
+do
+    CURRQ=$(echo "$HAPROXYSTATS" | grep $BACKEND | grep BACKEND | cut -d , -f 3)
+    MAXQ=$(echo "$HAPROXYSTATS"  | grep $BACKEND | grep BACKEND | cut -d , -f 4)
+
+    if [[ $CURRQ -gt $CURRQthrsh || $MAXQ -gt $MAXQthrsh ]] ; then
+        echo "CRITICAL: queue depth for $BACKEND - CURRENT:$CURRQ MAX:$MAXQ"
+        exit 2
+    fi
+done
+
+echo "OK: All haproxy queue depths looking good"
+exit 0
+

=== modified file 'hooks/hooks.py'
--- hooks/hooks.py	2012-11-20 15:50:24 +0000
+++ hooks/hooks.py	2012-12-21 11:12:22 +0000
@@ -10,6 +10,7 @@
 import subprocess
 import sys
 import yaml
+import nrpe
 
 
 ###############################################################################
@@ -511,6 +512,9 @@
 # Hook functions
 ###############################################################################
 def install_hook():
+    for f in glob.glob('exec.d/*/charm-pre-install'):
+        if os.path.isfile(f) and os.access(f, os.X_OK):
+            subprocess.check_call(['sh', '-c', f])
     if not os.path.exists(default_haproxy_service_config_dir):
         os.mkdir(default_haproxy_service_config_dir, 0600)
     return ((apt_get_install("haproxy") == enable_haproxy()) is True)
@@ -595,6 +599,12 @@
             open("%s/%s.is.proxy" %
             (default_haproxy_service_config_dir, service_name), 'a').close()
 
+def update_nrpe_config():
+    nrpe_compat = nrpe.NRPE()
+    nrpe_compat.add_check('haproxy','Check HAProxy', 'check_haproxy.sh')
+    nrpe_compat.add_check('haproxy_queue','Check HAProxy queue depth', 'check_haproxy_queue_depth.sh')
+    nrpe_compat.write()
+
 ###############################################################################
 # Main section
 ###############################################################################
@@ -602,6 +612,7 @@
     install_hook()
 elif hook_name == "config-changed":
     config_changed()
+    update_nrpe_config()
 elif hook_name == "start":
     start_hook()
 elif hook_name == "stop":
@@ -616,6 +627,8 @@
     website_interface("joined")
 elif hook_name == "website-relation-changed":
     website_interface("changed")
+elif hook_name == "nrpe-external-master-relation-changed":
+    update_nrpe_config()
 else:
     print "Unknown hook"
     sys.exit(1)

=== added symlink 'hooks/nrpe-external-master-relation-changed'
=== target is u'hooks.py'
=== added file 'hooks/nrpe.py'
--- hooks/nrpe.py	1970-01-01 00:00:00 +0000
+++ hooks/nrpe.py	2012-12-21 11:12:22 +0000
@@ -0,0 +1,170 @@
+import json
+import subprocess
+import pwd
+import grp
+import os
+import re
+import shlex
+
+# This module adds compatibility with the nrpe_external_master
+# subordinate charm. To use it in your charm:
+# 
+# 1. Update metadata.yaml
+#
+#   provides:
+#     (...)
+#     nrpe-external-master:
+#       interface: nrpe-external-master
+#           scope: container
+# 
+# 2. Add the following to config.yaml
+#
+#    nagios_context:
+#      default: "juju"
+#      type: string
+#      description: |
+#        Used by the nrpe-external-master subordinate charm.
+#        A string that will be prepended to instance name to set the host name
+#        in nagios. So for instance the hostname would be something like:
+#            juju-myservice-0
+#        If you're running multiple environments with the same services in them
+#        this allows you to differentiate between them.
+#
+# 3. Add custom checks (Nagios plugins) to files/nrpe-external-master
+#
+# 4. Update your hooks.py with something like this:
+#
+#    import nrpe
+#    (...)
+#    def update_nrpe_config():
+#        nrpe_compat = NRPE("myservice")
+#        nrpe_compat.add_check(
+#            shortname = "myservice",
+#            description = "Check MyService",
+#            check_cmd = "check_http -w 2 -c 10 http://localhost"
+#            )
+#        nrpe_compat.add_check(
+#            "myservice_other",
+#            "Check for widget failures",
+#            check_cmd = "/srv/myapp/scripts/widget_check"
+#            )
+#        nrpe_compat.write()
+#
+#    def config_changed():
+#        (...)
+#        update_nrpe_config()
+
+class ConfigurationError(Exception):
+    '''An error interacting with the Juju config'''
+    pass
+def config_get(scope=None):
+    '''Return the Juju config as a dictionary'''
+    try:
+        config_cmd_line = ['config-get']
+        if scope is not None:
+            config_cmd_line.append(scope)
+        config_cmd_line.append('--format=json')
+        return json.loads(subprocess.check_output(config_cmd_line))
+    except (ValueError, OSError, subprocess.CalledProcessError) as error:
+        subprocess.call(['juju-log', str(error)])
+        raise ConfigurationError(str(error))
+
+class CheckException(Exception): pass
+class Check(object):
+    shortname_re = '[A-Za-z0-9-_]*'
+    service_template = """
+#---------------------------------------------------
+# This file is Juju managed
+#---------------------------------------------------
+define service {{
+    use                             active-service
+    host_name                       {nagios_hostname}
+    service_description             {nagios_hostname} {shortname} {description}
+    check_command                   check_nrpe!check_{shortname}
+    servicegroups                   {nagios_servicegroup}
+}}
+"""
+    def __init__(self, shortname, description, check_cmd):
+        super(Check, self).__init__()
+        # XXX: could be better to calculate this from the service name
+        if not re.match(self.shortname_re, shortname):
+            raise CheckException("shortname must match {}".format(Check.shortname_re))
+        self.shortname = shortname
+        self.description = description
+        self.check_cmd = self._locate_cmd(check_cmd)
+
+    def _locate_cmd(self, check_cmd):
+        search_path = (
+            '/',
+            os.path.join(os.environ['CHARM_DIR'], 'files/nrpe-external-master'),
+            '/usr/lib/nagios/plugins',
+        )
+        command = shlex.split(check_cmd)
+        for path in search_path:
+            if os.path.exists(os.path.join(path,command[0])):
+                return os.path.join(path, command[0]) + " " + " ".join(command[1:])
+        subprocess.call(['juju-log', 'Check command not found: {}'.format(command[0])])
+        return ''
+
+    def write(self, nagios_context, hostname):
+        for f in os.listdir(NRPE.nagios_exportdir):
+            if re.search('.*check_{}.cfg'.format(self.shortname), f):
+                os.remove(os.path.join(NRPE.nagios_exportdir, f))
+
+        templ_vars = {
+            'nagios_hostname': hostname,
+            'nagios_servicegroup': nagios_context,
+            'description': self.description,
+            'shortname': self.shortname,
+        }
+        nrpe_service_text = Check.service_template.format(**templ_vars)
+        nrpe_service_file = '{}/service__{}_check_{}.cfg'.format(
+            NRPE.nagios_exportdir, hostname, self.shortname)
+        with open(nrpe_service_file, 'w') as nrpe_service_config:
+            nrpe_service_config.write(str(nrpe_service_text))
+
+        nrpe_check_file = '/etc/nagios/nrpe.d/check_{}.cfg'.format(self.shortname)
+        with open(nrpe_check_file, 'w') as nrpe_check_config:
+            nrpe_check_config.write("# check {}\n".format(self.shortname))
+            nrpe_check_config.write("command[check_{}]={}\n".format(
+                self.shortname, self.check_cmd))
+
+    def run(self):
+        subprocess.call(self.check_cmd)
+
+class NRPE(object):
+    nagios_logdir = '/var/log/nagios'
+    nagios_exportdir = '/var/lib/nagios/export'
+    nrpe_confdir = '/etc/nagios/nrpe.d'
+    def __init__(self):
+        super(NRPE, self).__init__()
+        self.config = config_get()
+        self.nagios_context = self.config['nagios_context']
+        self.unit_name = os.environ['JUJU_UNIT_NAME'].replace('/', '-')
+        self.hostname = "{}-{}".format(self.nagios_context, self.unit_name)
+        self.checks = []
+
+    def add_check(self, *args, **kwargs):
+        self.checks.append( Check(*args, **kwargs) )
+
+    def write(self):
+        try:
+            nagios_uid = pwd.getpwnam('nagios').pw_uid
+            nagios_gid = grp.getgrnam('nagios').gr_gid
+        except:
+            subprocess.call(['juju-log', "Nagios user not set up, nrpe checks not updated"])
+            return
+
+        if not os.path.exists(NRPE.nagios_exportdir):
+            subprocess.call(['juju-log', 'Exiting as {} is not accessible'.format(NRPE.nagios_exportdir)])
+            return
+
+        if not os.path.exists(NRPE.nagios_logdir):
+            os.mkdir(NRPE.nagios_logdir)
+            os.chown(NRPE.nagios_logdir, nagios_uid, nagios_gid)
+
+        for nrpecheck in self.checks:
+            nrpecheck.write(self.nagios_context, self.hostname)
+
+        if os.path.isfile('/etc/init.d/nagios-nrpe-server'):
+            subprocess.call(['service', 'nagios-nrpe-server', 'reload'])

=== modified file 'metadata.yaml'
--- metadata.yaml	2012-08-01 05:16:56 +0000
+++ metadata.yaml	2012-12-21 11:12:22 +0000
@@ -15,3 +15,6 @@
     interface: http
   munin:
     interface: munin-node
+  nrpe-external-master:
+    interface: nrpe-external-master
+    scope: container

=== modified file 'revision'
--- revision	2012-11-20 15:50:24 +0000
+++ revision	2012-12-21 11:12:22 +0000
@@ -1,1 +1,5 @@
+<<<<<<< TREE
 24
+=======
+31
+>>>>>>> MERGE-SOURCE

=== added directory 'templates'
=== added file 'templates/nrpe_service.tmpl'
--- templates/nrpe_service.tmpl	1970-01-01 00:00:00 +0000
+++ templates/nrpe_service.tmpl	2012-12-21 11:12:22 +0000
@@ -0,0 +1,10 @@
+#---------------------------------------------------
+# This file is Juju managed
+#---------------------------------------------------
+define service {
+    use                             active-service
+    host_name                       {{ nagios_hostname }}
+    service_description             {{ nagios_hostname }} {{ check.description }}
+    check_command                   check_nrpe!check_{{ check.shortname }}
+    servicegroups                   {{ nagios_servicegroup }}
+}