Merge lp:~abentley/charms/precise/charmworld/nagios into lp:~juju-jitsu/charms/precise/charmworld/trunk

[Aaron Bentley]

Implement nagios support

Change the way errors are reported so that any errors running ingest cause a
file, ~ubuntu/var/ingest-errors, to be created. The presence of this file is
used for a nagios check.

The files in charmsupport are copied almost verbatim from lp:charmsupport r27.
(Exception, the change in lp:~pjdc/charmsupport/hookenv-vs-nrpe was applied.)

Ideally we will switch to a packaged version of charmsupport, but not if its
nagios support is broken!

The shhh.py file is deleted because it is no longer needed.

https://codereview.appspot.com/7241058/

[Richard Harding (rharding)]

lgtm but I'd like to move the new files into a scripts directory. I've
started one in my branch as well. Things like the run-write-errors and
update-nrpe.py could find a home there and help prevent polluting the
root of the file tree.

https://codereview.appspot.com/7241058/

=== added directory 'charmsupport'
=== added file 'charmsupport/__init__.py'
=== added file 'charmsupport/hookenv.py'
--- charmsupport/hookenv.py	1970-01-01 00:00:00 +0000
+++ charmsupport/hookenv.py	2013-01-31 20:16:33 +0000
@@ -0,0 +1,150 @@
+"Interactions with the Juju environment"
+# source: 27:lp:charmsupport
+# Copyright 2012 Canonical Ltd.
+#
+# Authors:
+#  Matthew Wedgwood <[e-mail]>
+
+import os
+import json
+import yaml
+import subprocess
+
+CRITICAL = "CRITICAL"
+ERROR = "ERROR"
+WARNING = "WARNING"
+INFO = "INFO"
+DEBUG = "DEBUG"
+def log(message, level=DEBUG):
+    "Write a message to the juju log"
+    subprocess.call( [ 'juju-log', '-l', level, message ] )
+
+class Serializable(object):
+    "Wrapper, an object that can be serialized to yaml or json"
+    def __init__(self, obj):
+        # wrap the object
+        super(Serializable, self).__init__()
+        self._wrapped_obj = obj
+
+    def __getattr__(self, attr):
+        # see if this object has attr
+        if attr in self.__dict__:
+            return getattr(self, attr)
+        # proxy to the wrapped object
+        return self[attr]
+
+    def __getitem__(self, key):
+        return self._wrapped_obj[key]
+
+    def json(self):
+        "Serialize the object to json"
+        return json.dumps(self._wrapped_obj)
+
+    def yaml(self):
+        "Serialize the object to yaml"
+        return yaml.dump(self._wrapped_obj)
+
+def execution_environment():
+    """A convenient bundling of the current execution context"""
+    context = {}
+    context['conf'] = config()
+    context['unit'] = local_unit()
+    context['rel'] = relations_of_type()
+    context['env'] = os.environ
+    return context
+
+def in_relation_hook():
+    "Determine whether we're running in a relation hook"
+    return os.environ.has_key('JUJU_RELATION')
+
+def relation_type():
+    "The scope for the current relation hook"
+    return os.environ['JUJU_RELATION']
+def relation_id():
+    "The relation ID for the current relation hook"
+    return os.environ['JUJU_RELATION_ID']
+def local_unit():
+    "Local unit ID"
+    return os.environ['JUJU_UNIT_NAME']
+def remote_unit():
+    "The remote unit for the current relation hook"
+    return os.environ['JUJU_REMOTE_UNIT']
+
+def config(scope=None):
+    "Juju charm configuration"
+    config_cmd_line = ['config-get']
+    if scope is not None:
+        config_cmd_line.append(scope)
+    config_cmd_line.append('--format=json')
+    try:
+        config_data = json.loads(subprocess.check_output(config_cmd_line))
+    except (ValueError, OSError, subprocess.CalledProcessError) as err:
+        log(str(err), level=ERROR)
+        raise err
+    return Serializable(config_data)
+
+def relation_ids(reltype=None):
+    "A list of relation_ids"
+    reltype = reltype or relation_type()
+    relids = []
+    relid_cmd_line = ['relation-ids', '--format=json', reltype]
+    relids.extend(json.loads(subprocess.check_output(relid_cmd_line)))
+    return relids
+
+def related_units(relid=None):
+    "A list of related units"
+    relid = relid or relation_id()
+    units_cmd_line = ['relation-list', '--format=json', '-r', relid]
+    units = json.loads(subprocess.check_output(units_cmd_line))
+    return units
+
+def relation_for_unit(unit=None):
+    "Get the json represenation of a unit's relation"
+    unit = unit or remote_unit()
+    relation_cmd_line = ['relation-get', '--format=json', '-', unit]
+    try:
+        relation = json.loads(subprocess.check_output(relation_cmd_line))
+    except (ValueError, OSError, subprocess.CalledProcessError), err:
+        log(str(err), level=ERROR)
+        raise err
+    for key in relation:
+        if key.endswith('-list'):
+            relation[key] = relation[key].split()
+    relation['__unit__'] = unit
+    return Serializable(relation)
+
+def relations_for_id(relid=None):
+    "Get relations of a specific relation ID"
+    relation_data = []
+    relid = relid or relation_ids()
+    for unit in related_units(relid):
+        unit_data = relation_for_unit(unit)
+        unit_data['__relid__'] = relid
+        relation_data.append(unit_data)
+    return relation_data
+
+def relations_of_type(reltype=None):
+    "Get relations of a specific type"
+    relation_data = []
+    if in_relation_hook():
+        reltype = reltype or relation_type()
+        for relid in relation_ids(reltype):
+            for relation in relations_for_id(relid):
+                relation['__relid__'] = relid
+                relation_data.append(relation)
+    return relation_data
+
+class UnregisteredHookError(Exception): pass
+
+class Hooks(object):
+    def __init__(self):
+        super(Hooks, self).__init__()
+        self._hooks = {}
+    def register(self, name, function):
+        self._hooks[name] = function
+    def execute(self, args):
+        hook_name = os.path.basename(args[0])
+        if hook_name in self._hooks:
+            self._hooks[hook_name]()
+        else:
+            raise UnregisteredHookError(hook_name)
 
=== added file 'charmsupport/nrpe.py'
--- charmsupport/nrpe.py	1970-01-01 00:00:00 +0000
+++ charmsupport/nrpe.py	2013-01-31 20:16:33 +0000
@@ -0,0 +1,169 @@
+"""Compatibility with the nrpe-external-master charm"""
+# source: 27:lp:charmsupport
+# Copyright 2012 Canonical Ltd.
+#
+# Authors:
+#  Matthew Wedgwood <[e-mail]>
+
+import subprocess
+import pwd
+import grp
+import os
+import re
+import shlex
+
+from hookenv import config, local_unit
+
+# This module adds compatibility with the nrpe_external_master
+# subordinate charm. To use it in your charm:
+# 
+# 1. Update metadata.yaml
+#
+#   provides:
+#     (...)
+#     nrpe-external-master:
+#       interface: nrpe-external-master
+#           scope: container
+# 
+# 2. Add the following to config.yaml
+#
+#    nagios_context:
+#      default: "juju"
+#      type: string
+#      description: |
+#        Used by the nrpe-external-master subordinate charm.
+#        A string that will be prepended to instance name to set the host name
+#        in nagios. So for instance the hostname would be something like:
+#            juju-myservice-0
+#        If you're running multiple environments with the same services in them
+#        this allows you to differentiate between them.
+#
+# 3. Add custom checks (Nagios plugins) to files/nrpe-external-master
+#
+# 4. Update your hooks.py with something like this:
+#
+#    import nrpe
+#    (...)
+#    def update_nrpe_config():
+#        nrpe_compat = NRPE("myservice")
+#        nrpe_compat.add_check(
+#            shortname = "myservice",
+#            description = "Check MyService",
+#            check_cmd = "check_http -w 2 -c 10 http://localhost"
+#            )
+#        nrpe_compat.add_check(
+#            "myservice_other",
+#            "Check for widget failures",
+#            check_cmd = "/srv/myapp/scripts/widget_check"
+#            )
+#        nrpe_compat.write()
+#
+#    def config_changed():
+#        (...)
+#        update_nrpe_config()
+#    def nrpe_external_master_relation_changed():
+#        update_nrpe_config()
+#
+# 5. ln -s hooks.py nrpe-external-master-relation-changed
+
+class CheckException(Exception): pass
+class Check(object):
+    shortname_re = '[A-Za-z0-9-_]*'
+    service_template = """
+#---------------------------------------------------
+# This file is Juju managed
+#---------------------------------------------------
+define service {{
+    use                             active-service
+    host_name                       {nagios_hostname}
+    service_description             {nagios_hostname}[{shortname}] {description}
+    check_command                   check_nrpe!check_{shortname}
+    servicegroups                   {nagios_servicegroup}
+}}
+"""
+    def __init__(self, shortname, description, check_cmd):
+        super(Check, self).__init__()
+        # XXX: could be better to calculate this from the service name
+        if not re.match(self.shortname_re, shortname):
+            raise CheckException("shortname must match {}".format(Check.shortname_re))
+        self.shortname = shortname
+        # Note: a set of invalid characters is defined by the Nagios server config
+        # The default is: illegal_object_name_chars=`~!$%^&*"|'<>?,()=
+        self.description = description
+        self.check_cmd = self._locate_cmd(check_cmd)
+
+    def _locate_cmd(self, check_cmd):
+        search_path = (
+            '/',
+            os.path.join(os.environ['CHARM_DIR'], 'files/nrpe-external-master'),
+            '/usr/lib/nagios/plugins',
+        )
+        command = shlex.split(check_cmd)
+        for path in search_path:
+            if os.path.exists(os.path.join(path,command[0])):
+                return os.path.join(path, command[0]) + " " + " ".join(command[1:])
+        subprocess.call(['juju-log', 'Check command not found: {}'.format(command[0])])
+        return ''
+
+    def write(self, nagios_context, hostname):
+        for f in os.listdir(NRPE.nagios_exportdir):
+            if re.search('.*check_{}.cfg'.format(self.shortname), f):
+                os.remove(os.path.join(NRPE.nagios_exportdir, f))
+
+        templ_vars = {
+            'nagios_hostname': hostname,
+            'nagios_servicegroup': nagios_context,
+            'description': self.description,
+            'shortname': self.shortname,
+        }
+        nrpe_service_text = Check.service_template.format(**templ_vars)
+        nrpe_service_file = '{}/service__{}_check_{}.cfg'.format(
+            NRPE.nagios_exportdir, hostname, self.shortname)
+        with open(nrpe_service_file, 'w') as nrpe_service_config:
+            nrpe_service_config.write(str(nrpe_service_text))
+
+        nrpe_check_file = '/etc/nagios/nrpe.d/check_{}.cfg'.format(self.shortname)
+        with open(nrpe_check_file, 'w') as nrpe_check_config:
+            nrpe_check_config.write("# check {}\n".format(self.shortname))
+            nrpe_check_config.write("command[check_{}]={}\n".format(
+                self.shortname, self.check_cmd))
+
+    def run(self):
+        subprocess.call(self.check_cmd)
+
+class NRPE(object):
+    nagios_logdir = '/var/log/nagios'
+    nagios_exportdir = '/var/lib/nagios/export'
+    nrpe_confdir = '/etc/nagios/nrpe.d'
+    def __init__(self):
+        super(NRPE, self).__init__()
+        self.config = config()
+        self.nagios_context = self.config['nagios_context']
+        self.unit_name = local_unit().replace('/', '-')
+        self.hostname = "{}-{}".format(self.nagios_context, self.unit_name)
+        self.checks = []
+
+    def add_check(self, *args, **kwargs):
+        self.checks.append( Check(*args, **kwargs) )
+
+    def write(self):
+        try:
+            nagios_uid = pwd.getpwnam('nagios').pw_uid
+            nagios_gid = grp.getgrnam('nagios').gr_gid
+        except:
+            subprocess.call(['juju-log', "Nagios user not set up, nrpe checks not updated"])
+            return
+
+        if not os.path.exists(NRPE.nagios_exportdir):
+            subprocess.call(['juju-log', 'Exiting as {} is not accessible'.format(NRPE.nagios_exportdir)])
+            return
+
+        if not os.path.exists(NRPE.nagios_logdir):
+            os.mkdir(NRPE.nagios_logdir)
+            os.chown(NRPE.nagios_logdir, nagios_uid, nagios_gid)
+
+        for nrpecheck in self.checks:
+            nrpecheck.write(self.nagios_context, self.hostname)
+
+        if os.path.isfile('/etc/init.d/nagios-nrpe-server'):
+            subprocess.call(['service', 'nagios-nrpe-server', 'reload'])
 
=== modified file 'config.yaml'
--- config.yaml	2013-01-23 21:21:06 +0000
+++ config.yaml	2013-01-31 20:16:33 +0000
@@ -15,3 +15,13 @@
     type: string
     default: ""
     description: "Address to mail errors to."
+  nagios_context:
+    default: "juju"
+    type: string
+    description: |
+      Used by the nrpe-external-master subordinate charm.
+      A string that will be prepended to instance name to set the host name
+      in nagios. So for instance the hostname would be something like:
+          juju-myservice-0
+      If you're running multiple environments with the same services in them
+      this allows you to differentiate between them.
 
=== added directory 'files'
=== added directory 'files/nrpe-external-master'
=== added file 'files/nrpe-external-master/check_ingest.sh'
--- files/nrpe-external-master/check_ingest.sh	1970-01-01 00:00:00 +0000
+++ files/nrpe-external-master/check_ingest.sh	2013-01-31 20:16:33 +0000
@@ -0,0 +1,5 @@
+#!/bin/sh
+if [ -f /home/ubuntu/var/ingest-errors ]; then
+    echo Charmworld ingest failing
+    exit 1
+fi
 
=== modified file 'hooks/config-changed'
--- hooks/config-changed	2013-01-29 15:45:19 +0000
+++ hooks/config-changed	2013-01-31 20:16:33 +0000
@@ -79,9 +79,11 @@
 PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
 $mailto
 
-*/$interval * * * * ubuntu HOME=/home/ubuntu INI=$CONFIG_FILE ~ubuntu/shhh.py $project_dir/scripts/ingest
+*/$interval * * * * ubuntu HOME=/home/ubuntu INI=$CONFIG_FILE ~ubuntu/run-write-errors /home/ubuntu/var/ingest-errors $project_dir/scripts/ingest
 @daily ubuntu HOME=/home/ubuntu INI=$project_dir/production.ini $project_dir/bin/python $project_dir/charmworld/jobs/cstat.py
 
 EOF
 
 fi
+# nagios_context may have changed.
+./update-nrpe.py
 
=== modified file 'hooks/install'
--- hooks/install	2013-01-24 17:34:53 +0000
+++ hooks/install	2013-01-31 20:16:33 +0000
@@ -17,4 +17,4 @@
 postfix reload
 mkdir -p ~ubuntu/var/charms
 chown -R ubuntu.ubuntu ~ubuntu/var
-cp shhh.py ~ubuntu
+install -o ubuntu -g ubuntu run-write-errors ~ubuntu
 
=== added file 'hooks/nrpe-external-master-relation-changed'
--- hooks/nrpe-external-master-relation-changed	1970-01-01 00:00:00 +0000
+++ hooks/nrpe-external-master-relation-changed	2013-01-31 20:16:33 +0000
@@ -0,0 +1,2 @@
+#!/bin/sh
+./update-nrpe.py
 
=== modified file 'hooks/upgrade-charm'
--- hooks/upgrade-charm	2013-01-24 17:34:53 +0000
+++ hooks/upgrade-charm	2013-01-31 20:16:33 +0000
@@ -10,3 +10,6 @@
 fi
 hooks/config-changed
 hooks/restart
+if [ -f $HOME/shhh.py ]; then
+    rm $HOME/shhh.py
+fi
 
=== modified file 'metadata.yaml'
--- metadata.yaml	2012-08-20 19:57:46 +0000
+++ metadata.yaml	2013-01-31 20:16:33 +0000
@@ -6,6 +6,9 @@
 provides:
   website:
     interface: http
+  nrpe-external-master:
+    interface: nrpe-external-master
+    scope: container
 requires:
   database:
     interface: mongodb
 
=== modified file 'revision'
--- revision	2013-01-29 15:32:47 +0000
+++ revision	2013-01-31 20:16:33 +0000
@@ -1,1 +1,1 @@
-18
+19
 
=== added file 'run-write-errors'
--- run-write-errors	1970-01-01 00:00:00 +0000
+++ run-write-errors	2013-01-31 20:16:33 +0000
@@ -0,0 +1,16 @@
+#!/bin/sh
+# Run a command, and if it exits with non-zero status, leave its output behind
+# in the file specified.  Any existing file will be removed.
+error_file=$1
+shift
+"$@" > $error_file.tmp 2>&1
+status=$?
+if [ $status -ne 0 ]; then
+    mv $error_file.tmp $error_file
+else
+    rm $error_file.tmp
+    if [ -f $error_file ]; then
+        rm $error_file
+    fi
+fi
+exit $status
 
=== removed file 'shhh.py'
--- shhh.py	2013-01-24 21:00:17 +0000
+++ shhh.py	1970-01-01 00:00:00 +0000
@@ -1,81 +0,0 @@
-#! /usr/bin/python -S
-#
-# Copyright 2009 Canonical Ltd.  This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""
-Run a command and suppress output unless it returns a non-zero exit status
-"""
-
-__metaclass__ = type
-
-from subprocess import (
-    PIPE,
-    Popen,
-    )
-import sys
-
-
-def shhh(cmd):
-    r"""Run a command and suppress output unless it returns a non-zero exitcode
-
-    If output is generated, stderr will be output before stdout, so output
-    order may be messed up if the command attempts to control order by
-    flushing stdout at points or setting it to unbuffered.
-
-
-    To test, we invoke both this method and this script with some commands
-    and examine the output and exitvalue
-
-    >>> python = sys.executable
-
-    >>> def shhh_script(cmd):
-    ...     from subprocess import Popen, PIPE
-    ...     script = '%s %s' % (python, __file__)
-    ...     cmd = "%s '%s'" % (script, cmd)
-    ...     p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
-    ...     (out, err) = p.communicate()
-    ...     return (out, err, p.returncode)
-
-    >>> cmd = '''%s -c "import sys; sys.exit(%d)"''' % (python, 0)
-    >>> shhh(cmd)
-    0
-    >>> shhh_script(cmd)
-    ('', '', 0)
-
-    >>> cmd = '''%s -c "import sys; sys.exit(%d)"''' % (python, 1)
-    >>> shhh(cmd)
-    1
-    >>> shhh_script(cmd)
-    ('', '', 1)
-
-    >>> cmd = '''%s -c "import sys; print 666; sys.exit(%d)"''' % (
-    ...     python, 42)
-    >>> shhh(cmd)
-    666
-    42
-    >>> shhh_script(cmd)
-    ('666\n', '', 42)
-
-    >>> cmd = (
-    ...     '''%s -c "import sys; print 666; '''
-    ...     '''print >> sys.stderr, 667; sys.exit(42)"''' % python
-    ...     )
-    >>> shhh_script(cmd)
-    ('666\n', '667\n', 42)
-    """
-
-    process = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
-    (out, err) = process.communicate()
-    if process.returncode == 0:
-        return 0
-    else:
-        sys.stderr.write(err)
-        sys.stdout.write(out)
-        return process.returncode
-
-
-if __name__ == '__main__':
-    cmd = ' '.join(sys.argv[1:])
-    sys.exit(shhh(cmd))
-
 
=== added file 'update-nrpe.py'
--- update-nrpe.py	1970-01-01 00:00:00 +0000
+++ update-nrpe.py	2013-01-31 20:16:33 +0000
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+from charmsupport import nrpe
+
+
+def update_nrpe_config():
+    nrpe_compat = nrpe.NRPE()
+    nrpe_compat.add_check('ingest', 'Check ingest runs', 'check_ingest.sh')
+    nrpe_compat.write()
+
+
+if __name__ == '__main__':
+    update_nrpe_config()