Merge lp:~jjo/charms/trusty/neutron-openvswitch/add-nrpe-checks-lp1530227 into lp:~openstack-charmers-archive/charms/trusty/neutron-openvswitch/next

[JuanJo Ciarlante]

Unmerged revisions

99. By JuanJo Ciarlante on 2016-01-07

[jjo] WIP: add NRPE support via NRPESet passed at relation time to nova-compute principal lp#1530227

=== modified file 'charm-helpers-hooks.yaml'
--- charm-helpers-hooks.yaml	2015-09-28 09:47:21 +0000
+++ charm-helpers-hooks.yaml	2016-01-07 21:40:19 +0000
@@ -11,3 +11,4 @@
     - payload.execd
     - contrib.network.ip
     - contrib.python.packages
+    - contrib.charmsupport

=== modified file 'config.yaml'
--- config.yaml	2015-09-15 07:47:30 +0000
+++ config.yaml	2016-01-07 21:40:19 +0000
@@ -106,3 +106,9 @@
       which do not include a neutron-gateway (do not require l3, lbaas or vpnaas
       services) and should only be used in-conjunction with flat or VLAN provider
       networks configurations.
+  enable-nrpe-checks:
+    type: boolean
+    default: true
+    description: |
+      Provide nrpe data to main charm (ie nova-compute) using 'neutron-plugin'
+      relation via the 'nrpe-checks' key

=== added directory 'files'
=== added directory 'files/nrpe-external-master'
=== added file 'files/nrpe-external-master/neutron-check-tun_ids.py'
--- files/nrpe-external-master/neutron-check-tun_ids.py	1970-01-01 00:00:00 +0000
+++ files/nrpe-external-master/neutron-check-tun_ids.py	2016-01-07 21:40:19 +0000
@@ -0,0 +1,219 @@
+#!/usr/bin/python
+# vim: si et sw=4 ts=4
+#
+# Author: JuanJo Ciarlante <[e-mail]>
+# Copyright (C) 2015 Canonical
+# License: GPLv3
+"""
+Verify that all local OVS tun flows match the ones expected from locally
+running nova instances
+
+Example usage:
+{0}
+{0} -i br-tun                                    # peek at other OVS interface
+{0} --conf-file=/etc/nova/nova.conf              # config file to peek creds
+
+"""
+import sys
+import os
+import re
+import logging
+import argparse
+import ConfigParser
+import socket
+import subprocess
+
+from neutronclient.v2_0 import client as neutron_client
+from novaclient import client as nova_client
+
+
+(STATUS_OK, STATUS_WARN, STATUS_CRIT, STATUS_UNKNOWN) = range(0, 4)
+
+
+def get_creds(args):
+    """ return creds dictionary from conf-file (/etc/nova/nova.conf),
+        overridden by OS_ environment vars """
+    config = ConfigParser.RawConfigParser()
+    config.read(args.conf_file)
+    config_creds_section_prefix = {
+        'DEFAULT': 'neutron_admin_',
+        'neutron': 'admin_',
+    }
+
+    creds = {}
+    for key in ('auth_url', 'username', 'password', 'tenant_name'):
+        env_key = "OS_{}".format(key.upper())
+        value = os.environ.get(env_key)
+        # If no creds from environment, try known possible config_creds_keys
+        if value:
+            logging.info("get_creds: found {} as env['{}']={}".format(
+                key, env_key, value if key != 'password' else '...'))
+        if not value:
+            for section, prefix in config_creds_section_prefix.iteritems():
+                try:
+                    value = config.get(section, prefix + key)
+                    logging.info("get_creds: found {}.{}={}".format(
+                        section, prefix + key,
+                        value if key != 'password' else '...'))
+                    break
+                except ConfigParser.NoOptionError:
+                    pass
+                except ConfigParser.NoSectionError:
+                    pass
+        if value:
+            creds[key] = value
+        else:
+            raise KeyError("Couldn't find config value for '{}'".format(key))
+
+    logging.debug("creds: username={username} tenant_name={tenant_name} "
+                  "auth_url={auth_url} password=...".format(**creds))
+
+    return creds
+
+
+def nova_list_instances(nova_cli, host):
+    "return instances ids running at host"
+    logging.info('getting all instances running at host="{}" ...'.format(
+        socket.gethostname()))
+    search_opts = {'all_tenants': 1, 'host': host}
+    instances = [server.id for server in
+                 nova_cli.servers.list(search_opts=search_opts)
+                 if server.status == 'ACTIVE']
+    logging.info('instances count={}'.format(len(instances)))
+    logging.debug('instances: {}'.format(instances))
+    return instances
+
+
+def instances_port_nets(neutron_cli, instances):
+    "return instances ports attached network ids"
+    logging.info('getting all instances networks ...')
+    instances_nets = set()
+    for instance in instances:
+        for ports in neutron_cli.list_ports(
+                device_id=instance, fields=['id', 'network_id']).values():
+            for port in ports:
+                instances_nets.add(port['network_id'])
+    logging.info('instances networks count={}'.format(len(instances_nets)))
+    logging.debug('instances networks: {}'.format(instances_nets))
+    return instances_nets
+
+
+def neutron_networks_by_id(neutron_cli):
+    "return all neutron networks, keyed by id"
+    logging.info('getting all neutron networks ...')
+    all_nets = neutron_cli.list_networks().get('networks')
+    logging.info('neutron networks count={}'.format(len(all_nets)))
+    logging.debug('neutron networks: {}'.format(all_nets))
+    networks_by_id = {net['id']: net for net in all_nets}
+    return networks_by_id
+
+
+def get_instances_tun_ids(instances_nets, all_nets_by_id):
+    """return tun_ids from for passed instances networks
+       by looking up all_nets_by_id info"""
+    logging.info('getting network segmentation_id info for all instances...')
+    SEG_ID = 'provider:segmentation_id'
+    NET_TYPE = 'provider:network_type'
+    instances_tun_ids = {all_nets_by_id[net_id].get(SEG_ID)
+                         for net_id in instances_nets
+                         if all_nets_by_id[net_id].get(NET_TYPE) in
+                         ('gre', 'vxlan')}
+    logging.info('instances_tun_ids: {}'.format(instances_tun_ids))
+    return instances_tun_ids
+
+
+def get_ovs_tun_ids(interface):
+    """get local tun_ids from ovs-ofctl output, ala:
+       ovs-ofctl dump-flows br-tun |egrep -o 'tun_id=\w+' """
+    logging.info('local tun_ids: running: ovs-ofctl dump-flows {}'.format(
+        interface))
+    ovs_dump = subprocess.Popen(["ovs-ofctl", "dump-flows", interface],
+                                stdin=None,
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE)
+    ovs_tun_ids = set()
+    # match lines with: ... tun_id=0x<TUN_ID> ...
+    for line in ovs_dump.stdout:
+        match = re.search("tun_id=(?P<tun_id>0x\w+)", line)
+        if match:
+            ovs_tun_ids.add(int(match.group(1), 16))
+    logging.info('ovs_tun_ids: {}'.format(ovs_tun_ids))
+    return ovs_tun_ids
+
+
+def nrpe_check_tun_ids(expected_tun_ids, local_tun_ids, all_nets_by_id):
+    # order is important: substract local_tun_ids from expected_tun_ids,
+    # result should be empty
+    tun_ids_diff = expected_tun_ids.difference(local_tun_ids)
+    rc = STATUS_OK
+    msg = []
+    if tun_ids_diff:
+        tun_ids_str = ' '.join(['tun_id=0x{0:x}'.format(x)
+                                for x in tun_ids_diff])
+        msg.append('CRITICAL: host={} missing local tun_ids: {}'.format(
+            socket.gethostname(), tun_ids_str))
+        # helper dict by tun_id
+        net_by_tun_id = {net_val.get('provider:segmentation_id'): net_val
+                         for net_id, net_val in all_nets_by_id.iteritems()}
+        for tun_id in tun_ids_diff:
+            net = net_by_tun_id.get(tun_id, {})
+            msg.append('CRITICAL: tun_id=0x{0:x} network.id={id} '
+                       'network.name="{name}"'.format(tun_id, **net))
+        logging.info('exp_tun_ids: {}'.format(sorted(expected_tun_ids)))
+        logging.info('loc_tun_ids: {}'.format(sorted(local_tun_ids)))
+        rc = STATUS_CRIT
+    else:
+        msg.append('OK: host={} all needed tun_ids present: {}'
+                   ''.format(socket.gethostname(), list(local_tun_ids)))
+    return (rc, msg)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description=__doc__.format(*sys.argv),
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('--conf-file', default='/etc/nova/nova.conf',
+                        help='config file to peek creds from')
+    parser.add_argument('-i', '--interface', default='br-tun',
+                        help='OVS iface where to find tun_ids, as: '
+                        'ovs-ofctl dump-flows <interface>')
+    parser.add_argument('--test', default=False, action='store_true',
+                        help='simulate missing local tun_ids, force CRITICAL')
+    parser.add_argument('--verbose', default=False, action='store_true')
+    parser.add_argument('--debug', default=False, action='store_true')
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+    args = parse_args()
+    if args.verbose:
+        logging.basicConfig(level=logging.INFO)
+    if args.debug:
+        logging.basicConfig(level=logging.DEBUG)
+
+    # initialize needed clients
+    creds = get_creds(args)
+    logging.info("initializing nova_client")
+    nova_cli = nova_client.Client(1.1, creds['username'], creds['password'],
+                                  creds['tenant_name'], creds['auth_url'])
+    logging.info("initializing neutron_client")
+    neutron_cli = neutron_client.Client(**creds)
+
+    # instances:      local instances id-s (ie running at this host)
+    # instances_nets: local instances' networks id-s
+    # all_nets_by_id: all neutron networks, keyed by id
+    # exp_tun_ids:    local instances' networks' segmentation_id-s
+    # loc_tun_ids:    locally present tun_ids from ovs-ofctl dump-flows br-tun
+    instances = nova_list_instances(nova_cli, socket.gethostname())
+    instances_nets = instances_port_nets(neutron_cli, instances)
+    all_nets_by_id = neutron_networks_by_id(neutron_cli)
+    exp_tun_ids = get_instances_tun_ids(instances_nets, all_nets_by_id)
+    loc_tun_ids = get_ovs_tun_ids(args.interface)
+
+    if args.test:
+        logging.info('TEST: remove a local tun_id'.format(args.interface))
+        loc_tun_ids.pop()
+
+    rc, msg = nrpe_check_tun_ids(exp_tun_ids, loc_tun_ids, all_nets_by_id)
+    print "\n".join(msg)
+    sys.exit(rc)

=== added directory 'hooks/charmhelpers/contrib/charmsupport'
=== added file 'hooks/charmhelpers/contrib/charmsupport/__init__.py'
--- hooks/charmhelpers/contrib/charmsupport/__init__.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/charmsupport/__init__.py	2016-01-07 21:40:19 +0000
@@ -0,0 +1,15 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.

=== added file 'hooks/charmhelpers/contrib/charmsupport/nrpe.py'
--- hooks/charmhelpers/contrib/charmsupport/nrpe.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/charmsupport/nrpe.py	2016-01-07 21:40:19 +0000
@@ -0,0 +1,458 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Compatibility with the nrpe-external-master charm"""
+# Copyright 2012 Canonical Ltd.
+#
+# Authors:
+#  Matthew Wedgwood <[e-mail]>
+
+import subprocess
+import pwd
+import grp
+import os
+import glob
+import shutil
+import re
+import shlex
+import yaml
+
+from charmhelpers.core.hookenv import (
+    config,
+    local_unit,
+    log,
+    relation_ids,
+    relation_set,
+    relations_of_type,
+)
+
+from charmhelpers.core.host import (
+    service,
+    rsync,
+)
+NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins'
+
+# This module adds compatibility with the nrpe-external-master and plain nrpe
+# subordinate charms. To use it in your charm:
+#
+# 1. Update metadata.yaml
+#
+#   provides:
+#     (...)
+#     nrpe-external-master:
+#       interface: nrpe-external-master
+#       scope: container
+#
+#   and/or
+#
+#   provides:
+#     (...)
+#     local-monitors:
+#       interface: local-monitors
+#       scope: container
+
+#
+# 2. Add the following to config.yaml
+#
+#    nagios_context:
+#      default: "juju"
+#      type: string
+#      description: |
+#        Used by the nrpe subordinate charms.
+#        A string that will be prepended to instance name to set the host name
+#        in nagios. So for instance the hostname would be something like:
+#            juju-myservice-0
+#        If you're running multiple environments with the same services in them
+#        this allows you to differentiate between them.
+#    nagios_servicegroups:
+#      default: ""
+#      type: string
+#      description: |
+#        A comma-separated list of nagios servicegroups.
+#        If left empty, the nagios_context will be used as the servicegroup
+#
+# 3. Add custom checks (Nagios plugins) to files/nrpe-external-master
+#
+# 4. Update your hooks.py with something like this:
+#
+#    from charmsupport.nrpe import NRPE
+#    (...)
+#    def update_nrpe_config():
+#        nrpe_compat = NRPE()
+#        nrpe_compat.add_check(
+#            shortname = "myservice",
+#            description = "Check MyService",
+#            check_cmd = "check_http -w 2 -c 10 http://localhost"
+#            )
+#        nrpe_compat.add_check(
+#            "myservice_other",
+#            "Check for widget failures",
+#            check_cmd = "/srv/myapp/scripts/widget_check"
+#            )
+#        nrpe_compat.write()
+#
+#    def config_changed():
+#        (...)
+#        update_nrpe_config()
+#
+#    def nrpe_external_master_relation_changed():
+#        update_nrpe_config()
+#
+#    def local_monitors_relation_changed():
+#        update_nrpe_config()
+#
+# 5. ln -s hooks.py nrpe-external-master-relation-changed
+#    ln -s hooks.py local-monitors-relation-changed
+
+
+class CheckException(Exception):
+    pass
+
+
+class Check(object):
+    shortname_re = '[A-Za-z0-9-_]+$'
+    service_template = ("""
+#---------------------------------------------------
+# This file is Juju managed
+#---------------------------------------------------
+define service {{
+    use                             active-service
+    host_name                       {nagios_hostname}
+    service_description             {nagios_hostname}[{shortname}] """
+                        """{description}
+    check_command                   check_nrpe!{command}
+    servicegroups                   {nagios_servicegroup}
+}}
+""")
+
+    def __init__(self, shortname, description, check_cmd):
+        super(Check, self).__init__()
+        # XXX: could be better to calculate this from the service name
+        if not re.match(self.shortname_re, shortname):
+            raise CheckException("shortname must match {}".format(
+                Check.shortname_re))
+        self.shortname = shortname
+        self.command = "check_{}".format(shortname)
+        # Note: a set of invalid characters is defined by the
+        # Nagios server config
+        # The default is: illegal_object_name_chars=`~!$%^&*"|'<>?,()=
+        self.description = description
+        self.check_cmd = self._locate_cmd(check_cmd)
+
+    def _locate_cmd(self, check_cmd):
+        search_path = (
+            '/usr/lib/nagios/plugins',
+            '/usr/local/lib/nagios/plugins',
+        )
+        parts = shlex.split(check_cmd)
+        for path in search_path:
+            if os.path.exists(os.path.join(path, parts[0])):
+                command = os.path.join(path, parts[0])
+                if len(parts) > 1:
+                    command += " " + " ".join(parts[1:])
+                return command
+        log('Check command not found: {}'.format(parts[0]))
+        return ''
+
+    def write(self, nagios_context, hostname, nagios_servicegroups):
+        nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format(
+            self.command)
+        with open(nrpe_check_file, 'w') as nrpe_check_config:
+            nrpe_check_config.write("# check {}\n".format(self.shortname))
+            nrpe_check_config.write("command[{}]={}\n".format(
+                self.command, self.check_cmd))
+
+        if not os.path.exists(NRPE.nagios_exportdir):
+            log('Not writing service config as {} is not accessible'.format(
+                NRPE.nagios_exportdir))
+        else:
+            self.write_service_config(nagios_context, hostname,
+                                      nagios_servicegroups)
+
+    def write_service_config(self, nagios_context, hostname,
+                             nagios_servicegroups):
+        for f in os.listdir(NRPE.nagios_exportdir):
+            if re.search('.*{}.cfg'.format(self.command), f):
+                os.remove(os.path.join(NRPE.nagios_exportdir, f))
+
+        templ_vars = {
+            'nagios_hostname': hostname,
+            'nagios_servicegroup': nagios_servicegroups,
+            'description': self.description,
+            'shortname': self.shortname,
+            'command': self.command,
+        }
+        nrpe_service_text = Check.service_template.format(**templ_vars)
+        nrpe_service_file = '{}/service__{}_{}.cfg'.format(
+            NRPE.nagios_exportdir, hostname, self.command)
+        with open(nrpe_service_file, 'w') as nrpe_service_config:
+            nrpe_service_config.write(str(nrpe_service_text))
+
+    def run(self):
+        subprocess.call(self.check_cmd)
+
+
+class Cron(object):
+    def __init__(self, filename, cron_freq, cron_user, cron_cmd):
+        super(Cron, self).__init__()
+        self.filename = filename
+        self.cron_freq = cron_freq
+        self.cron_user = cron_user
+        self.cron_cmd = cron_cmd
+
+    def write(self):
+        cron_filename = '/etc/cron.d/{}'.format(self.filename)
+        log("cron.write: {}".format(cron_filename))
+        with open(cron_filename, 'w') as cron_file:
+            cron_file.write("# cron {}\n".format(self.filename))
+            cron_file.write("{} {} {}\n".format(self.cron_freq,
+                                                self.cron_user,
+                                                self.cron_cmd))
+
+class NagiosPlugin(object):
+    def __init__(self, filename):
+        super(NagiosPlugin, self).__init__()
+        self.filename = filename
+
+    def write(self):
+        if not os.path.exists(NAGIOS_PLUGINS):
+            os.makedirs(NAGIOS_PLUGINS)
+        if os.path.exists(self.filename):
+            log("NagiosPlugin.write: {} {}".format(self.filename, NAGIOS_PLUGINS))
+            rsync(self.filename, NAGIOS_PLUGINS)
+        else:
+            log("SKIPPED: NagiosPlugin.write: {} {}".format(self.filename, NAGIOS_PLUGINS))
+
+
+class NRPE(object):
+    nagios_logdir = '/var/log/nagios'
+    nagios_exportdir = '/var/lib/nagios/export'
+    nrpe_confdir = '/etc/nagios/nrpe.d'
+
+    def __init__(self, hostname=None):
+        super(NRPE, self).__init__()
+        self.config = config()
+        self.nagios_context = self.config['nagios_context']
+        if 'nagios_servicegroups' in self.config and self.config['nagios_servicegroups']:
+            self.nagios_servicegroups = self.config['nagios_servicegroups']
+        else:
+            self.nagios_servicegroups = self.nagios_context
+        self.unit_name = local_unit().replace('/', '-')
+        if hostname:
+            self.hostname = hostname
+        else:
+            self.hostname = "{}-{}".format(self.nagios_context, self.unit_name)
+        self.checks = []
+        self.crons = []
+        self.nagios_plugins = []
+
+    def add_check(self, *args, **kwargs):
+        self.checks.append(Check(*args, **kwargs))
+
+    def add_cron(self, *args, **kwargs):
+        self.crons.append(Cron(*args, **kwargs))
+
+    def add_nagios_plugin(self, *args, **kwargs):
+        self.nagios_plugins.append(NagiosPlugin(*args, **kwargs))
+
+    def add_from_config(self, config_key):
+        saved_config = config()
+        nrpe_set = NRPESet(saved_config.get(config_key))
+        log('NRPE.add_from_config: nrpe_set={}'.format(str(nrpe_set)))
+        if nrpe_set:
+            for check in nrpe_set.checks:
+                self.add_check(*check)
+            for cron in nrpe_set.crons:
+                self.add_cron(*cron)
+            for nagios_plugins in nrpe_set.nagios_plugins:
+                self.add_nagios_plugin(*nagios_plugins)
+
+    def write(self):
+        try:
+            nagios_uid = pwd.getpwnam('nagios').pw_uid
+            nagios_gid = grp.getgrnam('nagios').gr_gid
+        except:
+            log("Nagios user not set up, nrpe checks not updated")
+            return
+
+        if not os.path.exists(NRPE.nagios_logdir):
+            os.mkdir(NRPE.nagios_logdir)
+            os.chown(NRPE.nagios_logdir, nagios_uid, nagios_gid)
+
+        nrpe_monitors = {}
+        monitors = {"monitors": {"remote": {"nrpe": nrpe_monitors}}}
+        for nrpecheck in self.checks:
+            nrpecheck.write(self.nagios_context, self.hostname,
+                            self.nagios_servicegroups)
+            nrpe_monitors[nrpecheck.shortname] = {
+                "command": nrpecheck.command,
+            }
+
+        for cron in self.crons:
+            cron.write()
+
+        for nagios_plugins in self.nagios_plugins:
+            nagios_plugins.write()
+
+        service('restart', 'nagios-nrpe-server')
+
+        monitor_ids = relation_ids("local-monitors") + \
+            relation_ids("nrpe-external-master")
+        for rid in monitor_ids:
+            relation_set(relation_id=rid, monitors=yaml.dump(monitors))
+
+
+def get_nagios_hostcontext(relation_name='nrpe-external-master'):
+    """
+    Query relation with nrpe subordinate, return the nagios_host_context
+
+    :param str relation_name: Name of relation nrpe sub joined to
+    """
+    for rel in relations_of_type(relation_name):
+        if 'nagios_hostname' in rel:
+            return rel['nagios_host_context']
+
+
+def get_nagios_hostname(relation_name='nrpe-external-master'):
+    """
+    Query relation with nrpe subordinate, return the nagios_hostname
+
+    :param str relation_name: Name of relation nrpe sub joined to
+    """
+    for rel in relations_of_type(relation_name):
+        if 'nagios_hostname' in rel:
+            return rel['nagios_hostname']
+
+
+def get_nagios_unit_name(relation_name='nrpe-external-master'):
+    """
+    Return the nagios unit name prepended with host_context if needed
+
+    :param str relation_name: Name of relation nrpe sub joined to
+    """
+    host_context = get_nagios_hostcontext(relation_name)
+    if host_context:
+        unit = "%s:%s" % (host_context, local_unit())
+    else:
+        unit = local_unit()
+    return unit
+
+
+def add_init_service_checks(nrpe, services, unit_name):
+    """
+    Add checks for each service in list
+
+    :param NRPE nrpe: NRPE object to add check to
+    :param list services: List of services to check
+    :param str unit_name: Unit name to use in check description
+    """
+    for svc in services:
+        upstart_init = '/etc/init/%s.conf' % svc
+        sysv_init = '/etc/init.d/%s' % svc
+        if os.path.exists(upstart_init):
+            nrpe.add_check(
+                shortname=svc,
+                description='process check {%s}' % unit_name,
+                check_cmd='check_upstart_job %s' % svc
+            )
+        elif os.path.exists(sysv_init):
+            cronpath = '/etc/cron.d/nagios-service-check-%s' % svc
+            cron_file = ('*/5 * * * * root '
+                         '/usr/local/lib/nagios/plugins/check_exit_status.pl '
+                         '-s /etc/init.d/%s status > '
+                         '/var/lib/nagios/service-check-%s.txt\n' % (svc,
+                                                                     svc)
+                         )
+            f = open(cronpath, 'w')
+            f.write(cron_file)
+            f.close()
+            nrpe.add_check(
+                shortname=svc,
+                description='process check {%s}' % unit_name,
+                check_cmd='check_status_file.py -f '
+                          '/var/lib/nagios/service-check-%s.txt' % svc,
+            )
+
+
+def copy_nrpe_checks():
+    """
+    Copy the nrpe checks into place
+
+    """
+    nrpe_files_dir = os.path.join(os.getenv('CHARM_DIR'), 'hooks',
+                                  'charmhelpers', 'contrib', 'openstack',
+                                  'files')
+
+    if not os.path.exists(NAGIOS_PLUGINS):
+        os.makedirs(NAGIOS_PLUGINS)
+    for fname in glob.glob(os.path.join(nrpe_files_dir, "check_*")):
+        if os.path.isfile(fname):
+            shutil.copy2(fname,
+                         os.path.join(NAGIOS_PLUGINS, os.path.basename(fname)))
+
+
+def add_haproxy_checks(nrpe, unit_name):
+    """
+    Add checks for each service in list
+
+    :param NRPE nrpe: NRPE object to add check to
+    :param str unit_name: Unit name to use in check description
+    """
+    nrpe.add_check(
+        shortname='haproxy_servers',
+        description='Check HAProxy {%s}' % unit_name,
+        check_cmd='check_haproxy.sh')
+    nrpe.add_check(
+        shortname='haproxy_queue',
+        description='Check HAProxy queue depth {%s}' % unit_name,
+        check_cmd='check_haproxy_queue_depth.sh')
+
+class NRPESet:
+    checks = []
+    crons = []
+    nagios_plugins = []
+    def __init__(self, yaml_str='null'):
+        init_values = None
+        if yaml_str:
+            init_values = yaml.safe_load(yaml_str)
+        log('NRPESet: init_values={}'.format(init_values))
+        if type(init_values) == type({}):
+            self.checks = init_values.get('checks', [])
+            self.crons = init_values.get('crons', [])
+            self.nagios_plugins = init_values.get('nagios_plugins', [])
+
+    def add_check(self, shortname, description, check_cmd):
+        log('NRPESet: add_check({})'.format((shortname, description, check_cmd)))
+        self.checks.append((shortname, description, check_cmd))
+
+    def add_init_service_checks(self, services, unit_name=None):
+        if not unit_name:
+            unit_name = local_unit().replace('/', '-')
+        log('NRPESet: add_init_service_checks({}, {})'.format(services, unit_name))
+        add_init_service_checks(self, services, unit_name)
+
+    def add_cron(self, filename, cron_freq, cron_user, cron_cmd):
+        log('NRPESet: add_cron({}, ...)'.format(filename))
+        self.crons.append((filename, cron_freq, cron_user, cron_cmd))
+
+    def add_nagios_plugin(self, filename):
+        log('NRPESet: add_nagios_plugin({})'.format(filename))
+        self.nagios_plugins.append((filename,))
+
+    def __str__(self):
+        return yaml.safe_dump({'checks': self.checks,
+                          'crons': self.crons,
+                          'nagios_plugins': self.nagios_plugins})

=== added file 'hooks/charmhelpers/contrib/charmsupport/volumes.py'
--- hooks/charmhelpers/contrib/charmsupport/volumes.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/charmsupport/volumes.py	2016-01-07 21:40:19 +0000
@@ -0,0 +1,175 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+'''
+Functions for managing volumes in juju units. One volume is supported per unit.
+Subordinates may have their own storage, provided it is on its own partition.
+
+Configuration stanzas::
+
+  volume-ephemeral:
+    type: boolean
+    default: true
+    description: >
+      If false, a volume is mounted as sepecified in "volume-map"
+      If true, ephemeral storage will be used, meaning that log data
+         will only exist as long as the machine. YOU HAVE BEEN WARNED.
+  volume-map:
+    type: string
+    default: {}
+    description: >
+      YAML map of units to device names, e.g:
+        "{ rsyslog/0: /dev/vdb, rsyslog/1: /dev/vdb }"
+      Service units will raise a configure-error if volume-ephemeral
+      is 'true' and no volume-map value is set. Use 'juju set' to set a
+      value and 'juju resolved' to complete configuration.
+
+Usage::
+
+    from charmsupport.volumes import configure_volume, VolumeConfigurationError
+    from charmsupport.hookenv import log, ERROR
+    def post_mount_hook():
+        stop_service('myservice')
+    def post_mount_hook():
+        start_service('myservice')
+
+    if __name__ == '__main__':
+        try:
+            configure_volume(before_change=pre_mount_hook,
+                             after_change=post_mount_hook)
+        except VolumeConfigurationError:
+            log('Storage could not be configured', ERROR)
+
+'''
+
+# XXX: Known limitations
+# - fstab is neither consulted nor updated
+
+import os
+from charmhelpers.core import hookenv
+from charmhelpers.core import host
+import yaml
+
+
+MOUNT_BASE = '/srv/juju/volumes'
+
+
+class VolumeConfigurationError(Exception):
+    '''Volume configuration data is missing or invalid'''
+    pass
+
+
+def get_config():
+    '''Gather and sanity-check volume configuration data'''
+    volume_config = {}
+    config = hookenv.config()
+
+    errors = False
+
+    if config.get('volume-ephemeral') in (True, 'True', 'true', 'Yes', 'yes'):
+        volume_config['ephemeral'] = True
+    else:
+        volume_config['ephemeral'] = False
+
+    try:
+        volume_map = yaml.safe_load(config.get('volume-map', '{}'))
+    except yaml.YAMLError as e:
+        hookenv.log("Error parsing YAML volume-map: {}".format(e),
+                    hookenv.ERROR)
+        errors = True
+    if volume_map is None:
+        # probably an empty string
+        volume_map = {}
+    elif not isinstance(volume_map, dict):
+        hookenv.log("Volume-map should be a dictionary, not {}".format(
+            type(volume_map)))
+        errors = True
+
+    volume_config['device'] = volume_map.get(os.environ['JUJU_UNIT_NAME'])
+    if volume_config['device'] and volume_config['ephemeral']:
+        # asked for ephemeral storage but also defined a volume ID
+        hookenv.log('A volume is defined for this unit, but ephemeral '
+                    'storage was requested', hookenv.ERROR)
+        errors = True
+    elif not volume_config['device'] and not volume_config['ephemeral']:
+        # asked for permanent storage but did not define volume ID
+        hookenv.log('Ephemeral storage was requested, but there is no volume '
+                    'defined for this unit.', hookenv.ERROR)
+        errors = True
+
+    unit_mount_name = hookenv.local_unit().replace('/', '-')
+    volume_config['mountpoint'] = os.path.join(MOUNT_BASE, unit_mount_name)
+
+    if errors:
+        return None
+    return volume_config
+
+
+def mount_volume(config):
+    if os.path.exists(config['mountpoint']):
+        if not os.path.isdir(config['mountpoint']):
+            hookenv.log('Not a directory: {}'.format(config['mountpoint']))
+            raise VolumeConfigurationError()
+    else:
+        host.mkdir(config['mountpoint'])
+    if os.path.ismount(config['mountpoint']):
+        unmount_volume(config)
+    if not host.mount(config['device'], config['mountpoint'], persist=True):
+        raise VolumeConfigurationError()
+
+
+def unmount_volume(config):
+    if os.path.ismount(config['mountpoint']):
+        if not host.umount(config['mountpoint'], persist=True):
+            raise VolumeConfigurationError()
+
+
+def managed_mounts():
+    '''List of all mounted managed volumes'''
+    return filter(lambda mount: mount[0].startswith(MOUNT_BASE), host.mounts())
+
+
+def configure_volume(before_change=lambda: None, after_change=lambda: None):
+    '''Set up storage (or don't) according to the charm's volume configuration.
+       Returns the mount point or "ephemeral". before_change and after_change
+       are optional functions to be called if the volume configuration changes.
+    '''
+
+    config = get_config()
+    if not config:
+        hookenv.log('Failed to read volume configuration', hookenv.CRITICAL)
+        raise VolumeConfigurationError()
+
+    if config['ephemeral']:
+        if os.path.ismount(config['mountpoint']):
+            before_change()
+            unmount_volume(config)
+            after_change()
+        return 'ephemeral'
+    else:
+        # persistent storage
+        if os.path.ismount(config['mountpoint']):
+            mounts = dict(managed_mounts())
+            if mounts.get(config['mountpoint']) != config['device']:
+                before_change()
+                unmount_volume(config)
+                mount_volume(config)
+                after_change()
+        else:
+            before_change()
+            mount_volume(config)
+            after_change()
+        return config['mountpoint']

=== modified file 'hooks/neutron_ovs_hooks.py'
--- hooks/neutron_ovs_hooks.py	2015-11-12 09:33:27 +0000
+++ hooks/neutron_ovs_hooks.py	2016-01-07 21:40:19 +0000
@@ -1,5 +1,6 @@
 #!/usr/bin/python
 
+import os
 import sys
 
 from copy import deepcopy
@@ -16,6 +17,7 @@
     log,
     relation_set,
     relation_ids,
+    local_unit,
 )
 
 from charmhelpers.core.host import (
@@ -45,6 +47,7 @@
     REQUIRED_INTERFACES,
     check_optional_relations,
 )
+from charmhelpers.contrib.charmsupport import nrpe
 
 hooks = Hooks()
 CONFIGS = register_configs()
@@ -72,6 +75,10 @@
     for rid in relation_ids('neutron-plugin'):
         neutron_plugin_joined(relation_id=rid)
 
+    if config_value_changed('enable-nrpe-checks'):
+        for rid in relation_ids('neutron-plugin'):
+            neutron_plugin_joined(rid)
+
 
 @hooks.hook('neutron-plugin-api-relation-changed')
 @restart_on_change(restart_map())
@@ -87,6 +94,37 @@
         neutron_plugin_joined(relation_id=rid)
 
 
+def neutron_plugin_nrpe_checks():
+    log('neutron_plugin_nrpe_checks: enable-nrpe-checks={}'.format(
+        config('enable-nrpe-checks')))
+    if not config('enable-nrpe-checks'):
+        return ''
+    current_unit = local_unit().replace('/', '-')
+    nrpe_set = nrpe.NRPESet()
+    nrpe_set.add_nagios_plugin(
+        os.path.join(os.getenv('CHARM_DIR'),
+                     'files', 'nrpe-external-master',
+                     'neutron-check-tun_ids.py'))
+    nrpe_set.add_cron(
+        filename='nagios-check-tun_ids',
+        cron_freq='*/5 * * * *',
+        cron_user='root',
+        cron_cmd='{}/neutron-check-tun_ids.py > '
+                 '/var/lib/nagios/neutron-check-tun_ids.txt'
+                 ''.format(nrpe.NAGIOS_PLUGINS),
+    )
+    nrpe_set.add_check(
+        shortname='neutron_tun_ids',
+        description='Check neutron ovs tun_ids {%s}' % current_unit,
+        check_cmd='check_status_file.py -f '
+                  '/var/lib/nagios/neutron-check-tun_ids.txt'
+    )
+    nrpe_set.add_init_service_checks(
+        ['openvswitch-switch', 'neutron-plugin-openvswitch-agent'])
+    log('neutron_plugin_nrpe_checks: nrpe_set={}'.format(str(nrpe_set)))
+    return str(nrpe_set)
+
+
 @hooks.hook('neutron-plugin-relation-joined')
 def neutron_plugin_joined(relation_id=None):
     if enable_local_dhcp():
@@ -100,8 +138,10 @@
             pkgs.extend(METADATA_PACKAGES)
         purge_packages(pkgs)
     secret = get_shared_secret() if enable_nova_metadata() else None
+    nrpe_checks = neutron_plugin_nrpe_checks()
     rel_data = {
         'metadata-shared-secret': secret,
+        'nrpe-checks': nrpe_checks,
     }
     relation_set(relation_id=relation_id, **rel_data)