Merge lp:~gnuoy/charms/trusty/nova-compute/add-nrpe-checks into lp:~openstack-charmers-archive/charms/trusty/nova-compute/next

[Liam Young]

Add nrpe support. Based on branch from bradm with a few tweaks

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #667 nova-compute-next for gnuoy mp246158
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/667/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #696 nova-compute-next for gnuoy mp246158
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/696/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #852 nova-compute-next for gnuoy mp246158
    AMULET OK: passed

Build: http://10.245.162.77:8080/job/charm_amulet_test/852/

[Liam Young (gnuoy)]

<jamespage> gnuoy, as they are re-syncs + tweaks to the nrpe stuff in the charms, I'm happy to give a conditional +1 across the board based on osci checking things out OK
<gnuoy> jamespage, I'll take that! thanks
...
<gnuoy> jamespage, osci is still working through. But on the subject of those mps, does your +1 stand for branches with no amulet tests?
<jamespage> gnuoy, yes

review: Approve

=== modified file 'charm-helpers-hooks.yaml'
--- charm-helpers-hooks.yaml	2014-12-11 14:51:25 +0000
+++ charm-helpers-hooks.yaml	2015-01-12 14:02:09 +0000
@@ -11,3 +11,4 @@
     - contrib.network
     - contrib.python.packages
     - payload.execd
+    - contrib.charmsupport

=== added directory 'files'
=== added directory 'hooks/charmhelpers/contrib/charmsupport'
=== added file 'hooks/charmhelpers/contrib/charmsupport/__init__.py'
=== added file 'hooks/charmhelpers/contrib/charmsupport/nrpe.py'
--- hooks/charmhelpers/contrib/charmsupport/nrpe.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/charmsupport/nrpe.py	2015-01-12 14:02:09 +0000
@@ -0,0 +1,308 @@
+"""Compatibility with the nrpe-external-master charm"""
+# Copyright 2012 Canonical Ltd.
+#
+# Authors:
+#  Matthew Wedgwood <[e-mail]>
+
+import subprocess
+import pwd
+import grp
+import os
+import re
+import shlex
+import yaml
+
+from charmhelpers.core.hookenv import (
+    config,
+    local_unit,
+    log,
+    relation_ids,
+    relation_set,
+    relations_of_type,
+)
+
+from charmhelpers.core.host import service
+
+# This module adds compatibility with the nrpe-external-master and plain nrpe
+# subordinate charms. To use it in your charm:
+#
+# 1. Update metadata.yaml
+#
+#   provides:
+#     (...)
+#     nrpe-external-master:
+#       interface: nrpe-external-master
+#       scope: container
+#
+#   and/or
+#
+#   provides:
+#     (...)
+#     local-monitors:
+#       interface: local-monitors
+#       scope: container
+
+#
+# 2. Add the following to config.yaml
+#
+#    nagios_context:
+#      default: "juju"
+#      type: string
+#      description: |
+#        Used by the nrpe subordinate charms.
+#        A string that will be prepended to instance name to set the host name
+#        in nagios. So for instance the hostname would be something like:
+#            juju-myservice-0
+#        If you're running multiple environments with the same services in them
+#        this allows you to differentiate between them.
+#    nagios_servicegroups:
+#      default: ""
+#      type: string
+#      description: |
+#        A comma-separated list of nagios servicegroups.
+#        If left empty, the nagios_context will be used as the servicegroup
+#
+# 3. Add custom checks (Nagios plugins) to files/nrpe-external-master
+#
+# 4. Update your hooks.py with something like this:
+#
+#    from charmsupport.nrpe import NRPE
+#    (...)
+#    def update_nrpe_config():
+#        nrpe_compat = NRPE()
+#        nrpe_compat.add_check(
+#            shortname = "myservice",
+#            description = "Check MyService",
+#            check_cmd = "check_http -w 2 -c 10 http://localhost"
+#            )
+#        nrpe_compat.add_check(
+#            "myservice_other",
+#            "Check for widget failures",
+#            check_cmd = "/srv/myapp/scripts/widget_check"
+#            )
+#        nrpe_compat.write()
+#
+#    def config_changed():
+#        (...)
+#        update_nrpe_config()
+#
+#    def nrpe_external_master_relation_changed():
+#        update_nrpe_config()
+#
+#    def local_monitors_relation_changed():
+#        update_nrpe_config()
+#
+# 5. ln -s hooks.py nrpe-external-master-relation-changed
+#    ln -s hooks.py local-monitors-relation-changed
+
+
+class CheckException(Exception):
+    pass
+
+
+class Check(object):
+    shortname_re = '[A-Za-z0-9-_]+$'
+    service_template = ("""
+#---------------------------------------------------
+# This file is Juju managed
+#---------------------------------------------------
+define service {{
+    use                             active-service
+    host_name                       {nagios_hostname}
+    service_description             {nagios_hostname}[{shortname}] """
+                        """{description}
+    check_command                   check_nrpe!{command}
+    servicegroups                   {nagios_servicegroup}
+}}
+""")
+
+    def __init__(self, shortname, description, check_cmd):
+        super(Check, self).__init__()
+        # XXX: could be better to calculate this from the service name
+        if not re.match(self.shortname_re, shortname):
+            raise CheckException("shortname must match {}".format(
+                Check.shortname_re))
+        self.shortname = shortname
+        self.command = "check_{}".format(shortname)
+        # Note: a set of invalid characters is defined by the
+        # Nagios server config
+        # The default is: illegal_object_name_chars=`~!$%^&*"|'<>?,()=
+        self.description = description
+        self.check_cmd = self._locate_cmd(check_cmd)
+
+    def _locate_cmd(self, check_cmd):
+        search_path = (
+            '/usr/lib/nagios/plugins',
+            '/usr/local/lib/nagios/plugins',
+        )
+        parts = shlex.split(check_cmd)
+        for path in search_path:
+            if os.path.exists(os.path.join(path, parts[0])):
+                command = os.path.join(path, parts[0])
+                if len(parts) > 1:
+                    command += " " + " ".join(parts[1:])
+                return command
+        log('Check command not found: {}'.format(parts[0]))
+        return ''
+
+    def write(self, nagios_context, hostname, nagios_servicegroups=None):
+        nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format(
+            self.command)
+        with open(nrpe_check_file, 'w') as nrpe_check_config:
+            nrpe_check_config.write("# check {}\n".format(self.shortname))
+            nrpe_check_config.write("command[{}]={}\n".format(
+                self.command, self.check_cmd))
+
+        if not os.path.exists(NRPE.nagios_exportdir):
+            log('Not writing service config as {} is not accessible'.format(
+                NRPE.nagios_exportdir))
+        else:
+            self.write_service_config(nagios_context, hostname,
+                                      nagios_servicegroups)
+
+    def write_service_config(self, nagios_context, hostname,
+                             nagios_servicegroups=None):
+        for f in os.listdir(NRPE.nagios_exportdir):
+            if re.search('.*{}.cfg'.format(self.command), f):
+                os.remove(os.path.join(NRPE.nagios_exportdir, f))
+
+        if not nagios_servicegroups:
+            nagios_servicegroups = nagios_context
+
+        templ_vars = {
+            'nagios_hostname': hostname,
+            'nagios_servicegroup': nagios_servicegroups,
+            'description': self.description,
+            'shortname': self.shortname,
+            'command': self.command,
+        }
+        nrpe_service_text = Check.service_template.format(**templ_vars)
+        nrpe_service_file = '{}/service__{}_{}.cfg'.format(
+            NRPE.nagios_exportdir, hostname, self.command)
+        with open(nrpe_service_file, 'w') as nrpe_service_config:
+            nrpe_service_config.write(str(nrpe_service_text))
+
+    def run(self):
+        subprocess.call(self.check_cmd)
+
+
+class NRPE(object):
+    nagios_logdir = '/var/log/nagios'
+    nagios_exportdir = '/var/lib/nagios/export'
+    nrpe_confdir = '/etc/nagios/nrpe.d'
+
+    def __init__(self, hostname=None):
+        super(NRPE, self).__init__()
+        self.config = config()
+        self.nagios_context = self.config['nagios_context']
+        if 'nagios_servicegroups' in self.config:
+            self.nagios_servicegroups = self.config['nagios_servicegroups']
+        else:
+            self.nagios_servicegroups = 'juju'
+        self.unit_name = local_unit().replace('/', '-')
+        if hostname:
+            self.hostname = hostname
+        else:
+            self.hostname = "{}-{}".format(self.nagios_context, self.unit_name)
+        self.checks = []
+
+    def add_check(self, *args, **kwargs):
+        self.checks.append(Check(*args, **kwargs))
+
+    def write(self):
+        try:
+            nagios_uid = pwd.getpwnam('nagios').pw_uid
+            nagios_gid = grp.getgrnam('nagios').gr_gid
+        except:
+            log("Nagios user not set up, nrpe checks not updated")
+            return
+
+        if not os.path.exists(NRPE.nagios_logdir):
+            os.mkdir(NRPE.nagios_logdir)
+            os.chown(NRPE.nagios_logdir, nagios_uid, nagios_gid)
+
+        nrpe_monitors = {}
+        monitors = {"monitors": {"remote": {"nrpe": nrpe_monitors}}}
+        for nrpecheck in self.checks:
+            nrpecheck.write(self.nagios_context, self.hostname,
+                            self.nagios_servicegroups)
+            nrpe_monitors[nrpecheck.shortname] = {
+                "command": nrpecheck.command,
+            }
+
+        service('restart', 'nagios-nrpe-server')
+
+        for rid in relation_ids("local-monitors"):
+            relation_set(relation_id=rid, monitors=yaml.dump(monitors))
+
+
+def get_nagios_hostcontext(relation_name='nrpe-external-master'):
+    """
+    Query relation with nrpe subordinate, return the nagios_host_context
+
+    :param str relation_name: Name of relation nrpe sub joined to
+    """
+    for rel in relations_of_type(relation_name):
+        if 'nagios_hostname' in rel:
+            return rel['nagios_host_context']
+
+
+def get_nagios_hostname(relation_name='nrpe-external-master'):
+    """
+    Query relation with nrpe subordinate, return the nagios_hostname
+
+    :param str relation_name: Name of relation nrpe sub joined to
+    """
+    for rel in relations_of_type(relation_name):
+        if 'nagios_hostname' in rel:
+            return rel['nagios_hostname']
+
+
+def get_nagios_unit_name(relation_name='nrpe-external-master'):
+    """
+    Return the nagios unit name prepended with host_context if needed
+
+    :param str relation_name: Name of relation nrpe sub joined to
+    """
+    host_context = get_nagios_hostcontext(relation_name)
+    if host_context:
+        unit = "%s:%s" % (host_context, local_unit())
+    else:
+        unit = local_unit()
+    return unit
+
+
+def add_init_service_checks(nrpe, services, unit_name):
+    """
+    Add checks for each service in list
+
+    :param NRPE nrpe: NRPE object to add check to
+    :param list services: List of services to check
+    :param str unit_name: Unit name to use in check description
+    """
+    for svc in services:
+        upstart_init = '/etc/init/%s.conf' % svc
+        sysv_init = '/etc/init.d/%s' % svc
+        if os.path.exists(upstart_init):
+            nrpe.add_check(
+                shortname=svc,
+                description='process check {%s}' % unit_name,
+                check_cmd='check_upstart_job %s' % svc
+            )
+        elif os.path.exists(sysv_init):
+            cronpath = '/etc/cron.d/nagios-service-check-%s' % svc
+            cron_file = ('*/5 * * * * root '
+                         '/usr/local/lib/nagios/plugins/check_exit_status.pl '
+                         '-s /etc/init.d/%s status > '
+                         '/var/lib/nagios/service-check-%s.txt\n' % (svc,
+                                                                     svc)
+                         )
+            f = open(cronpath, 'w')
+            f.write(cron_file)
+            f.close()
+            nrpe.add_check(
+                shortname=svc,
+                description='process check {%s}' % unit_name,
+                check_cmd='check_status_file.py -f '
+                          '/var/lib/nagios/service-check-%s.txt' % svc,
+            )

=== added file 'hooks/charmhelpers/contrib/charmsupport/volumes.py'
--- hooks/charmhelpers/contrib/charmsupport/volumes.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/charmsupport/volumes.py	2015-01-12 14:02:09 +0000
@@ -0,0 +1,159 @@
+'''
+Functions for managing volumes in juju units. One volume is supported per unit.
+Subordinates may have their own storage, provided it is on its own partition.
+
+Configuration stanzas::
+
+  volume-ephemeral:
+    type: boolean
+    default: true
+    description: >
+      If false, a volume is mounted as sepecified in "volume-map"
+      If true, ephemeral storage will be used, meaning that log data
+         will only exist as long as the machine. YOU HAVE BEEN WARNED.
+  volume-map:
+    type: string
+    default: {}
+    description: >
+      YAML map of units to device names, e.g:
+        "{ rsyslog/0: /dev/vdb, rsyslog/1: /dev/vdb }"
+      Service units will raise a configure-error if volume-ephemeral
+      is 'true' and no volume-map value is set. Use 'juju set' to set a
+      value and 'juju resolved' to complete configuration.
+
+Usage::
+
+    from charmsupport.volumes import configure_volume, VolumeConfigurationError
+    from charmsupport.hookenv import log, ERROR
+    def post_mount_hook():
+        stop_service('myservice')
+    def post_mount_hook():
+        start_service('myservice')
+
+    if __name__ == '__main__':
+        try:
+            configure_volume(before_change=pre_mount_hook,
+                             after_change=post_mount_hook)
+        except VolumeConfigurationError:
+            log('Storage could not be configured', ERROR)
+
+'''
+
+# XXX: Known limitations
+# - fstab is neither consulted nor updated
+
+import os
+from charmhelpers.core import hookenv
+from charmhelpers.core import host
+import yaml
+
+
+MOUNT_BASE = '/srv/juju/volumes'
+
+
+class VolumeConfigurationError(Exception):
+    '''Volume configuration data is missing or invalid'''
+    pass
+
+
+def get_config():
+    '''Gather and sanity-check volume configuration data'''
+    volume_config = {}
+    config = hookenv.config()
+
+    errors = False
+
+    if config.get('volume-ephemeral') in (True, 'True', 'true', 'Yes', 'yes'):
+        volume_config['ephemeral'] = True
+    else:
+        volume_config['ephemeral'] = False
+
+    try:
+        volume_map = yaml.safe_load(config.get('volume-map', '{}'))
+    except yaml.YAMLError as e:
+        hookenv.log("Error parsing YAML volume-map: {}".format(e),
+                    hookenv.ERROR)
+        errors = True
+    if volume_map is None:
+        # probably an empty string
+        volume_map = {}
+    elif not isinstance(volume_map, dict):
+        hookenv.log("Volume-map should be a dictionary, not {}".format(
+            type(volume_map)))
+        errors = True
+
+    volume_config['device'] = volume_map.get(os.environ['JUJU_UNIT_NAME'])
+    if volume_config['device'] and volume_config['ephemeral']:
+        # asked for ephemeral storage but also defined a volume ID
+        hookenv.log('A volume is defined for this unit, but ephemeral '
+                    'storage was requested', hookenv.ERROR)
+        errors = True
+    elif not volume_config['device'] and not volume_config['ephemeral']:
+        # asked for permanent storage but did not define volume ID
+        hookenv.log('Ephemeral storage was requested, but there is no volume '
+                    'defined for this unit.', hookenv.ERROR)
+        errors = True
+
+    unit_mount_name = hookenv.local_unit().replace('/', '-')
+    volume_config['mountpoint'] = os.path.join(MOUNT_BASE, unit_mount_name)
+
+    if errors:
+        return None
+    return volume_config
+
+
+def mount_volume(config):
+    if os.path.exists(config['mountpoint']):
+        if not os.path.isdir(config['mountpoint']):
+            hookenv.log('Not a directory: {}'.format(config['mountpoint']))
+            raise VolumeConfigurationError()
+    else:
+        host.mkdir(config['mountpoint'])
+    if os.path.ismount(config['mountpoint']):
+        unmount_volume(config)
+    if not host.mount(config['device'], config['mountpoint'], persist=True):
+        raise VolumeConfigurationError()
+
+
+def unmount_volume(config):
+    if os.path.ismount(config['mountpoint']):
+        if not host.umount(config['mountpoint'], persist=True):
+            raise VolumeConfigurationError()
+
+
+def managed_mounts():
+    '''List of all mounted managed volumes'''
+    return filter(lambda mount: mount[0].startswith(MOUNT_BASE), host.mounts())
+
+
+def configure_volume(before_change=lambda: None, after_change=lambda: None):
+    '''Set up storage (or don't) according to the charm's volume configuration.
+       Returns the mount point or "ephemeral". before_change and after_change
+       are optional functions to be called if the volume configuration changes.
+    '''
+
+    config = get_config()
+    if not config:
+        hookenv.log('Failed to read volume configuration', hookenv.CRITICAL)
+        raise VolumeConfigurationError()
+
+    if config['ephemeral']:
+        if os.path.ismount(config['mountpoint']):
+            before_change()
+            unmount_volume(config)
+            after_change()
+        return 'ephemeral'
+    else:
+        # persistent storage
+        if os.path.ismount(config['mountpoint']):
+            mounts = dict(managed_mounts())
+            if mounts.get(config['mountpoint']) != config['device']:
+                before_change()
+                unmount_volume(config)
+                mount_volume(config)
+                after_change()
+        else:
+            before_change()
+            mount_volume(config)
+            after_change()
+        return config['mountpoint']

=== modified file 'hooks/charmhelpers/contrib/hahelpers/cluster.py'
--- hooks/charmhelpers/contrib/hahelpers/cluster.py	2014-12-15 09:10:57 +0000
+++ hooks/charmhelpers/contrib/hahelpers/cluster.py	2015-01-12 14:02:09 +0000
@@ -13,6 +13,7 @@
 
 import subprocess
 import os
+
 from socket import gethostname as get_unit_hostname
 
 import six
@@ -28,12 +29,19 @@
     WARNING,
     unit_get,
 )
+from charmhelpers.core.decorators import (
+    retry_on_exception,
+)
 
 
 class HAIncompleteConfig(Exception):
     pass
 
 
+class CRMResourceNotFound(Exception):
+    pass
+
+
 def is_elected_leader(resource):
     """
     Returns True if the charm executing this is the elected cluster leader.
@@ -68,24 +76,30 @@
     return False
 
 
-def is_crm_leader(resource):
+@retry_on_exception(5, base_delay=2, exc_type=CRMResourceNotFound)
+def is_crm_leader(resource, retry=False):
     """
     Returns True if the charm calling this is the elected corosync leader,
     as returned by calling the external "crm" command.
+
+    We allow this operation to be retried to avoid the possibility of getting a
+    false negative. See LP #1396246 for more info.
     """
-    cmd = [
-        "crm", "resource",
-        "show", resource
-    ]
+    cmd = ['crm', 'resource', 'show', resource]
     try:
-        status = subprocess.check_output(cmd).decode('UTF-8')
+        status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+        if not isinstance(status, six.text_type):
+            status = six.text_type(status, "utf-8")
     except subprocess.CalledProcessError:
-        return False
-    else:
-        if get_unit_hostname() in status:
-            return True
-        else:
-            return False
+        status = None
+
+    if status and get_unit_hostname() in status:
+        return True
+
+    if status and "resource %s is NOT running" % (resource) in status:
+        raise CRMResourceNotFound("CRM resource %s not found" % (resource))
+
+    return False
 
 
 def is_leader(resource):

=== modified file 'hooks/charmhelpers/contrib/network/ufw.py'
--- hooks/charmhelpers/contrib/network/ufw.py	2014-12-10 20:28:56 +0000
+++ hooks/charmhelpers/contrib/network/ufw.py	2015-01-12 14:02:09 +0000
@@ -54,6 +54,17 @@
     if is_enabled():
         return True
 
+    if not os.path.isdir('/proc/sys/net/ipv6'):
+        # disable IPv6 support in ufw
+        hookenv.log("This machine doesn't have IPv6 enabled", level="INFO")
+        exit_code = subprocess.call(['sed', '-i', 's/IPV6=yes/IPV6=no/g',
+                                     '/etc/default/ufw'])
+        if exit_code == 0:
+            hookenv.log('IPv6 support in ufw disabled', level='INFO')
+        else:
+            hookenv.log("Couldn't disable IPv6 support in ufw", level="ERROR")
+            raise Exception("Couldn't disable IPv6 support in ufw")
+
     output = subprocess.check_output(['ufw', 'enable'],
                                      env={'LANG': 'en_US',
                                           'PATH': os.environ['PATH']})

=== modified file 'hooks/charmhelpers/contrib/openstack/context.py'
--- hooks/charmhelpers/contrib/openstack/context.py	2014-12-15 10:28:47 +0000
+++ hooks/charmhelpers/contrib/openstack/context.py	2015-01-12 14:02:09 +0000
@@ -491,6 +491,7 @@
             ctxt['haproxy_client_timeout'] = config('haproxy-client-timeout')
 
         if config('prefer-ipv6'):
+            ctxt['ipv6'] = True
             ctxt['local_host'] = 'ip6-localhost'
             ctxt['haproxy_host'] = '::'
             ctxt['stat_port'] = ':::8888'

=== modified file 'hooks/charmhelpers/contrib/openstack/neutron.py'
--- hooks/charmhelpers/contrib/openstack/neutron.py	2014-12-15 10:28:47 +0000
+++ hooks/charmhelpers/contrib/openstack/neutron.py	2015-01-12 14:02:09 +0000
@@ -152,9 +152,15 @@
                                         database=config('neutron-database'),
                                         relation_prefix='neutron',
                                         ssl_dir=NEUTRON_CONF_DIR)],
-            'services': ['calico-compute', 'bird', 'neutron-dhcp-agent'],
+            'services': ['calico-felix',
+                         'bird',
+                         'neutron-dhcp-agent',
+                         'nova-api-metadata'],
             'packages': [[headers_package()] + determine_dkms_package(),
-                         ['calico-compute', 'bird', 'neutron-dhcp-agent']],
+                         ['calico-compute',
+                          'bird',
+                          'neutron-dhcp-agent',
+                          'nova-api-metadata']],
             'server_packages': ['neutron-server', 'calico-control'],
             'server_services': ['neutron-server']
         }

=== modified file 'hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg'
--- hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg	2014-12-15 09:10:57 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg	2015-01-12 14:02:09 +0000
@@ -38,7 +38,9 @@
 {% for service, ports in service_ports.items() -%}
 frontend tcp-in_{{ service }}
     bind *:{{ ports[0] }}
+    {% if ipv6 -%}
     bind :::{{ ports[0] }}
+    {% endif -%}
     {% for frontend in frontends -%}
     acl net_{{ frontend }} dst {{ frontends[frontend]['network'] }}
     use_backend {{ service }}_{{ frontend }} if net_{{ frontend }}

=== modified file 'hooks/charmhelpers/contrib/openstack/utils.py'
--- hooks/charmhelpers/contrib/openstack/utils.py	2014-12-15 10:28:47 +0000
+++ hooks/charmhelpers/contrib/openstack/utils.py	2015-01-12 14:02:09 +0000
@@ -53,6 +53,7 @@
     ('saucy', 'havana'),
     ('trusty', 'icehouse'),
     ('utopic', 'juno'),
+    ('vivid', 'kilo'),
 ])
 
 
@@ -64,6 +65,7 @@
     ('2013.2', 'havana'),
     ('2014.1', 'icehouse'),
     ('2014.2', 'juno'),
+    ('2015.1', 'kilo'),
 ])
 
 # The ugly duckling
@@ -84,6 +86,7 @@
     ('2.0.0', 'juno'),
     ('2.1.0', 'juno'),
     ('2.2.0', 'juno'),
+    ('2.2.1', 'kilo'),
 ])
 
 DEFAULT_LOOPBACK_SIZE = '5G'
@@ -289,6 +292,9 @@
             'juno': 'trusty-updates/juno',
             'juno/updates': 'trusty-updates/juno',
             'juno/proposed': 'trusty-proposed/juno',
+            'kilo': 'trusty-updates/kilo',
+            'kilo/updates': 'trusty-updates/kilo',
+            'kilo/proposed': 'trusty-proposed/kilo',
         }
 
         try:

=== added file 'hooks/charmhelpers/core/decorators.py'
--- hooks/charmhelpers/core/decorators.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/decorators.py	2015-01-12 14:02:09 +0000
@@ -0,0 +1,41 @@
+#
+# Copyright 2014 Canonical Ltd.
+#
+# Authors:
+#  Edward Hope-Morley <[e-mail]>
+#
+
+import time
+
+from charmhelpers.core.hookenv import (
+    log,
+    INFO,
+)
+
+
+def retry_on_exception(num_retries, base_delay=0, exc_type=Exception):
+    """If the decorated function raises exception exc_type, allow num_retries
+    retry attempts before raise the exception.
+    """
+    def _retry_on_exception_inner_1(f):
+        def _retry_on_exception_inner_2(*args, **kwargs):
+            retries = num_retries
+            multiplier = 1
+            while True:
+                try:
+                    return f(*args, **kwargs)
+                except exc_type:
+                    if not retries:
+                        raise
+
+                delay = base_delay * multiplier
+                multiplier += 1
+                log("Retrying '%s' %d more times (delay=%s)" %
+                    (f.__name__, retries, delay), level=INFO)
+                retries -= 1
+                if delay:
+                    time.sleep(delay)
+
+        return _retry_on_exception_inner_2
+
+    return _retry_on_exception_inner_1

=== modified file 'hooks/charmhelpers/core/host.py'
--- hooks/charmhelpers/core/host.py	2014-12-15 10:28:47 +0000
+++ hooks/charmhelpers/core/host.py	2015-01-12 14:02:09 +0000
@@ -162,13 +162,16 @@
     uid = pwd.getpwnam(owner).pw_uid
     gid = grp.getgrnam(group).gr_gid
     realpath = os.path.abspath(path)
-    if os.path.exists(realpath):
-        if force and not os.path.isdir(realpath):
+    path_exists = os.path.exists(realpath)
+    if path_exists and force:
+        if not os.path.isdir(realpath):
             log("Removing non-directory file {} prior to mkdir()".format(path))
             os.unlink(realpath)
-    else:
+            os.makedirs(realpath, perms)
+        os.chown(realpath, uid, gid)
+    elif not path_exists:
         os.makedirs(realpath, perms)
-    os.chown(realpath, uid, gid)
+        os.chown(realpath, uid, gid)
 
 
 def write_file(path, content, owner='root', group='root', perms=0o444):

=== modified file 'hooks/charmhelpers/fetch/__init__.py'
--- hooks/charmhelpers/fetch/__init__.py	2014-12-15 10:28:47 +0000
+++ hooks/charmhelpers/fetch/__init__.py	2015-01-12 14:02:09 +0000
@@ -64,9 +64,16 @@
     'trusty-juno/updates': 'trusty-updates/juno',
     'trusty-updates/juno': 'trusty-updates/juno',
     'juno/proposed': 'trusty-proposed/juno',
-    'juno/proposed': 'trusty-proposed/juno',
     'trusty-juno/proposed': 'trusty-proposed/juno',
     'trusty-proposed/juno': 'trusty-proposed/juno',
+    # Kilo
+    'kilo': 'trusty-updates/kilo',
+    'trusty-kilo': 'trusty-updates/kilo',
+    'trusty-kilo/updates': 'trusty-updates/kilo',
+    'trusty-updates/kilo': 'trusty-updates/kilo',
+    'kilo/proposed': 'trusty-proposed/kilo',
+    'trusty-kilo/proposed': 'trusty-proposed/kilo',
+    'trusty-proposed/kilo': 'trusty-proposed/kilo',
 }
 
 # The order of this list is very important. Handlers should be listed in from

=== added symlink 'hooks/compute-peer-relation-joined'
=== target is u'nova_compute_hooks.py'
=== added symlink 'hooks/neutron-plugin-relation-changed'
=== target is u'nova_compute_hooks.py'
=== added symlink 'hooks/neutron-plugin-relation-departed'
=== target is u'nova_compute_hooks.py'
=== added symlink 'hooks/neutron-plugin-relation-joined'
=== target is u'nova_compute_hooks.py'
=== modified file 'hooks/nova_compute_hooks.py'
--- hooks/nova_compute_hooks.py	2014-12-15 10:28:47 +0000
+++ hooks/nova_compute_hooks.py	2015-01-12 14:02:09 +0000
@@ -48,6 +48,7 @@
     do_openstack_upgrade,
     public_ssh_key,
     restart_map,
+    services,
     register_configs,
     NOVA_CONF,
     QUANTUM_CONF, NEUTRON_CONF,
@@ -65,6 +66,7 @@
     CEPH_SECRET_UUID,
     assert_libvirt_imagebackend_allowed
 )
+from charmhelpers.contrib.charmsupport import nrpe
 from charmhelpers.core.sysctl import create as create_sysctl
 
 from socket import gethostname
@@ -114,6 +116,8 @@
 
     [compute_joined(rid) for rid in relation_ids('cloud-compute')]
 
+    update_nrpe_config()
+
     CONFIGS.write_all()
 
 
@@ -292,6 +296,7 @@
 def upgrade_charm():
     for r_id in relation_ids('amqp'):
         amqp_joined(relation_id=r_id)
+    update_nrpe_config()
 
 
 @hooks.hook('nova-ceilometer-relation-changed')
@@ -300,6 +305,18 @@
     CONFIGS.write_all()
 
 
[e-mail]('nrpe-external-master-relation-joined',
+            'nrpe-external-master-relation-changed')
+def update_nrpe_config():
+    # python-dbus is used by check_upstart_job
+    apt_install('python-dbus')
+    hostname = nrpe.get_nagios_hostname()
+    current_unit = nrpe.get_nagios_unit_name()
+    nrpe_setup = nrpe.NRPE(hostname=hostname)
+    nrpe.add_init_service_checks(nrpe_setup, services(), current_unit)
+    nrpe_setup.write()
+
+
 def main():
     try:
         hooks.execute(sys.argv)

=== added symlink 'hooks/nrpe-external-master-relation-changed'
=== target is u'nova_compute_hooks.py'
=== added symlink 'hooks/start'
=== target is u'nova_compute_hooks.py'
=== added symlink 'hooks/stop'
=== target is u'nova_compute_hooks.py'
=== modified file 'unit_tests/test_nova_compute_hooks.py'
--- unit_tests/test_nova_compute_hooks.py	2014-12-15 10:28:47 +0000
+++ unit_tests/test_nova_compute_hooks.py	2015-01-12 14:02:09 +0000
@@ -46,6 +46,7 @@
     'register_configs',
     'disable_shell',
     'enable_shell',
+    'update_nrpe_config',
     # misc_utils
     'ensure_ceph_keyring',
     'execd_preinstall',