Merge lp:~gnuoy/charms/trusty/cisco-vpp/dhcp into lp:~openstack-charmers/charms/trusty/cisco-vpp/next

[Liam Young]

This merge proposal adds support for serving dhcp and metadata requests to guests.

[James Page (james-page)]

review: Needs Information

[lp:~gnuoy/charms/trusty/cisco-vpp/dhcp]

121. By Liam Young on 2015-11-27

Fix typo in context that was retuning the wrong ip for keystone

122. By Liam Young on 2015-12-01

ODL initially returns 404s when querying nodes so backoff and retry node query

123. By Liam Young on 2015-12-01

Fix bug causing charm to fail if /etc/neutron does not exists

[James Page (james-page)]

Just a few niggles - but need taking care of.

review: Needs Fixing

[lp:~gnuoy/charms/trusty/cisco-vpp/dhcp]

124. By Liam Young on 2015-12-01

General tidyup/fixes from mp feedback from JamesPage

[James Page (james-page)]

review: Approve

=== modified file 'charm-helpers-hooks.yaml'
--- charm-helpers-hooks.yaml	2015-06-24 09:56:23 +0000
+++ charm-helpers-hooks.yaml	2015-12-01 15:05:49 +0000
@@ -1,4 +1,4 @@
-branch: lp:~gnuoy/charm-helpers/cisco-vpp/
+branch: lp:charm-helpers
 destination: hooks/charmhelpers
 include:
     - core
 
=== modified file 'config.yaml'
--- config.yaml	2015-08-14 07:27:33 +0000
+++ config.yaml	2015-12-01 15:05:49 +0000
@@ -41,3 +41,11 @@
   mac-network-map:
     default: ''
     type: string
+  rabbit-user:
+    default: neutron
+    type: string
+    description: Username used to access RabbitMQ queue
+  rabbit-vhost:
+    default: openstack
+    type: string
+    description: RabbitMQ vhost
 
=== modified file 'hooks/ODL.py'
--- hooks/ODL.py	2015-09-14 16:44:47 +0000
+++ hooks/ODL.py	2015-12-01 15:05:49 +0000
@@ -73,7 +73,8 @@
 
     def get_odl_registered_nodes(self):
         log('Querying nodes registered with odl')
-        odl_req = self.contact_odl('GET', self.node_query_url)
+        odl_req = self.contact_odl('GET', self.node_query_url,
+                                   retry_rcs=[requests.codes.not_found])
         odl_json = odl_req.json()
         odl_node_ids = []
         if odl_json.get('nodes'):
 
=== modified file 'hooks/charmhelpers/contrib/network/ip.py'
--- hooks/charmhelpers/contrib/network/ip.py	2015-06-10 15:45:48 +0000
+++ hooks/charmhelpers/contrib/network/ip.py	2015-12-01 15:05:49 +0000
@@ -23,7 +23,7 @@
 from functools import partial
 
 from charmhelpers.core.hookenv import unit_get
-from charmhelpers.fetch import apt_install
+from charmhelpers.fetch import apt_install, apt_update
 from charmhelpers.core.hookenv import (
     log,
     WARNING,
@@ -32,13 +32,15 @@
 try:
     import netifaces
 except ImportError:
-    apt_install('python-netifaces')
+    apt_update(fatal=True)
+    apt_install('python-netifaces', fatal=True)
     import netifaces
 
 try:
     import netaddr
 except ImportError:
-    apt_install('python-netaddr')
+    apt_update(fatal=True)
+    apt_install('python-netaddr', fatal=True)
     import netaddr
 
 
@@ -435,8 +437,12 @@
 
         rev = dns.reversename.from_address(address)
         result = ns_query(rev)
+
         if not result:
-            return None
+            try:
+                result = socket.gethostbyaddr(address)[0]
+            except:
+                return None
     else:
         result = address
 
 
=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/deployment.py'
--- hooks/charmhelpers/contrib/openstack/amulet/deployment.py	2015-06-16 07:53:15 +0000
+++ hooks/charmhelpers/contrib/openstack/amulet/deployment.py	2015-12-01 15:05:49 +0000
@@ -14,12 +14,18 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
+import re
+import sys
 import six
 from collections import OrderedDict
 from charmhelpers.contrib.amulet.deployment import (
     AmuletDeployment
 )
 
+DEBUG = logging.DEBUG
+ERROR = logging.ERROR
+
 
 class OpenStackAmuletDeployment(AmuletDeployment):
     """OpenStack amulet deployment.
@@ -28,9 +34,12 @@
        that is specifically for use by OpenStack charms.
        """
 
-    def __init__(self, series=None, openstack=None, source=None, stable=True):
+    def __init__(self, series=None, openstack=None, source=None,
+                 stable=True, log_level=DEBUG):
         """Initialize the deployment environment."""
         super(OpenStackAmuletDeployment, self).__init__(series)
+        self.log = self.get_logger(level=log_level)
+        self.log.info('OpenStackAmuletDeployment:  init')
         self.openstack = openstack
         self.source = source
         self.stable = stable
@@ -38,30 +47,55 @@
         # out.
         self.current_next = "trusty"
 
+    def get_logger(self, name="deployment-logger", level=logging.DEBUG):
+        """Get a logger object that will log to stdout."""
+        log = logging
+        logger = log.getLogger(name)
+        fmt = log.Formatter("%(asctime)s %(funcName)s "
+                            "%(levelname)s: %(message)s")
+
+        handler = log.StreamHandler(stream=sys.stdout)
+        handler.setLevel(level)
+        handler.setFormatter(fmt)
+
+        logger.addHandler(handler)
+        logger.setLevel(level)
+
+        return logger
+
     def _determine_branch_locations(self, other_services):
         """Determine the branch locations for the other services.
 
            Determine if the local branch being tested is derived from its
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
-        base_charms = ['mysql', 'mongodb']
+
+        self.log.info('OpenStackAmuletDeployment:  determine branch locations')
+
+        # Charms outside the lp:~openstack-charmers namespace
+        base_charms = ['mysql', 'mongodb', 'nrpe']
+
+        # Force these charms to current series even when using an older series.
+        # ie. Use trusty/nrpe even when series is precise, as the P charm
+        # does not possess the necessary external master config and hooks.
+        force_series_current = ['nrpe']
 
         if self.series in ['precise', 'trusty']:
             base_series = self.series
         else:
             base_series = self.current_next
 
-        if self.stable:
-            for svc in other_services:
-                if svc.get('location'):
-                    continue
+        for svc in other_services:
+            if svc['name'] in force_series_current:
+                base_series = self.current_next
+            # If a location has been explicitly set, use it
+            if svc.get('location'):
+                continue
+            if self.stable:
                 temp = 'lp:charms/{}/{}'
                 svc['location'] = temp.format(base_series,
                                               svc['name'])
-        else:
-            for svc in other_services:
-                if svc.get('location'):
-                    continue
+            else:
                 if svc['name'] in base_charms:
                     temp = 'lp:charms/{}/{}'
                     svc['location'] = temp.format(base_series,
@@ -70,10 +104,13 @@
                     temp = 'lp:~openstack-charmers/charms/{}/{}/next'
                     svc['location'] = temp.format(self.current_next,
                                                   svc['name'])
+
         return other_services
 
     def _add_services(self, this_service, other_services):
         """Add services to the deployment and set openstack-origin/source."""
+        self.log.info('OpenStackAmuletDeployment:  adding services')
+
         other_services = self._determine_branch_locations(other_services)
 
         super(OpenStackAmuletDeployment, self)._add_services(this_service,
@@ -81,29 +118,102 @@
 
         services = other_services
         services.append(this_service)
+
+        # Charms which should use the source config option
         use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
                       'ceph-osd', 'ceph-radosgw']
-        # Openstack subordinate charms do not expose an origin option as that
-        # is controlled by the principle
-        ignore = ['neutron-openvswitch', 'cisco-vpp']
+
+        # Charms which can not use openstack-origin, ie. many subordinates
+        no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe',
+                     'openvswitch-odl', 'neutron-api-odl', 'odl-controller']
 
         if self.openstack:
             for svc in services:
-                if svc['name'] not in use_source + ignore:
+                if svc['name'] not in use_source + no_origin:
                     config = {'openstack-origin': self.openstack}
                     self.d.configure(svc['name'], config)
 
         if self.source:
             for svc in services:
-                if svc['name'] in use_source and svc['name'] not in ignore:
+                if svc['name'] in use_source and svc['name'] not in no_origin:
                     config = {'source': self.source}
                     self.d.configure(svc['name'], config)
 
     def _configure_services(self, configs):
         """Configure all of the services."""
+        self.log.info('OpenStackAmuletDeployment:  configure services')
         for service, config in six.iteritems(configs):
             self.d.configure(service, config)
 
+    def _auto_wait_for_status(self, message=None, exclude_services=None,
+                              include_only=None, timeout=1800):
+        """Wait for all units to have a specific extended status, except
+        for any defined as excluded.  Unless specified via message, any
+        status containing any case of 'ready' will be considered a match.
+
+        Examples of message usage:
+
+          Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
+              message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
+
+          Wait for all units to reach this status (exact match):
+              message = re.compile('^Unit is ready and clustered$')
+
+          Wait for all units to reach any one of these (exact match):
+              message = re.compile('Unit is ready|OK|Ready')
+
+          Wait for at least one unit to reach this status (exact match):
+              message = {'ready'}
+
+        See Amulet's sentry.wait_for_messages() for message usage detail.
+        https://github.com/juju/amulet/blob/master/amulet/sentry.py
+
+        :param message: Expected status match
+        :param exclude_services: List of juju service names to ignore,
+            not to be used in conjuction with include_only.
+        :param include_only: List of juju service names to exclusively check,
+            not to be used in conjuction with exclude_services.
+        :param timeout: Maximum time in seconds to wait for status match
+        :returns: None.  Raises if timeout is hit.
+        """
+        self.log.info('Waiting for extended status on units...')
+
+        all_services = self.d.services.keys()
+
+        if exclude_services and include_only:
+            raise ValueError('exclude_services can not be used '
+                             'with include_only')
+
+        if message:
+            if isinstance(message, re._pattern_type):
+                match = message.pattern
+            else:
+                match = message
+
+            self.log.debug('Custom extended status wait match: '
+                           '{}'.format(match))
+        else:
+            self.log.debug('Default extended status wait match:  contains '
+                           'READY (case-insensitive)')
+            message = re.compile('.*ready.*', re.IGNORECASE)
+
+        if exclude_services:
+            self.log.debug('Excluding services from extended status match: '
+                           '{}'.format(exclude_services))
+        else:
+            exclude_services = []
+
+        if include_only:
+            services = include_only
+        else:
+            services = list(set(all_services) - set(exclude_services))
+
+        self.log.debug('Waiting up to {}s for extended status on services: '
+                       '{}'.format(timeout, services))
+        service_messages = {service: message for service in services}
+        self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
+        self.log.info('OK')
+
     def _get_openstack_release(self):
         """Get openstack release.
 
@@ -152,3 +262,36 @@
             return os_origin.split('%s-' % self.series)[1].split('/')[0]
         else:
             return releases[self.series]
+
+    def get_ceph_expected_pools(self, radosgw=False):
+        """Return a list of expected ceph pools in a ceph + cinder + glance
+        test scenario, based on OpenStack release and whether ceph radosgw
+        is flagged as present or not."""
+
+        if self._get_openstack_release() >= self.trusty_kilo:
+            # Kilo or later
+            pools = [
+                'rbd',
+                'cinder',
+                'glance'
+            ]
+        else:
+            # Juno or earlier
+            pools = [
+                'data',
+                'metadata',
+                'rbd',
+                'cinder',
+                'glance'
+            ]
+
+        if radosgw:
+            pools.extend([
+                '.rgw.root',
+                '.rgw.control',
+                '.rgw',
+                '.rgw.gc',
+                '.users.uid'
+            ])
+
+        return pools
 
=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/utils.py'
--- hooks/charmhelpers/contrib/openstack/amulet/utils.py	2015-06-10 07:35:12 +0000
+++ hooks/charmhelpers/contrib/openstack/amulet/utils.py	2015-12-01 15:05:49 +0000
@@ -14,16 +14,22 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import amulet
+import json
 import logging
 import os
+import re
+import six
 import time
 import urllib
 
+import cinderclient.v1.client as cinder_client
 import glanceclient.v1.client as glance_client
+import heatclient.v1.client as heat_client
 import keystoneclient.v2_0 as keystone_client
 import novaclient.v1_1.client as nova_client
-
-import six
+import pika
+import swiftclient
 
 from charmhelpers.contrib.amulet.utils import (
     AmuletUtils
@@ -37,7 +43,7 @@
     """OpenStack amulet utilities.
 
        This class inherits from AmuletUtils and has additional support
-       that is specifically for use by OpenStack charms.
+       that is specifically for use by OpenStack charm tests.
        """
 
     def __init__(self, log_level=ERROR):
@@ -51,6 +57,8 @@
            Validate actual endpoint data vs expected endpoint data. The ports
            are used to find the matching endpoint.
            """
+        self.log.debug('Validating endpoint data...')
+        self.log.debug('actual: {}'.format(repr(endpoints)))
         found = False
         for ep in endpoints:
             self.log.debug('endpoint: {}'.format(repr(ep)))
@@ -77,6 +85,7 @@
            Validate a list of actual service catalog endpoints vs a list of
            expected service catalog endpoints.
            """
+        self.log.debug('Validating service catalog endpoint data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for k, v in six.iteritems(expected):
             if k in actual:
@@ -93,6 +102,7 @@
            Validate a list of actual tenant data vs list of expected tenant
            data.
            """
+        self.log.debug('Validating tenant data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -114,6 +124,7 @@
            Validate a list of actual role data vs a list of expected role
            data.
            """
+        self.log.debug('Validating role data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -134,6 +145,7 @@
            Validate a list of actual user data vs a list of expected user
            data.
            """
+        self.log.debug('Validating user data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -155,17 +167,30 @@
 
            Validate a list of actual flavors vs a list of expected flavors.
            """
+        self.log.debug('Validating flavor data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         act = [a.name for a in actual]
         return self._validate_list_data(expected, act)
 
     def tenant_exists(self, keystone, tenant):
         """Return True if tenant exists."""
+        self.log.debug('Checking if tenant exists ({})...'.format(tenant))
         return tenant in [t.name for t in keystone.tenants.list()]
 
+    def authenticate_cinder_admin(self, keystone_sentry, username,
+                                  password, tenant):
+        """Authenticates admin user with cinder."""
+        # NOTE(beisner): cinder python client doesn't accept tokens.
+        service_ip = \
+            keystone_sentry.relation('shared-db',
+                                     'mysql:shared-db')['private-address']
+        ept = "http://{}:5000/v2.0".format(service_ip.strip().decode('utf-8'))
+        return cinder_client.Client(username, password, tenant, ept)
+
     def authenticate_keystone_admin(self, keystone_sentry, user, password,
                                     tenant):
         """Authenticates admin user with the keystone admin endpoint."""
+        self.log.debug('Authenticating keystone admin...')
         unit = keystone_sentry
         service_ip = unit.relation('shared-db',
                                    'mysql:shared-db')['private-address']
@@ -175,6 +200,7 @@
 
     def authenticate_keystone_user(self, keystone, user, password, tenant):
         """Authenticates a regular user with the keystone public endpoint."""
+        self.log.debug('Authenticating keystone user ({})...'.format(user))
         ep = keystone.service_catalog.url_for(service_type='identity',
                                               endpoint_type='publicURL')
         return keystone_client.Client(username=user, password=password,
@@ -182,19 +208,49 @@
 
     def authenticate_glance_admin(self, keystone):
         """Authenticates admin user with glance."""
+        self.log.debug('Authenticating glance admin...')
         ep = keystone.service_catalog.url_for(service_type='image',
                                               endpoint_type='adminURL')
         return glance_client.Client(ep, token=keystone.auth_token)
 
+    def authenticate_heat_admin(self, keystone):
+        """Authenticates the admin user with heat."""
+        self.log.debug('Authenticating heat admin...')
+        ep = keystone.service_catalog.url_for(service_type='orchestration',
+                                              endpoint_type='publicURL')
+        return heat_client.Client(endpoint=ep, token=keystone.auth_token)
+
     def authenticate_nova_user(self, keystone, user, password, tenant):
         """Authenticates a regular user with nova-api."""
+        self.log.debug('Authenticating nova user ({})...'.format(user))
         ep = keystone.service_catalog.url_for(service_type='identity',
                                               endpoint_type='publicURL')
         return nova_client.Client(username=user, api_key=password,
                                   project_id=tenant, auth_url=ep)
 
+    def authenticate_swift_user(self, keystone, user, password, tenant):
+        """Authenticates a regular user with swift api."""
+        self.log.debug('Authenticating swift user ({})...'.format(user))
+        ep = keystone.service_catalog.url_for(service_type='identity',
+                                              endpoint_type='publicURL')
+        return swiftclient.Connection(authurl=ep,
+                                      user=user,
+                                      key=password,
+                                      tenant_name=tenant,
+                                      auth_version='2.0')
+
     def create_cirros_image(self, glance, image_name):
-        """Download the latest cirros image and upload it to glance."""
+        """Download the latest cirros image and upload it to glance,
+        validate and return a resource pointer.
+
+        :param glance: pointer to authenticated glance connection
+        :param image_name: display name for new image
+        :returns: glance image pointer
+        """
+        self.log.debug('Creating glance cirros image '
+                       '({})...'.format(image_name))
+
+        # Download cirros image
         http_proxy = os.getenv('AMULET_HTTP_PROXY')
         self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
         if http_proxy:
@@ -203,57 +259,67 @@
         else:
             opener = urllib.FancyURLopener()
 
-        f = opener.open("http://download.cirros-cloud.net/version/released")
+        f = opener.open('http://download.cirros-cloud.net/version/released')
         version = f.read().strip()
-        cirros_img = "cirros-{}-x86_64-disk.img".format(version)
+        cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
         local_path = os.path.join('tests', cirros_img)
 
         if not os.path.exists(local_path):
-            cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net",
+            cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
                                                   version, cirros_img)
             opener.retrieve(cirros_url, local_path)
         f.close()
 
+        # Create glance image
         with open(local_path) as f:
             image = glance.images.create(name=image_name, is_public=True,
                                          disk_format='qcow2',
                                          container_format='bare', data=f)
-        count = 1
-        status = image.status
-        while status != 'active' and count < 10:
-            time.sleep(3)
-            image = glance.images.get(image.id)
-            status = image.status
-            self.log.debug('image status: {}'.format(status))
-            count += 1
-
-        if status != 'active':
-            self.log.error('image creation timed out')
-            return None
+
+        # Wait for image to reach active status
+        img_id = image.id
+        ret = self.resource_reaches_status(glance.images, img_id,
+                                           expected_stat='active',
+                                           msg='Image status wait')
+        if not ret:
+            msg = 'Glance image failed to reach expected state.'
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Re-validate new image
+        self.log.debug('Validating image attributes...')
+        val_img_name = glance.images.get(img_id).name
+        val_img_stat = glance.images.get(img_id).status
+        val_img_pub = glance.images.get(img_id).is_public
+        val_img_cfmt = glance.images.get(img_id).container_format
+        val_img_dfmt = glance.images.get(img_id).disk_format
+        msg_attr = ('Image attributes - name:{} public:{} id:{} stat:{} '
+                    'container fmt:{} disk fmt:{}'.format(
+                        val_img_name, val_img_pub, img_id,
+                        val_img_stat, val_img_cfmt, val_img_dfmt))
+
+        if val_img_name == image_name and val_img_stat == 'active' \
+                and val_img_pub is True and val_img_cfmt == 'bare' \
+                and val_img_dfmt == 'qcow2':
+            self.log.debug(msg_attr)
+        else:
+            msg = ('Volume validation failed, {}'.format(msg_attr))
+            amulet.raise_status(amulet.FAIL, msg=msg)
 
         return image
 
     def delete_image(self, glance, image):
         """Delete the specified image."""
-        num_before = len(list(glance.images.list()))
-        glance.images.delete(image)
-
-        count = 1
-        num_after = len(list(glance.images.list()))
-        while num_after != (num_before - 1) and count < 10:
-            time.sleep(3)
-            num_after = len(list(glance.images.list()))
-            self.log.debug('number of images: {}'.format(num_after))
-            count += 1
-
-        if num_after != (num_before - 1):
-            self.log.error('image deletion timed out')
-            return False
-
-        return True
+
+        # /!\ DEPRECATION WARNING
+        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+                      'delete_resource instead of delete_image.')
+        self.log.debug('Deleting glance image ({})...'.format(image))
+        return self.delete_resource(glance.images, image, msg='glance image')
 
     def create_instance(self, nova, image_name, instance_name, flavor):
         """Create the specified instance."""
+        self.log.debug('Creating instance '
+                       '({}|{}|{})'.format(instance_name, image_name, flavor))
         image = nova.images.find(name=image_name)
         flavor = nova.flavors.find(name=flavor)
         instance = nova.servers.create(name=instance_name, image=image,
@@ -276,19 +342,644 @@
 
     def delete_instance(self, nova, instance):
         """Delete the specified instance."""
-        num_before = len(list(nova.servers.list()))
-        nova.servers.delete(instance)
-
-        count = 1
-        num_after = len(list(nova.servers.list()))
-        while num_after != (num_before - 1) and count < 10:
-            time.sleep(3)
-            num_after = len(list(nova.servers.list()))
-            self.log.debug('number of instances: {}'.format(num_after))
-            count += 1
-
-        if num_after != (num_before - 1):
-            self.log.error('instance deletion timed out')
-            return False
-
-        return True
+
+        # /!\ DEPRECATION WARNING
+        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+                      'delete_resource instead of delete_instance.')
+        self.log.debug('Deleting instance ({})...'.format(instance))
+        return self.delete_resource(nova.servers, instance,
+                                    msg='nova instance')
+
+    def create_or_get_keypair(self, nova, keypair_name="testkey"):
+        """Create a new keypair, or return pointer if it already exists."""
+        try:
+            _keypair = nova.keypairs.get(keypair_name)
+            self.log.debug('Keypair ({}) already exists, '
+                           'using it.'.format(keypair_name))
+            return _keypair
+        except:
+            self.log.debug('Keypair ({}) does not exist, '
+                           'creating it.'.format(keypair_name))
+
+        _keypair = nova.keypairs.create(name=keypair_name)
+        return _keypair
+
+    def create_cinder_volume(self, cinder, vol_name="demo-vol", vol_size=1,
+                             img_id=None, src_vol_id=None, snap_id=None):
+        """Create cinder volume, optionally from a glance image, OR
+        optionally as a clone of an existing volume, OR optionally
+        from a snapshot.  Wait for the new volume status to reach
+        the expected status, validate and return a resource pointer.
+
+        :param vol_name: cinder volume display name
+        :param vol_size: size in gigabytes
+        :param img_id: optional glance image id
+        :param src_vol_id: optional source volume id to clone
+        :param snap_id: optional snapshot id to use
+        :returns: cinder volume pointer
+        """
+        # Handle parameter input and avoid impossible combinations
+        if img_id and not src_vol_id and not snap_id:
+            # Create volume from image
+            self.log.debug('Creating cinder volume from glance image...')
+            bootable = 'true'
+        elif src_vol_id and not img_id and not snap_id:
+            # Clone an existing volume
+            self.log.debug('Cloning cinder volume...')
+            bootable = cinder.volumes.get(src_vol_id).bootable
+        elif snap_id and not src_vol_id and not img_id:
+            # Create volume from snapshot
+            self.log.debug('Creating cinder volume from snapshot...')
+            snap = cinder.volume_snapshots.find(id=snap_id)
+            vol_size = snap.size
+            snap_vol_id = cinder.volume_snapshots.get(snap_id).volume_id
+            bootable = cinder.volumes.get(snap_vol_id).bootable
+        elif not img_id and not src_vol_id and not snap_id:
+            # Create volume
+            self.log.debug('Creating cinder volume...')
+            bootable = 'false'
+        else:
+            # Impossible combination of parameters
+            msg = ('Invalid method use - name:{} size:{} img_id:{} '
+                   'src_vol_id:{} snap_id:{}'.format(vol_name, vol_size,
+                                                     img_id, src_vol_id,
+                                                     snap_id))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Create new volume
+        try:
+            vol_new = cinder.volumes.create(display_name=vol_name,
+                                            imageRef=img_id,
+                                            size=vol_size,
+                                            source_volid=src_vol_id,
+                                            snapshot_id=snap_id)
+            vol_id = vol_new.id
+        except Exception as e:
+            msg = 'Failed to create volume: {}'.format(e)
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Wait for volume to reach available status
+        ret = self.resource_reaches_status(cinder.volumes, vol_id,
+                                           expected_stat="available",
+                                           msg="Volume status wait")
+        if not ret:
+            msg = 'Cinder volume failed to reach expected state.'
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Re-validate new volume
+        self.log.debug('Validating volume attributes...')
+        val_vol_name = cinder.volumes.get(vol_id).display_name
+        val_vol_boot = cinder.volumes.get(vol_id).bootable
+        val_vol_stat = cinder.volumes.get(vol_id).status
+        val_vol_size = cinder.volumes.get(vol_id).size
+        msg_attr = ('Volume attributes - name:{} id:{} stat:{} boot:'
+                    '{} size:{}'.format(val_vol_name, vol_id,
+                                        val_vol_stat, val_vol_boot,
+                                        val_vol_size))
+
+        if val_vol_boot == bootable and val_vol_stat == 'available' \
+                and val_vol_name == vol_name and val_vol_size == vol_size:
+            self.log.debug(msg_attr)
+        else:
+            msg = ('Volume validation failed, {}'.format(msg_attr))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        return vol_new
+
+    def delete_resource(self, resource, resource_id,
+                        msg="resource", max_wait=120):
+        """Delete one openstack resource, such as one instance, keypair,
+        image, volume, stack, etc., and confirm deletion within max wait time.
+
+        :param resource: pointer to os resource type, ex:glance_client.images
+        :param resource_id: unique name or id for the openstack resource
+        :param msg: text to identify purpose in logging
+        :param max_wait: maximum wait time in seconds
+        :returns: True if successful, otherwise False
+        """
+        self.log.debug('Deleting OpenStack resource '
+                       '{} ({})'.format(resource_id, msg))
+        num_before = len(list(resource.list()))
+        resource.delete(resource_id)
+
+        tries = 0
+        num_after = len(list(resource.list()))
+        while num_after != (num_before - 1) and tries < (max_wait / 4):
+            self.log.debug('{} delete check: '
+                           '{} [{}:{}] {}'.format(msg, tries,
+                                                  num_before,
+                                                  num_after,
+                                                  resource_id))
+            time.sleep(4)
+            num_after = len(list(resource.list()))
+            tries += 1
+
+        self.log.debug('{}:  expected, actual count = {}, '
+                       '{}'.format(msg, num_before - 1, num_after))
+
+        if num_after == (num_before - 1):
+            return True
+        else:
+            self.log.error('{} delete timed out'.format(msg))
+            return False
+
+    def resource_reaches_status(self, resource, resource_id,
+                                expected_stat='available',
+                                msg='resource', max_wait=120):
+        """Wait for an openstack resources status to reach an
+           expected status within a specified time.  Useful to confirm that
+           nova instances, cinder vols, snapshots, glance images, heat stacks
+           and other resources eventually reach the expected status.
+
+        :param resource: pointer to os resource type, ex: heat_client.stacks
+        :param resource_id: unique id for the openstack resource
+        :param expected_stat: status to expect resource to reach
+        :param msg: text to identify purpose in logging
+        :param max_wait: maximum wait time in seconds
+        :returns: True if successful, False if status is not reached
+        """
+
+        tries = 0
+        resource_stat = resource.get(resource_id).status
+        while resource_stat != expected_stat and tries < (max_wait / 4):
+            self.log.debug('{} status check: '
+                           '{} [{}:{}] {}'.format(msg, tries,
+                                                  resource_stat,
+                                                  expected_stat,
+                                                  resource_id))
+            time.sleep(4)
+            resource_stat = resource.get(resource_id).status
+            tries += 1
+
+        self.log.debug('{}:  expected, actual status = {}, '
+                       '{}'.format(msg, resource_stat, expected_stat))
+
+        if resource_stat == expected_stat:
+            return True
+        else:
+            self.log.debug('{} never reached expected status: '
+                           '{}'.format(resource_id, expected_stat))
+            return False
+
+    def get_ceph_osd_id_cmd(self, index):
+        """Produce a shell command that will return a ceph-osd id."""
+        return ("`initctl list | grep 'ceph-osd ' | "
+                "awk 'NR=={} {{ print $2 }}' | "
+                "grep -o '[0-9]*'`".format(index + 1))
+
+    def get_ceph_pools(self, sentry_unit):
+        """Return a dict of ceph pools from a single ceph unit, with
+        pool name as keys, pool id as vals."""
+        pools = {}
+        cmd = 'sudo ceph osd lspools'
+        output, code = sentry_unit.run(cmd)
+        if code != 0:
+            msg = ('{} `{}` returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, code, output))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance,
+        for pool in str(output).split(','):
+            pool_id_name = pool.split(' ')
+            if len(pool_id_name) == 2:
+                pool_id = pool_id_name[0]
+                pool_name = pool_id_name[1]
+                pools[pool_name] = int(pool_id)
+
+        self.log.debug('Pools on {}: {}'.format(sentry_unit.info['unit_name'],
+                                                pools))
+        return pools
+
+    def get_ceph_df(self, sentry_unit):
+        """Return dict of ceph df json output, including ceph pool state.
+
+        :param sentry_unit: Pointer to amulet sentry instance (juju unit)
+        :returns: Dict of ceph df output
+        """
+        cmd = 'sudo ceph df --format=json'
+        output, code = sentry_unit.run(cmd)
+        if code != 0:
+            msg = ('{} `{}` returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, code, output))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+        return json.loads(output)
+
+    def get_ceph_pool_sample(self, sentry_unit, pool_id=0):
+        """Take a sample of attributes of a ceph pool, returning ceph
+        pool name, object count and disk space used for the specified
+        pool ID number.
+
+        :param sentry_unit: Pointer to amulet sentry instance (juju unit)
+        :param pool_id: Ceph pool ID
+        :returns: List of pool name, object count, kb disk space used
+        """
+        df = self.get_ceph_df(sentry_unit)
+        pool_name = df['pools'][pool_id]['name']
+        obj_count = df['pools'][pool_id]['stats']['objects']
+        kb_used = df['pools'][pool_id]['stats']['kb_used']
+        self.log.debug('Ceph {} pool (ID {}): {} objects, '
+                       '{} kb used'.format(pool_name, pool_id,
+                                           obj_count, kb_used))
+        return pool_name, obj_count, kb_used
+
+    def validate_ceph_pool_samples(self, samples, sample_type="resource pool"):
+        """Validate ceph pool samples taken over time, such as pool
+        object counts or pool kb used, before adding, after adding, and
+        after deleting items which affect those pool attributes.  The
+        2nd element is expected to be greater than the 1st; 3rd is expected
+        to be less than the 2nd.
+
+        :param samples: List containing 3 data samples
+        :param sample_type: String for logging and usage context
+        :returns: None if successful, Failure message otherwise
+        """
+        original, created, deleted = range(3)
+        if samples[created] <= samples[original] or \
+                samples[deleted] >= samples[created]:
+            return ('Ceph {} samples ({}) '
+                    'unexpected.'.format(sample_type, samples))
+        else:
+            self.log.debug('Ceph {} samples (OK): '
+                           '{}'.format(sample_type, samples))
+            return None
+
+    # rabbitmq/amqp specific helpers:
+
+    def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
+        """Wait for rmq units extended status to show cluster readiness,
+        after an optional initial sleep period.  Initial sleep is likely
+        necessary to be effective following a config change, as status
+        message may not instantly update to non-ready."""
+
+        if init_sleep:
+            time.sleep(init_sleep)
+
+        message = re.compile('^Unit is ready and clustered$')
+        deployment._auto_wait_for_status(message=message,
+                                         timeout=timeout,
+                                         include_only=['rabbitmq-server'])
+
+    def add_rmq_test_user(self, sentry_units,
+                          username="testuser1", password="changeme"):
+        """Add a test user via the first rmq juju unit, check connection as
+        the new user against all sentry units.
+
+        :param sentry_units: list of sentry unit pointers
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :returns: None if successful.  Raise on error.
+        """
+        self.log.debug('Adding rmq user ({})...'.format(username))
+
+        # Check that user does not already exist
+        cmd_user_list = 'rabbitmqctl list_users'
+        output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list)
+        if username in output:
+            self.log.warning('User ({}) already exists, returning '
+                             'gracefully.'.format(username))
+            return
+
+        perms = '".*" ".*" ".*"'
+        cmds = ['rabbitmqctl add_user {} {}'.format(username, password),
+                'rabbitmqctl set_permissions {} {}'.format(username, perms)]
+
+        # Add user via first unit
+        for cmd in cmds:
+            output, _ = self.run_cmd_unit(sentry_units[0], cmd)
+
+        # Check connection against the other sentry_units
+        self.log.debug('Checking user connect against units...')
+        for sentry_unit in sentry_units:
+            connection = self.connect_amqp_by_unit(sentry_unit, ssl=False,
+                                                   username=username,
+                                                   password=password)
+            connection.close()
+
+    def delete_rmq_test_user(self, sentry_units, username="testuser1"):
+        """Delete a rabbitmq user via the first rmq juju unit.
+
+        :param sentry_units: list of sentry unit pointers
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :returns: None if successful or no such user.
+        """
+        self.log.debug('Deleting rmq user ({})...'.format(username))
+
+        # Check that the user exists
+        cmd_user_list = 'rabbitmqctl list_users'
+        output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list)
+
+        if username not in output:
+            self.log.warning('User ({}) does not exist, returning '
+                             'gracefully.'.format(username))
+            return
+
+        # Delete the user
+        cmd_user_del = 'rabbitmqctl delete_user {}'.format(username)
+        output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_del)
+
+    def get_rmq_cluster_status(self, sentry_unit):
+        """Execute rabbitmq cluster status command on a unit and return
+        the full output.
+
+        :param unit: sentry unit
+        :returns: String containing console output of cluster status command
+        """
+        cmd = 'rabbitmqctl cluster_status'
+        output, _ = self.run_cmd_unit(sentry_unit, cmd)
+        self.log.debug('{} cluster_status:\n{}'.format(
+            sentry_unit.info['unit_name'], output))
+        return str(output)
+
+    def get_rmq_cluster_running_nodes(self, sentry_unit):
+        """Parse rabbitmqctl cluster_status output string, return list of
+        running rabbitmq cluster nodes.
+
+        :param unit: sentry unit
+        :returns: List containing node names of running nodes
+        """
+        # NOTE(beisner): rabbitmqctl cluster_status output is not
+        # json-parsable, do string chop foo, then json.loads that.
+        str_stat = self.get_rmq_cluster_status(sentry_unit)
+        if 'running_nodes' in str_stat:
+            pos_start = str_stat.find("{running_nodes,") + 15
+            pos_end = str_stat.find("]},", pos_start) + 1
+            str_run_nodes = str_stat[pos_start:pos_end].replace("'", '"')
+            run_nodes = json.loads(str_run_nodes)
+            return run_nodes
+        else:
+            return []
+
+    def validate_rmq_cluster_running_nodes(self, sentry_units):
+        """Check that all rmq unit hostnames are represented in the
+        cluster_status output of all units.
+
+        :param host_names: dict of juju unit names to host names
+        :param units: list of sentry unit pointers (all rmq units)
+        :returns: None if successful, otherwise return error message
+        """
+        host_names = self.get_unit_hostnames(sentry_units)
+        errors = []
+
+        # Query every unit for cluster_status running nodes
+        for query_unit in sentry_units:
+            query_unit_name = query_unit.info['unit_name']
+            running_nodes = self.get_rmq_cluster_running_nodes(query_unit)
+
+            # Confirm that every unit is represented in the queried unit's
+            # cluster_status running nodes output.
+            for validate_unit in sentry_units:
+                val_host_name = host_names[validate_unit.info['unit_name']]
+                val_node_name = 'rabbit@{}'.format(val_host_name)
+
+                if val_node_name not in running_nodes:
+                    errors.append('Cluster member check failed on {}: {} not '
+                                  'in {}\n'.format(query_unit_name,
+                                                   val_node_name,
+                                                   running_nodes))
+        if errors:
+            return ''.join(errors)
+
+    def rmq_ssl_is_enabled_on_unit(self, sentry_unit, port=None):
+        """Check a single juju rmq unit for ssl and port in the config file."""
+        host = sentry_unit.info['public-address']
+        unit_name = sentry_unit.info['unit_name']
+
+        conf_file = '/etc/rabbitmq/rabbitmq.config'
+        conf_contents = str(self.file_contents_safe(sentry_unit,
+                                                    conf_file, max_wait=16))
+        # Checks
+        conf_ssl = 'ssl' in conf_contents
+        conf_port = str(port) in conf_contents
+
+        # Port explicitly checked in config
+        if port and conf_port and conf_ssl:
+            self.log.debug('SSL is enabled  @{}:{} '
+                           '({})'.format(host, port, unit_name))
+            return True
+        elif port and not conf_port and conf_ssl:
+            self.log.debug('SSL is enabled @{} but not on port {} '
+                           '({})'.format(host, port, unit_name))
+            return False
+        # Port not checked (useful when checking that ssl is disabled)
+        elif not port and conf_ssl:
+            self.log.debug('SSL is enabled  @{}:{} '
+                           '({})'.format(host, port, unit_name))
+            return True
+        elif not conf_ssl:
+            self.log.debug('SSL not enabled @{}:{} '
+                           '({})'.format(host, port, unit_name))
+            return False
+        else:
+            msg = ('Unknown condition when checking SSL status @{}:{} '
+                   '({})'.format(host, port, unit_name))
+            amulet.raise_status(amulet.FAIL, msg)
+
+    def validate_rmq_ssl_enabled_units(self, sentry_units, port=None):
+        """Check that ssl is enabled on rmq juju sentry units.
+
+        :param sentry_units: list of all rmq sentry units
+        :param port: optional ssl port override to validate
+        :returns: None if successful, otherwise return error message
+        """
+        for sentry_unit in sentry_units:
+            if not self.rmq_ssl_is_enabled_on_unit(sentry_unit, port=port):
+                return ('Unexpected condition:  ssl is disabled on unit '
+                        '({})'.format(sentry_unit.info['unit_name']))
+        return None
+
+    def validate_rmq_ssl_disabled_units(self, sentry_units):
+        """Check that ssl is enabled on listed rmq juju sentry units.
+
+        :param sentry_units: list of all rmq sentry units
+        :returns: True if successful.  Raise on error.
+        """
+        for sentry_unit in sentry_units:
+            if self.rmq_ssl_is_enabled_on_unit(sentry_unit):
+                return ('Unexpected condition:  ssl is enabled on unit '
+                        '({})'.format(sentry_unit.info['unit_name']))
+        return None
+
+    def configure_rmq_ssl_on(self, sentry_units, deployment,
+                             port=None, max_wait=60):
+        """Turn ssl charm config option on, with optional non-default
+        ssl port specification.  Confirm that it is enabled on every
+        unit.
+
+        :param sentry_units: list of sentry units
+        :param deployment: amulet deployment object pointer
+        :param port: amqp port, use defaults if None
+        :param max_wait: maximum time to wait in seconds to confirm
+        :returns: None if successful.  Raise on error.
+        """
+        self.log.debug('Setting ssl charm config option:  on')
+
+        # Enable RMQ SSL
+        config = {'ssl': 'on'}
+        if port:
+            config['ssl_port'] = port
+
+        deployment.d.configure('rabbitmq-server', config)
+
+        # Wait for unit status
+        self.rmq_wait_for_cluster(deployment)
+
+        # Confirm
+        tries = 0
+        ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port)
+        while ret and tries < (max_wait / 4):
+            time.sleep(4)
+            self.log.debug('Attempt {}: {}'.format(tries, ret))
+            ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port)
+            tries += 1
+
+        if ret:
+            amulet.raise_status(amulet.FAIL, ret)
+
+    def configure_rmq_ssl_off(self, sentry_units, deployment, max_wait=60):
+        """Turn ssl charm config option off, confirm that it is disabled
+        on every unit.
+
+        :param sentry_units: list of sentry units
+        :param deployment: amulet deployment object pointer
+        :param max_wait: maximum time to wait in seconds to confirm
+        :returns: None if successful.  Raise on error.
+        """
+        self.log.debug('Setting ssl charm config option:  off')
+
+        # Disable RMQ SSL
+        config = {'ssl': 'off'}
+        deployment.d.configure('rabbitmq-server', config)
+
+        # Wait for unit status
+        self.rmq_wait_for_cluster(deployment)
+
+        # Confirm
+        tries = 0
+        ret = self.validate_rmq_ssl_disabled_units(sentry_units)
+        while ret and tries < (max_wait / 4):
+            time.sleep(4)
+            self.log.debug('Attempt {}: {}'.format(tries, ret))
+            ret = self.validate_rmq_ssl_disabled_units(sentry_units)
+            tries += 1
+
+        if ret:
+            amulet.raise_status(amulet.FAIL, ret)
+
+    def connect_amqp_by_unit(self, sentry_unit, ssl=False,
+                             port=None, fatal=True,
+                             username="testuser1", password="changeme"):
+        """Establish and return a pika amqp connection to the rabbitmq service
+        running on a rmq juju unit.
+
+        :param sentry_unit: sentry unit pointer
+        :param ssl: boolean, default to False
+        :param port: amqp port, use defaults if None
+        :param fatal: boolean, default to True (raises on connect error)
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :returns: pika amqp connection pointer or None if failed and non-fatal
+        """
+        host = sentry_unit.info['public-address']
+        unit_name = sentry_unit.info['unit_name']
+
+        # Default port logic if port is not specified
+        if ssl and not port:
+            port = 5671
+        elif not ssl and not port:
+            port = 5672
+
+        self.log.debug('Connecting to amqp on {}:{} ({}) as '
+                       '{}...'.format(host, port, unit_name, username))
+
+        try:
+            credentials = pika.PlainCredentials(username, password)
+            parameters = pika.ConnectionParameters(host=host, port=port,
+                                                   credentials=credentials,
+                                                   ssl=ssl,
+                                                   connection_attempts=3,
+                                                   retry_delay=5,
+                                                   socket_timeout=1)
+            connection = pika.BlockingConnection(parameters)
+            assert connection.server_properties['product'] == 'RabbitMQ'
+            self.log.debug('Connect OK')
+            return connection
+        except Exception as e:
+            msg = ('amqp connection failed to {}:{} as '
+                   '{} ({})'.format(host, port, username, str(e)))
+            if fatal:
+                amulet.raise_status(amulet.FAIL, msg)
+            else:
+                self.log.warn(msg)
+                return None
+
+    def publish_amqp_message_by_unit(self, sentry_unit, message,
+                                     queue="test", ssl=False,
+                                     username="testuser1",
+                                     password="changeme",
+                                     port=None):
+        """Publish an amqp message to a rmq juju unit.
+
+        :param sentry_unit: sentry unit pointer
+        :param message: amqp message string
+        :param queue: message queue, default to test
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :param ssl: boolean, default to False
+        :param port: amqp port, use defaults if None
+        :returns: None.  Raises exception if publish failed.
+        """
+        self.log.debug('Publishing message to {} queue:\n{}'.format(queue,
+                                                                    message))
+        connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl,
+                                               port=port,
+                                               username=username,
+                                               password=password)
+
+        # NOTE(beisner): extra debug here re: pika hang potential:
+        #   https://github.com/pika/pika/issues/297
+        #   https://groups.google.com/forum/#!topic/rabbitmq-users/Ja0iyfF0Szw
+        self.log.debug('Defining channel...')
+        channel = connection.channel()
+        self.log.debug('Declaring queue...')
+        channel.queue_declare(queue=queue, auto_delete=False, durable=True)
+        self.log.debug('Publishing message...')
+        channel.basic_publish(exchange='', routing_key=queue, body=message)
+        self.log.debug('Closing channel...')
+        channel.close()
+        self.log.debug('Closing connection...')
+        connection.close()
+
+    def get_amqp_message_by_unit(self, sentry_unit, queue="test",
+                                 username="testuser1",
+                                 password="changeme",
+                                 ssl=False, port=None):
+        """Get an amqp message from a rmq juju unit.
+
+        :param sentry_unit: sentry unit pointer
+        :param queue: message queue, default to test
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :param ssl: boolean, default to False
+        :param port: amqp port, use defaults if None
+        :returns: amqp message body as string.  Raise if get fails.
+        """
+        connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl,
+                                               port=port,
+                                               username=username,
+                                               password=password)
+        channel = connection.channel()
+        method_frame, _, body = channel.basic_get(queue)
+
+        if method_frame:
+            self.log.debug('Retreived message from {} queue:\n{}'.format(queue,
+                                                                         body))
+            channel.basic_ack(method_frame.delivery_tag)
+            channel.close()
+            connection.close()
+            return body
+        else:
+            msg = 'No message retrieved.'
+            amulet.raise_status(amulet.FAIL, msg)
 
=== modified file 'hooks/charmhelpers/contrib/openstack/context.py'
--- hooks/charmhelpers/contrib/openstack/context.py	2015-06-10 07:35:12 +0000
+++ hooks/charmhelpers/contrib/openstack/context.py	2015-12-01 15:05:49 +0000
@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import glob
 import json
 import os
 import re
@@ -50,6 +51,8 @@
 from charmhelpers.core.strutils import bool_from_string
 
 from charmhelpers.core.host import (
+    get_bond_master,
+    is_phy_iface,
     list_nics,
     get_nic_hwaddr,
     mkdir,
@@ -122,21 +125,24 @@
          of specifying multiple key value pairs within the same string. For
          example, a string in the format of 'key1=value1, key2=value2' will
          return a dict of:
-         {'key1': 'value1',
-          'key2': 'value2'}.
+
+             {'key1': 'value1',
+              'key2': 'value2'}.
 
       2. A string in the above format, but supporting a comma-delimited list
          of values for the same key. For example, a string in the format of
          'key1=value1, key2=value3,value4,value5' will return a dict of:
-         {'key1', 'value1',
-          'key2', 'value2,value3,value4'}
+
+             {'key1', 'value1',
+              'key2', 'value2,value3,value4'}
 
       3. A string containing a colon character (:) prior to an equal
          character (=) will be treated as yaml and parsed as such. This can be
          used to specify more complex key value pairs. For example,
          a string in the format of 'key1: subkey1=value1, subkey2=value2' will
          return a dict of:
-         {'key1', 'subkey1=value1, subkey2=value2'}
+
+             {'key1', 'subkey1=value1, subkey2=value2'}
 
     The provided config_flags string may be a list of comma-separated values
     which themselves may be comma-separated list of values.
@@ -189,10 +195,50 @@
 class OSContextGenerator(object):
     """Base class for all context generators."""
     interfaces = []
+    related = False
+    complete = False
+    missing_data = []
 
     def __call__(self):
         raise NotImplementedError
 
+    def context_complete(self, ctxt):
+        """Check for missing data for the required context data.
+        Set self.missing_data if it exists and return False.
+        Set self.complete if no missing data and return True.
+        """
+        # Fresh start
+        self.complete = False
+        self.missing_data = []
+        for k, v in six.iteritems(ctxt):
+            if v is None or v == '':
+                if k not in self.missing_data:
+                    self.missing_data.append(k)
+
+        if self.missing_data:
+            self.complete = False
+            log('Missing required data: %s' % ' '.join(self.missing_data), level=INFO)
+        else:
+            self.complete = True
+        return self.complete
+
+    def get_related(self):
+        """Check if any of the context interfaces have relation ids.
+        Set self.related and return True if one of the interfaces
+        has relation ids.
+        """
+        # Fresh start
+        self.related = False
+        try:
+            for interface in self.interfaces:
+                if relation_ids(interface):
+                    self.related = True
+            return self.related
+        except AttributeError as e:
+            log("{} {}"
+                "".format(self, e), 'INFO')
+            return self.related
+
 
 class SharedDBContext(OSContextGenerator):
     interfaces = ['shared-db']
@@ -208,6 +254,7 @@
         self.database = database
         self.user = user
         self.ssl_dir = ssl_dir
+        self.rel_name = self.interfaces[0]
 
     def __call__(self):
         self.database = self.database or config('database')
@@ -240,7 +287,8 @@
         if self.relation_prefix:
             password_setting = self.relation_prefix + '_password'
 
-        for rid in relation_ids('shared-db'):
+        for rid in relation_ids(self.interfaces[0]):
+            self.related = True
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
                 host = rdata.get('db_host')
@@ -252,7 +300,7 @@
                     'database_password': rdata.get(password_setting),
                     'database_type': 'mysql'
                 }
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     db_ssl(rdata, ctxt, self.ssl_dir)
                     return ctxt
         return {}
@@ -273,6 +321,7 @@
 
         ctxt = {}
         for rid in relation_ids(self.interfaces[0]):
+            self.related = True
             for unit in related_units(rid):
                 rel_host = relation_get('host', rid=rid, unit=unit)
                 rel_user = relation_get('user', rid=rid, unit=unit)
@@ -282,7 +331,7 @@
                         'database_user': rel_user,
                         'database_password': rel_passwd,
                         'database_type': 'postgresql'}
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     return ctxt
 
         return {}
@@ -343,6 +392,7 @@
             ctxt['signing_dir'] = cachedir
 
         for rid in relation_ids(self.rel_name):
+            self.related = True
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
                 serv_host = rdata.get('service_host')
@@ -361,7 +411,7 @@
                              'service_protocol': svc_protocol,
                              'auth_protocol': auth_protocol})
 
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     # NOTE(jamespage) this is required for >= icehouse
                     # so a missing value just indicates keystone needs
                     # upgrading
@@ -400,6 +450,7 @@
         ctxt = {}
         for rid in relation_ids(self.rel_name):
             ha_vip_only = False
+            self.related = True
             for unit in related_units(rid):
                 if relation_get('clustered', rid=rid, unit=unit):
                     ctxt['clustered'] = True
@@ -432,7 +483,7 @@
                 ha_vip_only = relation_get('ha-vip-only',
                                            rid=rid, unit=unit) is not None
 
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     if 'rabbit_ssl_ca' in ctxt:
                         if not self.ssl_dir:
                             log("Charm not setup for ssl support but ssl ca "
@@ -464,7 +515,7 @@
             ctxt['oslo_messaging_flags'] = config_flags_parser(
                 oslo_messaging_flags)
 
-        if not context_complete(ctxt):
+        if not self.complete:
             return {}
 
         return ctxt
@@ -480,13 +531,15 @@
 
         log('Generating template context for ceph', level=DEBUG)
         mon_hosts = []
-        auth = None
-        key = None
-        use_syslog = str(config('use-syslog')).lower()
+        ctxt = {
+            'use_syslog': str(config('use-syslog')).lower()
+        }
         for rid in relation_ids('ceph'):
             for unit in related_units(rid):
-                auth = relation_get('auth', rid=rid, unit=unit)
-                key = relation_get('key', rid=rid, unit=unit)
+                if not ctxt.get('auth'):
+                    ctxt['auth'] = relation_get('auth', rid=rid, unit=unit)
+                if not ctxt.get('key'):
+                    ctxt['key'] = relation_get('key', rid=rid, unit=unit)
                 ceph_pub_addr = relation_get('ceph-public-address', rid=rid,
                                              unit=unit)
                 unit_priv_addr = relation_get('private-address', rid=rid,
@@ -495,15 +548,12 @@
                 ceph_addr = format_ipv6_addr(ceph_addr) or ceph_addr
                 mon_hosts.append(ceph_addr)
 
-        ctxt = {'mon_hosts': ' '.join(sorted(mon_hosts)),
-                'auth': auth,
-                'key': key,
-                'use_syslog': use_syslog}
+        ctxt['mon_hosts'] = ' '.join(sorted(mon_hosts))
 
         if not os.path.isdir('/etc/ceph'):
             os.mkdir('/etc/ceph')
 
-        if not context_complete(ctxt):
+        if not self.context_complete(ctxt):
             return {}
 
         ensure_packages(['ceph-common'])
@@ -890,9 +940,32 @@
                 'neutron_url': '%s://%s:%s' % (proto, host, '9696')}
         return ctxt
 
+    def pg_ctxt(self):
+        driver = neutron_plugin_attribute(self.plugin, 'driver',
+                                          self.network_manager)
+        config = neutron_plugin_attribute(self.plugin, 'config',
+                                          self.network_manager)
+        ovs_ctxt = {'core_plugin': driver,
+                    'neutron_plugin': 'plumgrid',
+                    'neutron_security_groups': self.neutron_security_groups,
+                    'local_ip': unit_private_ip(),
+                    'config': config}
+        return ovs_ctxt
+
+    def midonet_ctxt(self):
+        driver = neutron_plugin_attribute(self.plugin, 'driver',
+                                          self.network_manager)
+        midonet_config = neutron_plugin_attribute(self.plugin, 'config',
+                                                  self.network_manager)
+        mido_ctxt = {'core_plugin': driver,
+                     'neutron_plugin': 'midonet',
+                     'neutron_security_groups': self.neutron_security_groups,
+                     'local_ip': unit_private_ip(),
+                     'config': midonet_config}
+
+        return mido_ctxt
+
     def __call__(self):
-        self._ensure_packages()
-
         if self.network_manager not in ['quantum', 'neutron']:
             return {}
 
@@ -911,6 +984,10 @@
             ctxt.update(self.calico_ctxt())
         elif self.plugin == 'vsp':
             ctxt.update(self.nuage_ctxt())
+        elif self.plugin == 'plumgrid':
+            ctxt.update(self.pg_ctxt())
+        elif self.plugin == 'midonet':
+            ctxt.update(self.midonet_ctxt())
 
         alchemy_flags = config('neutron-alchemy-flags')
         if alchemy_flags:
@@ -922,7 +999,6 @@
 
 
 class NeutronPortContext(OSContextGenerator):
-    NIC_PREFIXES = ['eth', 'bond']
 
     def resolve_ports(self, ports):
         """Resolve NICs not yet bound to bridge(s)
@@ -934,7 +1010,18 @@
 
         hwaddr_to_nic = {}
         hwaddr_to_ip = {}
-        for nic in list_nics(self.NIC_PREFIXES):
+        for nic in list_nics():
+            # Ignore virtual interfaces (bond masters will be identified from
+            # their slaves)
+            if not is_phy_iface(nic):
+                continue
+
+            _nic = get_bond_master(nic)
+            if _nic:
+                log("Replacing iface '%s' with bond master '%s'" % (nic, _nic),
+                    level=DEBUG)
+                nic = _nic
+
             hwaddr = get_nic_hwaddr(nic)
             hwaddr_to_nic[hwaddr] = nic
             addresses = get_ipv4_addr(nic, fatal=False)
@@ -960,7 +1047,8 @@
                 # trust it to be the real external network).
                 resolved.append(entry)
 
-        return resolved
+        # Ensure no duplicates
+        return list(set(resolved))
 
 
 class OSConfigFlagContext(OSContextGenerator):
@@ -1000,6 +1088,20 @@
                 config_flags_parser(config_flags)}
 
 
+class LibvirtConfigFlagsContext(OSContextGenerator):
+    """
+    This context provides support for extending
+    the libvirt section through user-defined flags.
+    """
+    def __call__(self):
+        ctxt = {}
+        libvirt_flags = config('libvirt-flags')
+        if libvirt_flags:
+            ctxt['libvirt_flags'] = config_flags_parser(
+                libvirt_flags)
+        return ctxt
+
+
 class SubordinateConfigContext(OSContextGenerator):
 
     """
@@ -1032,7 +1134,7 @@
 
         ctxt = {
             ... other context ...
-            'subordinate_config': {
+            'subordinate_configuration': {
                 'DEFAULT': {
                     'key1': 'value1',
                 },
@@ -1050,13 +1152,22 @@
         :param config_file : Service's config file to query sections
         :param interface   : Subordinate interface to inspect
         """
-        self.service = service
         self.config_file = config_file
-        self.interface = interface
+        if isinstance(service, list):
+            self.services = service
+        else:
+            self.services = [service]
+        if isinstance(interface, list):
+            self.interfaces = interface
+        else:
+            self.interfaces = [interface]
 
     def __call__(self):
         ctxt = {'sections': {}}
-        for rid in relation_ids(self.interface):
+        rids = []
+        for interface in self.interfaces:
+            rids.extend(relation_ids(interface))
+        for rid in rids:
             for unit in related_units(rid):
                 sub_config = relation_get('subordinate_configuration',
                                           rid=rid, unit=unit)
@@ -1064,33 +1175,37 @@
                     try:
                         sub_config = json.loads(sub_config)
                     except:
-                        log('Could not parse JSON from subordinate_config '
-                            'setting from %s' % rid, level=ERROR)
-                        continue
-
-                    if self.service not in sub_config:
-                        log('Found subordinate_config on %s but it contained'
-                            'nothing for %s service' % (rid, self.service),
-                            level=INFO)
-                        continue
-
-                    sub_config = sub_config[self.service]
-                    if self.config_file not in sub_config:
-                        log('Found subordinate_config on %s but it contained'
-                            'nothing for %s' % (rid, self.config_file),
-                            level=INFO)
-                        continue
-
-                    sub_config = sub_config[self.config_file]
-                    for k, v in six.iteritems(sub_config):
-                        if k == 'sections':
-                            for section, config_dict in six.iteritems(v):
-                                log("adding section '%s'" % (section),
-                                    level=DEBUG)
-                                ctxt[k][section] = config_dict
-                        else:
-                            ctxt[k] = v
-
+                        log('Could not parse JSON from '
+                            'subordinate_configuration setting from %s'
+                            % rid, level=ERROR)
+                        continue
+
+                    for service in self.services:
+                        if service not in sub_config:
+                            log('Found subordinate_configuration on %s but it '
+                                'contained nothing for %s service'
+                                % (rid, service), level=INFO)
+                            continue
+
+                        sub_config = sub_config[service]
+                        if self.config_file not in sub_config:
+                            log('Found subordinate_configuration on %s but it '
+                                'contained nothing for %s'
+                                % (rid, self.config_file), level=INFO)
+                            continue
+
+                        sub_config = sub_config[self.config_file]
+                        for k, v in six.iteritems(sub_config):
+                            if k == 'sections':
+                                for section, config_list in six.iteritems(v):
+                                    log("adding section '%s'" % (section),
+                                        level=DEBUG)
+                                    if ctxt[k].get(section):
+                                        ctxt[k][section].extend(config_list)
+                                    else:
+                                        ctxt[k][section] = config_list
+                            else:
+                                ctxt[k] = v
         log("%d section(s) found" % (len(ctxt['sections'])), level=DEBUG)
         return ctxt
 
@@ -1267,15 +1382,19 @@
     def __call__(self):
         ports = config('data-port')
         if ports:
+            # Map of {port/mac:bridge}
             portmap = parse_data_port_mappings(ports)
-            ports = portmap.values()
+            ports = portmap.keys()
+            # Resolve provided ports or mac addresses and filter out those
+            # already attached to a bridge.
             resolved = self.resolve_ports(ports)
+            # FIXME: is this necessary?
             normalized = {get_nic_hwaddr(port): port for port in resolved
                           if port not in ports}
             normalized.update({port: port for port in resolved
                                if port in ports})
             if resolved:
-                return {bridge: normalized[port] for bridge, port in
+                return {normalized[port]: bridge for port, bridge in
                         six.iteritems(portmap) if port in normalized.keys()}
 
         return None
@@ -1286,12 +1405,22 @@
     def __call__(self):
         ctxt = {}
         mappings = super(PhyNICMTUContext, self).__call__()
-        if mappings and mappings.values():
-            ports = mappings.values()
+        if mappings and mappings.keys():
+            ports = sorted(mappings.keys())
             napi_settings = NeutronAPIContext()()
             mtu = napi_settings.get('network_device_mtu')
+            all_ports = set()
+            # If any of ports is a vlan device, its underlying device must have
+            # mtu applied first.
+            for port in ports:
+                for lport in glob.glob("/sys/class/net/%s/lower_*" % port):
+                    lport = os.path.basename(lport)
+                    all_ports.add(lport.split('_')[1])
+
+            all_ports = list(all_ports)
+            all_ports.extend(ports)
             if mtu:
-                ctxt["devs"] = '\\n'.join(ports)
+                ctxt["devs"] = '\\n'.join(all_ports)
                 ctxt['mtu'] = mtu
 
         return ctxt
@@ -1323,6 +1452,6 @@
                     'auth_protocol':
                     rdata.get('auth_protocol') or 'http',
                 }
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     return ctxt
         return {}
 
=== modified file 'hooks/charmhelpers/contrib/openstack/neutron.py'
--- hooks/charmhelpers/contrib/openstack/neutron.py	2015-06-10 15:45:48 +0000
+++ hooks/charmhelpers/contrib/openstack/neutron.py	2015-12-01 15:05:49 +0000
@@ -195,6 +195,34 @@
             'packages': [],
             'server_packages': ['neutron-server', 'neutron-plugin-nuage'],
             'server_services': ['neutron-server']
+        },
+        'plumgrid': {
+            'config': '/etc/neutron/plugins/plumgrid/plumgrid.ini',
+            'driver': 'neutron.plugins.plumgrid.plumgrid_plugin.plumgrid_plugin.NeutronPluginPLUMgridV2',
+            'contexts': [
+                context.SharedDBContext(user=config('database-user'),
+                                        database=config('database'),
+                                        ssl_dir=NEUTRON_CONF_DIR)],
+            'services': [],
+            'packages': ['plumgrid-lxc',
+                         'iovisor-dkms'],
+            'server_packages': ['neutron-server',
+                                'neutron-plugin-plumgrid'],
+            'server_services': ['neutron-server']
+        },
+        'midonet': {
+            'config': '/etc/neutron/plugins/midonet/midonet.ini',
+            'driver': 'midonet.neutron.plugin.MidonetPluginV2',
+            'contexts': [
+                context.SharedDBContext(user=config('neutron-database-user'),
+                                        database=config('neutron-database'),
+                                        relation_prefix='neutron',
+                                        ssl_dir=NEUTRON_CONF_DIR)],
+            'services': [],
+            'packages': [[headers_package()] + determine_dkms_package()],
+            'server_packages': ['neutron-server',
+                                'python-neutron-plugin-midonet'],
+            'server_services': ['neutron-server']
         }
     }
     if release >= 'icehouse':
@@ -255,17 +283,30 @@
         return 'neutron'
 
 
-def parse_mappings(mappings):
+def parse_mappings(mappings, key_rvalue=False):
+    """By default mappings are lvalue keyed.
+
+    If key_rvalue is True, the mapping will be reversed to allow multiple
+    configs for the same lvalue.
+    """
     parsed = {}
     if mappings:
         mappings = mappings.split()
         for m in mappings:
             p = m.partition(':')
-            key = p[0].strip()
-            if p[1]:
-                parsed[key] = p[2].strip()
+
+            if key_rvalue:
+                key_index = 2
+                val_index = 0
+                # if there is no rvalue skip to next
+                if not p[1]:
+                    continue
             else:
-                parsed[key] = ''
+                key_index = 0
+                val_index = 2
+
+            key = p[key_index].strip()
+            parsed[key] = p[val_index].strip()
 
     return parsed
 
@@ -283,25 +324,25 @@
 def parse_data_port_mappings(mappings, default_bridge='br-data'):
     """Parse data port mappings.
 
-    Mappings must be a space-delimited list of bridge:port mappings.
+    Mappings must be a space-delimited list of bridge:port.
 
-    Returns dict of the form {bridge:port}.
+    Returns dict of the form {port:bridge} where ports may be mac addresses or
+    interface names.
     """
-    _mappings = parse_mappings(mappings)
+
+    # NOTE(dosaboy): we use rvalue for key to allow multiple values to be
+    # proposed for <port> since it may be a mac address which will differ
+    # across units this allowing first-known-good to be chosen.
+    _mappings = parse_mappings(mappings, key_rvalue=True)
     if not _mappings or list(_mappings.values()) == ['']:
         if not mappings:
             return {}
 
         # For backwards-compatibility we need to support port-only provided in
         # config.
-        _mappings = {default_bridge: mappings.split()[0]}
-
-    bridges = _mappings.keys()
-    ports = _mappings.values()
-    if len(set(bridges)) != len(bridges):
-        raise Exception("It is not allowed to have more than one port "
-                        "configured on the same bridge")
-
+        _mappings = {mappings.split()[0]: default_bridge}
+
+    ports = _mappings.keys()
     if len(set(ports)) != len(ports):
         raise Exception("It is not allowed to have the same port configured "
                         "on more than one bridge")
 
=== modified file 'hooks/charmhelpers/contrib/openstack/templates/ceph.conf'
--- hooks/charmhelpers/contrib/openstack/templates/ceph.conf	2015-06-10 07:35:12 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/ceph.conf	2015-12-01 15:05:49 +0000
@@ -5,11 +5,17 @@
 ###############################################################################
 [global]
 {% if auth -%}
- auth_supported = {{ auth }}
- keyring = /etc/ceph/$cluster.$name.keyring
- mon host = {{ mon_hosts }}
+auth_supported = {{ auth }}
+keyring = /etc/ceph/$cluster.$name.keyring
+mon host = {{ mon_hosts }}
 {% endif -%}
- log to syslog = {{ use_syslog }}
- err to syslog = {{ use_syslog }}
- clog to syslog = {{ use_syslog }}
+log to syslog = {{ use_syslog }}
+err to syslog = {{ use_syslog }}
+clog to syslog = {{ use_syslog }}
 
+[client]
+{% if rbd_client_cache_settings -%}
+{% for key, value in rbd_client_cache_settings.iteritems() -%}
+{{ key }} = {{ value }}
+{% endfor -%}
+{%- endif %}
 \ No newline at end of file
 
=== modified file 'hooks/charmhelpers/contrib/openstack/templating.py'
--- hooks/charmhelpers/contrib/openstack/templating.py	2015-06-11 14:20:09 +0000
+++ hooks/charmhelpers/contrib/openstack/templating.py	2015-12-01 15:05:49 +0000
@@ -18,7 +18,7 @@
 
 import six
 
-from charmhelpers.fetch import apt_install
+from charmhelpers.fetch import apt_install, apt_update
 from charmhelpers.core.hookenv import (
     log,
     ERROR,
@@ -29,39 +29,15 @@
 try:
     from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
 except ImportError:
-    # python-jinja2 may not be installed yet, or we're running unittests.
-    FileSystemLoader = ChoiceLoader = Environment = exceptions = None
+    apt_update(fatal=True)
+    apt_install('python-jinja2', fatal=True)
+    from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
 
 
 class OSConfigException(Exception):
     pass
 
 
-def os_template_dirs(templates_dir, os_release):
-    tmpl_dirs = [(rel, os.path.join(templates_dir, rel))
-                 for rel in six.itervalues(OPENSTACK_CODENAMES)]
-
-    if not os.path.isdir(templates_dir):
-        log('Templates directory not found @ %s.' % templates_dir,
-            level=ERROR)
-        raise OSConfigException
-    dirs = [templates_dir]
-    helper_templates = os.path.join(os.path.dirname(__file__), 'templates')
-    if os.path.isdir(helper_templates):
-        dirs.append(helper_templates)
-
-    for rel, tmpl_dir in tmpl_dirs:
-        if os.path.isdir(tmpl_dir):
-            dirs.insert(0, tmpl_dir)
-        if rel == os_release:
-            break
-    ch_templates = os.path.dirname(__file__) + '/charmhelpers/contrib/openstack/templates'
-    dirs.append(ch_templates)
-    log('Template search path: %s' %
-        ' '.join(dirs), level=INFO)
-    return dirs
-
-
 def get_loader(templates_dir, os_release):
     """
     Create a jinja2.ChoiceLoader containing template dirs up to
@@ -137,7 +113,7 @@
 
     def complete_contexts(self):
         '''
-        Return a list of interfaces that have atisfied contexts.
+        Return a list of interfaces that have satisfied contexts.
         '''
         if self._complete_contexts:
             return self._complete_contexts
@@ -318,3 +294,30 @@
         [interfaces.extend(i.complete_contexts())
          for i in six.itervalues(self.templates)]
         return interfaces
+
+    def get_incomplete_context_data(self, interfaces):
+        '''
+        Return dictionary of relation status of interfaces and any missing
+        required context data. Example:
+            {'amqp': {'missing_data': ['rabbitmq_password'], 'related': True},
+             'zeromq-configuration': {'related': False}}
+        '''
+        incomplete_context_data = {}
+
+        for i in six.itervalues(self.templates):
+            for context in i.contexts:
+                for interface in interfaces:
+                    related = False
+                    if interface in context.interfaces:
+                        related = context.get_related()
+                        missing_data = context.missing_data
+                        if missing_data:
+                            incomplete_context_data[interface] = {'missing_data': missing_data}
+                        if related:
+                            if incomplete_context_data.get(interface):
+                                incomplete_context_data[interface].update({'related': True})
+                            else:
+                                incomplete_context_data[interface] = {'related': True}
+                        else:
+                            incomplete_context_data[interface] = {'related': False}
+        return incomplete_context_data
 
=== modified file 'hooks/charmhelpers/contrib/openstack/utils.py'
--- hooks/charmhelpers/contrib/openstack/utils.py	2015-06-17 12:23:31 +0000
+++ hooks/charmhelpers/contrib/openstack/utils.py	2015-12-01 15:05:49 +0000
@@ -1,5 +1,3 @@
-#!/usr/bin/python
-
 # Copyright 2014-2015 Canonical Limited.
 #
 # This file is part of charm-helpers.
@@ -24,9 +22,11 @@
 import json
 import os
 import sys
+import re
+
+import six
+import traceback
 import uuid
-
-import six
 import yaml
 
 from charmhelpers.contrib.network import ip
@@ -36,13 +36,17 @@
 )
 
 from charmhelpers.core.hookenv import (
+    action_fail,
+    action_set,
     config,
     log as juju_log,
     charm_dir,
     INFO,
+    related_units,
     relation_ids,
-    related_units,
     relation_set,
+    status_set,
+    hook_name
 )
 
 from charmhelpers.contrib.storage.linux.lvm import (
@@ -52,7 +56,8 @@
 )
 
 from charmhelpers.contrib.network.ip import (
-    get_ipv6_addr
+    get_ipv6_addr,
+    is_ipv6,
 )
 
 from charmhelpers.contrib.python.packages import (
@@ -71,7 +76,6 @@
 DISTRO_PROPOSED = ('deb http://archive.ubuntu.com/ubuntu/ %s-proposed '
                    'restricted main multiverse universe')
 
-
 UBUNTU_OPENSTACK_RELEASE = OrderedDict([
     ('oneiric', 'diablo'),
     ('precise', 'essex'),
@@ -81,6 +85,7 @@
     ('trusty', 'icehouse'),
     ('utopic', 'juno'),
     ('vivid', 'kilo'),
+    ('wily', 'liberty'),
 ])
 
 
@@ -93,6 +98,7 @@
     ('2014.1', 'icehouse'),
     ('2014.2', 'juno'),
     ('2015.1', 'kilo'),
+    ('2015.2', 'liberty'),
 ])
 
 # The ugly duckling
@@ -115,8 +121,42 @@
     ('2.2.0', 'juno'),
     ('2.2.1', 'kilo'),
     ('2.2.2', 'kilo'),
+    ('2.3.0', 'liberty'),
+    ('2.4.0', 'liberty'),
+    ('2.5.0', 'liberty'),
 ])
 
+# >= Liberty version->codename mapping
+PACKAGE_CODENAMES = {
+    'nova-common': OrderedDict([
+        ('12.0.0', 'liberty'),
+    ]),
+    'neutron-common': OrderedDict([
+        ('7.0.0', 'liberty'),
+    ]),
+    'cinder-common': OrderedDict([
+        ('7.0.0', 'liberty'),
+    ]),
+    'keystone': OrderedDict([
+        ('8.0.0', 'liberty'),
+    ]),
+    'horizon-common': OrderedDict([
+        ('8.0.0', 'liberty'),
+    ]),
+    'ceilometer-common': OrderedDict([
+        ('5.0.0', 'liberty'),
+    ]),
+    'heat-common': OrderedDict([
+        ('5.0.0', 'liberty'),
+    ]),
+    'glance-common': OrderedDict([
+        ('11.0.0', 'liberty'),
+    ]),
+    'openstack-dashboard': OrderedDict([
+        ('8.0.0', 'liberty'),
+    ]),
+}
+
 DEFAULT_LOOPBACK_SIZE = '5G'
 
 
@@ -166,9 +206,9 @@
         error_out(e)
 
 
-def get_os_version_codename(codename):
+def get_os_version_codename(codename, version_map=OPENSTACK_CODENAMES):
     '''Determine OpenStack version number from codename.'''
-    for k, v in six.iteritems(OPENSTACK_CODENAMES):
+    for k, v in six.iteritems(version_map):
         if v == codename:
             return k
     e = 'Could not derive OpenStack version for '\
@@ -200,20 +240,31 @@
         error_out(e)
 
     vers = apt.upstream_version(pkg.current_ver.ver_str)
+    match = re.match('^(\d+)\.(\d+)\.(\d+)', vers)
+    if match:
+        vers = match.group(0)
 
-    try:
-        if 'swift' in pkg.name:
-            swift_vers = vers[:5]
-            if swift_vers not in SWIFT_CODENAMES:
-                # Deal with 1.10.0 upward
-                swift_vers = vers[:6]
-            return SWIFT_CODENAMES[swift_vers]
-        else:
-            vers = vers[:6]
-            return OPENSTACK_CODENAMES[vers]
-    except KeyError:
-        e = 'Could not determine OpenStack codename for version %s' % vers
-        error_out(e)
+    # >= Liberty independent project versions
+    if (package in PACKAGE_CODENAMES and
+            vers in PACKAGE_CODENAMES[package]):
+        return PACKAGE_CODENAMES[package][vers]
+    else:
+        # < Liberty co-ordinated project versions
+        try:
+            if 'swift' in pkg.name:
+                swift_vers = vers[:5]
+                if swift_vers not in SWIFT_CODENAMES:
+                    # Deal with 1.10.0 upward
+                    swift_vers = vers[:6]
+                return SWIFT_CODENAMES[swift_vers]
+            else:
+                vers = vers[:6]
+                return OPENSTACK_CODENAMES[vers]
+        except KeyError:
+            if not fatal:
+                return None
+            e = 'Could not determine OpenStack codename for version %s' % vers
+            error_out(e)
 
 
 def get_os_version_package(pkg, fatal=True):
@@ -323,6 +374,9 @@
             'kilo': 'trusty-updates/kilo',
             'kilo/updates': 'trusty-updates/kilo',
             'kilo/proposed': 'trusty-proposed/kilo',
+            'liberty': 'trusty-updates/liberty',
+            'liberty/updates': 'trusty-updates/liberty',
+            'liberty/proposed': 'trusty-proposed/liberty',
         }
 
         try:
@@ -388,7 +442,11 @@
     import apt_pkg as apt
     src = config('openstack-origin')
     cur_vers = get_os_version_package(package)
-    available_vers = get_os_version_install_source(src)
+    if "swift" in package:
+        codename = get_os_codename_install_source(src)
+        available_vers = get_os_version_codename(codename, SWIFT_CODENAMES)
+    else:
+        available_vers = get_os_version_install_source(src)
     apt.init()
     return apt.version_compare(available_vers, cur_vers) == 1
 
@@ -465,6 +523,12 @@
                                       relation_prefix=None):
     hosts = get_ipv6_addr(dynamic_only=False)
 
+    if config('vip'):
+        vips = config('vip').split()
+        for vip in vips:
+            if vip and is_ipv6(vip):
+                hosts.append(vip)
+
     kwargs = {'database': database,
               'username': database_user,
               'hostname': json.dumps(hosts)}
@@ -518,6 +582,7 @@
     Clone/install all specified OpenStack repositories.
 
     The expected format of projects_yaml is:
+
         repositories:
           - {name: keystone,
              repository: 'git://git.openstack.org/openstack/keystone.git',
@@ -525,11 +590,13 @@
           - {name: requirements,
              repository: 'git://git.openstack.org/openstack/requirements.git',
              branch: 'stable/icehouse'}
+
         directory: /mnt/openstack-git
         http_proxy: squid-proxy-url
         https_proxy: squid-proxy-url
 
-        The directory, http_proxy, and https_proxy keys are optional.
+    The directory, http_proxy, and https_proxy keys are optional.
+
     """
     global requirements_dir
     parent_dir = '/mnt/openstack-git'
@@ -551,6 +618,12 @@
 
     pip_create_virtualenv(os.path.join(parent_dir, 'venv'))
 
+    # Upgrade setuptools and pip from default virtualenv versions. The default
+    # versions in trusty break master OpenStack branch deployments.
+    for p in ['pip', 'setuptools']:
+        pip_install(p, upgrade=True, proxy=http_proxy,
+                    venv=os.path.join(parent_dir, 'venv'))
+
     for p in projects['repositories']:
         repo = p['repository']
         branch = p['branch']
@@ -612,24 +685,24 @@
     else:
         repo_dir = dest_dir
 
+    venv = os.path.join(parent_dir, 'venv')
+
     if update_requirements:
         if not requirements_dir:
             error_out('requirements repo must be cloned before '
                       'updating from global requirements.')
-        _git_update_requirements(repo_dir, requirements_dir)
+        _git_update_requirements(venv, repo_dir, requirements_dir)
 
     juju_log('Installing git repo from dir: {}'.format(repo_dir))
     if http_proxy:
-        pip_install(repo_dir, proxy=http_proxy,
-                    venv=os.path.join(parent_dir, 'venv'))
+        pip_install(repo_dir, proxy=http_proxy, venv=venv)
     else:
-        pip_install(repo_dir,
-                    venv=os.path.join(parent_dir, 'venv'))
+        pip_install(repo_dir, venv=venv)
 
     return repo_dir
 
 
-def _git_update_requirements(package_dir, reqs_dir):
+def _git_update_requirements(venv, package_dir, reqs_dir):
     """
     Update from global requirements.
 
@@ -638,12 +711,14 @@
     """
     orig_dir = os.getcwd()
     os.chdir(reqs_dir)
-    cmd = ['python', 'update.py', package_dir]
+    python = os.path.join(venv, 'bin/python')
+    cmd = [python, 'update.py', package_dir]
     try:
         subprocess.check_call(cmd)
     except subprocess.CalledProcessError:
         package = os.path.basename(package_dir)
-        error_out("Error updating {} from global-requirements.txt".format(package))
+        error_out("Error updating {} from "
+                  "global-requirements.txt".format(package))
     os.chdir(orig_dir)
 
 
@@ -691,6 +766,222 @@
     return None
 
 
+def os_workload_status(configs, required_interfaces, charm_func=None):
+    """
+    Decorator to set workload status based on complete contexts
+    """
+    def wrap(f):
+        @wraps(f)
+        def wrapped_f(*args, **kwargs):
+            # Run the original function first
+            f(*args, **kwargs)
+            # Set workload status now that contexts have been
+            # acted on
+            set_os_workload_status(configs, required_interfaces, charm_func)
+        return wrapped_f
+    return wrap
+
+
+def set_os_workload_status(configs, required_interfaces, charm_func=None):
+    """
+    Set workload status based on complete contexts.
+    status-set missing or incomplete contexts
+    and juju-log details of missing required data.
+    charm_func is a charm specific function to run checking
+    for charm specific requirements such as a VIP setting.
+    """
+    incomplete_rel_data = incomplete_relation_data(configs, required_interfaces)
+    state = 'active'
+    missing_relations = []
+    incomplete_relations = []
+    message = None
+    charm_state = None
+    charm_message = None
+
+    for generic_interface in incomplete_rel_data.keys():
+        related_interface = None
+        missing_data = {}
+        # Related or not?
+        for interface in incomplete_rel_data[generic_interface]:
+            if incomplete_rel_data[generic_interface][interface].get('related'):
+                related_interface = interface
+                missing_data = incomplete_rel_data[generic_interface][interface].get('missing_data')
+        # No relation ID for the generic_interface
+        if not related_interface:
+            juju_log("{} relation is missing and must be related for "
+                     "functionality. ".format(generic_interface), 'WARN')
+            state = 'blocked'
+            if generic_interface not in missing_relations:
+                missing_relations.append(generic_interface)
+        else:
+            # Relation ID exists but no related unit
+            if not missing_data:
+                # Edge case relation ID exists but departing
+                if ('departed' in hook_name() or 'broken' in hook_name()) \
+                        and related_interface in hook_name():
+                    state = 'blocked'
+                    if generic_interface not in missing_relations:
+                        missing_relations.append(generic_interface)
+                    juju_log("{} relation's interface, {}, "
+                             "relationship is departed or broken "
+                             "and is required for functionality."
+                             "".format(generic_interface, related_interface), "WARN")
+                # Normal case relation ID exists but no related unit
+                # (joining)
+                else:
+                    juju_log("{} relations's interface, {}, is related but has "
+                             "no units in the relation."
+                             "".format(generic_interface, related_interface), "INFO")
+            # Related unit exists and data missing on the relation
+            else:
+                juju_log("{} relation's interface, {}, is related awaiting "
+                         "the following data from the relationship: {}. "
+                         "".format(generic_interface, related_interface,
+                                   ", ".join(missing_data)), "INFO")
+            if state != 'blocked':
+                state = 'waiting'
+            if generic_interface not in incomplete_relations \
+                    and generic_interface not in missing_relations:
+                incomplete_relations.append(generic_interface)
+
+    if missing_relations:
+        message = "Missing relations: {}".format(", ".join(missing_relations))
+        if incomplete_relations:
+            message += "; incomplete relations: {}" \
+                       "".format(", ".join(incomplete_relations))
+        state = 'blocked'
+    elif incomplete_relations:
+        message = "Incomplete relations: {}" \
+                  "".format(", ".join(incomplete_relations))
+        state = 'waiting'
+
+    # Run charm specific checks
+    if charm_func:
+        charm_state, charm_message = charm_func(configs)
+        if charm_state != 'active' and charm_state != 'unknown':
+            state = workload_state_compare(state, charm_state)
+            if message:
+                charm_message = charm_message.replace("Incomplete relations: ",
+                                                      "")
+                message = "{}, {}".format(message, charm_message)
+            else:
+                message = charm_message
+
+    # Set to active if all requirements have been met
+    if state == 'active':
+        message = "Unit is ready"
+        juju_log(message, "INFO")
+
+    status_set(state, message)
+
+
+def workload_state_compare(current_workload_state, workload_state):
+    """ Return highest priority of two states"""
+    hierarchy = {'unknown': -1,
+                 'active': 0,
+                 'maintenance': 1,
+                 'waiting': 2,
+                 'blocked': 3,
+                 }
+
+    if hierarchy.get(workload_state) is None:
+        workload_state = 'unknown'
+    if hierarchy.get(current_workload_state) is None:
+        current_workload_state = 'unknown'
+
+    # Set workload_state based on hierarchy of statuses
+    if hierarchy.get(current_workload_state) > hierarchy.get(workload_state):
+        return current_workload_state
+    else:
+        return workload_state
+
+
+def incomplete_relation_data(configs, required_interfaces):
+    """
+    Check complete contexts against required_interfaces
+    Return dictionary of incomplete relation data.
+
+    configs is an OSConfigRenderer object with configs registered
+
+    required_interfaces is a dictionary of required general interfaces
+    with dictionary values of possible specific interfaces.
+    Example:
+    required_interfaces = {'database': ['shared-db', 'pgsql-db']}
+
+    The interface is said to be satisfied if anyone of the interfaces in the
+    list has a complete context.
+
+    Return dictionary of incomplete or missing required contexts with relation
+    status of interfaces and any missing data points. Example:
+        {'message':
+             {'amqp': {'missing_data': ['rabbitmq_password'], 'related': True},
+              'zeromq-configuration': {'related': False}},
+         'identity':
+             {'identity-service': {'related': False}},
+         'database':
+             {'pgsql-db': {'related': False},
+              'shared-db': {'related': True}}}
+    """
+    complete_ctxts = configs.complete_contexts()
+    incomplete_relations = []
+    for svc_type in required_interfaces.keys():
+        # Avoid duplicates
+        found_ctxt = False
+        for interface in required_interfaces[svc_type]:
+            if interface in complete_ctxts:
+                found_ctxt = True
+        if not found_ctxt:
+            incomplete_relations.append(svc_type)
+    incomplete_context_data = {}
+    for i in incomplete_relations:
+        incomplete_context_data[i] = configs.get_incomplete_context_data(required_interfaces[i])
+    return incomplete_context_data
+
+
+def do_action_openstack_upgrade(package, upgrade_callback, configs):
+    """Perform action-managed OpenStack upgrade.
+
+    Upgrades packages to the configured openstack-origin version and sets
+    the corresponding action status as a result.
+
+    If the charm was installed from source we cannot upgrade it.
+    For backwards compatibility a config flag (action-managed-upgrade) must
+    be set for this code to run, otherwise a full service level upgrade will
+    fire on config-changed.
+
+    @param package: package name for determining if upgrade available
+    @param upgrade_callback: function callback to charm's upgrade function
+    @param configs: templating object derived from OSConfigRenderer class
+
+    @return: True if upgrade successful; False if upgrade failed or skipped
+    """
+    ret = False
+
+    if git_install_requested():
+        action_set({'outcome': 'installed from source, skipped upgrade.'})
+    else:
+        if openstack_upgrade_available(package):
+            if config('action-managed-upgrade'):
+                juju_log('Upgrading OpenStack release')
+
+                try:
+                    upgrade_callback(configs=configs)
+                    action_set({'outcome': 'success, upgrade completed.'})
+                    ret = True
+                except:
+                    action_set({'outcome': 'upgrade failed, see traceback.'})
+                    action_set({'traceback': traceback.format_exc()})
+                    action_fail('do_openstack_upgrade resulted in an '
+                                'unexpected error')
+            else:
+                action_set({'outcome': 'action-managed-upgrade config is '
+                                       'False, skipped upgrade.'})
+        else:
+            action_set({'outcome': 'no upgrade available.'})
+
+    return ret
+
+
 def remote_restart(rel_name, remote_service=None):
     trigger = {
         'restart-trigger': str(uuid.uuid4()),
@@ -700,7 +991,7 @@
     for rid in relation_ids(rel_name):
         # This subordinate can be related to two seperate services using
         # different subordinate relations so only issue the restart if
-        # thr principle is conencted down the relation we think it is
+        # the principle is conencted down the relation we think it is
         if related_units(relid=rid):
             relation_set(relation_id=rid,
                          relation_settings=trigger,
 
=== modified file 'hooks/charmhelpers/contrib/python/packages.py'
--- hooks/charmhelpers/contrib/python/packages.py	2015-06-10 15:45:48 +0000
+++ hooks/charmhelpers/contrib/python/packages.py	2015-12-01 15:05:49 +0000
@@ -36,6 +36,8 @@
 def parse_options(given, available):
     """Given a set of options, check if available"""
     for key, value in sorted(given.items()):
+        if not value:
+            continue
         if key in available:
             yield "--{0}={1}".format(key, value)
 
 
=== modified file 'hooks/charmhelpers/contrib/storage/linux/ceph.py'
--- hooks/charmhelpers/contrib/storage/linux/ceph.py	2015-06-10 15:45:48 +0000
+++ hooks/charmhelpers/contrib/storage/linux/ceph.py	2015-12-01 15:05:49 +0000
@@ -26,8 +26,10 @@
 
 import os
 import shutil
+import six
 import json
 import time
+import uuid
 
 from subprocess import (
     check_call,
@@ -35,8 +37,10 @@
     CalledProcessError,
 )
 from charmhelpers.core.hookenv import (
+    local_unit,
     relation_get,
     relation_ids,
+    relation_set,
     related_units,
     log,
     DEBUG,
@@ -56,16 +60,18 @@
     apt_install,
 )
 
+from charmhelpers.core.kernel import modprobe
+
 KEYRING = '/etc/ceph/ceph.client.{}.keyring'
 KEYFILE = '/etc/ceph/ceph.client.{}.key'
 
 CEPH_CONF = """[global]
- auth supported = {auth}
- keyring = {keyring}
- mon host = {mon_hosts}
- log to syslog = {use_syslog}
- err to syslog = {use_syslog}
- clog to syslog = {use_syslog}
+auth supported = {auth}
+keyring = {keyring}
+mon host = {mon_hosts}
+log to syslog = {use_syslog}
+err to syslog = {use_syslog}
+clog to syslog = {use_syslog}
 """
 
 
@@ -120,29 +126,37 @@
     return None
 
 
-def create_pool(service, name, replicas=3):
+def update_pool(client, pool, settings):
+    cmd = ['ceph', '--id', client, 'osd', 'pool', 'set', pool]
+    for k, v in six.iteritems(settings):
+        cmd.append(k)
+        cmd.append(v)
+
+    check_call(cmd)
+
+
+def create_pool(service, name, replicas=3, pg_num=None):
     """Create a new RADOS pool."""
     if pool_exists(service, name):
         log("Ceph pool {} already exists, skipping creation".format(name),
             level=WARNING)
         return
 
-    # Calculate the number of placement groups based
-    # on upstream recommended best practices.
-    osds = get_osds(service)
-    if osds:
-        pgnum = (len(osds) * 100 // replicas)
-    else:
-        # NOTE(james-page): Default to 200 for older ceph versions
-        # which don't support OSD query from cli
-        pgnum = 200
-
-    cmd = ['ceph', '--id', service, 'osd', 'pool', 'create', name, str(pgnum)]
-    check_call(cmd)
-
-    cmd = ['ceph', '--id', service, 'osd', 'pool', 'set', name, 'size',
-           str(replicas)]
-    check_call(cmd)
+    if not pg_num:
+        # Calculate the number of placement groups based
+        # on upstream recommended best practices.
+        osds = get_osds(service)
+        if osds:
+            pg_num = (len(osds) * 100 // replicas)
+        else:
+            # NOTE(james-page): Default to 200 for older ceph versions
+            # which don't support OSD query from cli
+            pg_num = 200
+
+    cmd = ['ceph', '--id', service, 'osd', 'pool', 'create', name, str(pg_num)]
+    check_call(cmd)
+
+    update_pool(service, name, settings={'size': str(replicas)})
 
 
 def delete_pool(service, name):
@@ -197,10 +211,10 @@
     log('Created new keyfile at %s.' % keyfile, level=INFO)
 
 
-def get_ceph_nodes():
-    """Query named relation 'ceph' to determine current nodes."""
+def get_ceph_nodes(relation='ceph'):
+    """Query named relation to determine current nodes."""
     hosts = []
-    for r_id in relation_ids('ceph'):
+    for r_id in relation_ids(relation):
         for unit in related_units(r_id):
             hosts.append(relation_get('private-address', unit=unit, rid=r_id))
 
@@ -288,17 +302,6 @@
     os.chown(data_src_dst, uid, gid)
 
 
-# TODO: re-use
-def modprobe(module):
-    """Load a kernel module and configure for auto-load on reboot."""
-    log('Loading kernel module', level=INFO)
-    cmd = ['modprobe', module]
-    check_call(cmd)
-    with open('/etc/modules', 'r+') as modules:
-        if module not in modules.read():
-            modules.write(module)
-
-
 def copy_files(src, dst, symlinks=False, ignore=None):
     """Copy files from src to dst."""
     for item in os.listdir(src):
@@ -363,14 +366,14 @@
             service_start(svc)
 
 
-def ensure_ceph_keyring(service, user=None, group=None):
+def ensure_ceph_keyring(service, user=None, group=None, relation='ceph'):
     """Ensures a ceph keyring is created for a named service and optionally
     ensures user and group ownership.
 
     Returns False if no ceph key is available in relation state.
     """
     key = None
-    for rid in relation_ids('ceph'):
+    for rid in relation_ids(relation):
         for unit in related_units(rid):
             key = relation_get('key', rid=rid, unit=unit)
             if key:
@@ -411,17 +414,59 @@
 
     The API is versioned and defaults to version 1.
     """
-    def __init__(self, api_version=1):
+    def __init__(self, api_version=1, request_id=None):
         self.api_version = api_version
+        if request_id:
+            self.request_id = request_id
+        else:
+            self.request_id = str(uuid.uuid1())
         self.ops = []
 
-    def add_op_create_pool(self, name, replica_count=3):
+    def add_op_create_pool(self, name, replica_count=3, pg_num=None):
+        """Adds an operation to create a pool.
+
+        @param pg_num setting:  optional setting. If not provided, this value
+        will be calculated by the broker based on how many OSDs are in the
+        cluster at the time of creation. Note that, if provided, this value
+        will be capped at the current available maximum.
+        """
         self.ops.append({'op': 'create-pool', 'name': name,
-                         'replicas': replica_count})
+                         'replicas': replica_count, 'pg_num': pg_num})
+
+    def set_ops(self, ops):
+        """Set request ops to provided value.
+
+        Useful for injecting ops that come from a previous request
+        to allow comparisons to ensure validity.
+        """
+        self.ops = ops
 
     @property
     def request(self):
-        return json.dumps({'api-version': self.api_version, 'ops': self.ops})
+        return json.dumps({'api-version': self.api_version, 'ops': self.ops,
+                           'request-id': self.request_id})
+
+    def _ops_equal(self, other):
+        if len(self.ops) == len(other.ops):
+            for req_no in range(0, len(self.ops)):
+                for key in ['replicas', 'name', 'op', 'pg_num']:
+                    if self.ops[req_no].get(key) != other.ops[req_no].get(key):
+                        return False
+        else:
+            return False
+        return True
+
+    def __eq__(self, other):
+        if not isinstance(other, self.__class__):
+            return False
+        if self.api_version == other.api_version and \
+                self._ops_equal(other):
+            return True
+        else:
+            return False
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
 
 
 class CephBrokerRsp(object):
@@ -431,14 +476,198 @@
 
     The API is versioned and defaults to version 1.
     """
+
     def __init__(self, encoded_rsp):
         self.api_version = None
         self.rsp = json.loads(encoded_rsp)
 
     @property
+    def request_id(self):
+        return self.rsp.get('request-id')
+
+    @property
     def exit_code(self):
         return self.rsp.get('exit-code')
 
     @property
     def exit_msg(self):
         return self.rsp.get('stderr')
+
+
+# Ceph Broker Conversation:
+# If a charm needs an action to be taken by ceph it can create a CephBrokerRq
+# and send that request to ceph via the ceph relation. The CephBrokerRq has a
+# unique id so that the client can identity which CephBrokerRsp is associated
+# with the request. Ceph will also respond to each client unit individually
+# creating a response key per client unit eg glance/0 will get a CephBrokerRsp
+# via key broker-rsp-glance-0
+#
+# To use this the charm can just do something like:
+#
+# from charmhelpers.contrib.storage.linux.ceph import (
+#     send_request_if_needed,
+#     is_request_complete,
+#     CephBrokerRq,
+# )
+#
+# @hooks.hook('ceph-relation-changed')
+# def ceph_changed():
+#     rq = CephBrokerRq()
+#     rq.add_op_create_pool(name='poolname', replica_count=3)
+#
+#     if is_request_complete(rq):
+#         <Request complete actions>
+#     else:
+#         send_request_if_needed(get_ceph_request())
+#
+# CephBrokerRq and CephBrokerRsp are serialized into JSON. Below is an example
+# of glance having sent a request to ceph which ceph has successfully processed
+#  'ceph:8': {
+#      'ceph/0': {
+#          'auth': 'cephx',
+#          'broker-rsp-glance-0': '{"request-id": "0bc7dc54", "exit-code": 0}',
+#          'broker_rsp': '{"request-id": "0da543b8", "exit-code": 0}',
+#          'ceph-public-address': '10.5.44.103',
+#          'key': 'AQCLDttVuHXINhAAvI144CB09dYchhHyTUY9BQ==',
+#          'private-address': '10.5.44.103',
+#      },
+#      'glance/0': {
+#          'broker_req': ('{"api-version": 1, "request-id": "0bc7dc54", '
+#                         '"ops": [{"replicas": 3, "name": "glance", '
+#                         '"op": "create-pool"}]}'),
+#          'private-address': '10.5.44.109',
+#      },
+#  }
+
+def get_previous_request(rid):
+    """Return the last ceph broker request sent on a given relation
+
+    @param rid: Relation id to query for request
+    """
+    request = None
+    broker_req = relation_get(attribute='broker_req', rid=rid,
+                              unit=local_unit())
+    if broker_req:
+        request_data = json.loads(broker_req)
+        request = CephBrokerRq(api_version=request_data['api-version'],
+                               request_id=request_data['request-id'])
+        request.set_ops(request_data['ops'])
+
+    return request
+
+
+def get_request_states(request, relation='ceph'):
+    """Return a dict of requests per relation id with their corresponding
+       completion state.
+
+    This allows a charm, which has a request for ceph, to see whether there is
+    an equivalent request already being processed and if so what state that
+    request is in.
+
+    @param request: A CephBrokerRq object
+    """
+    complete = []
+    requests = {}
+    for rid in relation_ids(relation):
+        complete = False
+        previous_request = get_previous_request(rid)
+        if request == previous_request:
+            sent = True
+            complete = is_request_complete_for_rid(previous_request, rid)
+        else:
+            sent = False
+            complete = False
+
+        requests[rid] = {
+            'sent': sent,
+            'complete': complete,
+        }
+
+    return requests
+
+
+def is_request_sent(request, relation='ceph'):
+    """Check to see if a functionally equivalent request has already been sent
+
+    Returns True if a similair request has been sent
+
+    @param request: A CephBrokerRq object
+    """
+    states = get_request_states(request, relation=relation)
+    for rid in states.keys():
+        if not states[rid]['sent']:
+            return False
+
+    return True
+
+
+def is_request_complete(request, relation='ceph'):
+    """Check to see if a functionally equivalent request has already been
+    completed
+
+    Returns True if a similair request has been completed
+
+    @param request: A CephBrokerRq object
+    """
+    states = get_request_states(request, relation=relation)
+    for rid in states.keys():
+        if not states[rid]['complete']:
+            return False
+
+    return True
+
+
+def is_request_complete_for_rid(request, rid):
+    """Check if a given request has been completed on the given relation
+
+    @param request: A CephBrokerRq object
+    @param rid: Relation ID
+    """
+    broker_key = get_broker_rsp_key()
+    for unit in related_units(rid):
+        rdata = relation_get(rid=rid, unit=unit)
+        if rdata.get(broker_key):
+            rsp = CephBrokerRsp(rdata.get(broker_key))
+            if rsp.request_id == request.request_id:
+                if not rsp.exit_code:
+                    return True
+        else:
+            # The remote unit sent no reply targeted at this unit so either the
+            # remote ceph cluster does not support unit targeted replies or it
+            # has not processed our request yet.
+            if rdata.get('broker_rsp'):
+                request_data = json.loads(rdata['broker_rsp'])
+                if request_data.get('request-id'):
+                    log('Ignoring legacy broker_rsp without unit key as remote '
+                        'service supports unit specific replies', level=DEBUG)
+                else:
+                    log('Using legacy broker_rsp as remote service does not '
+                        'supports unit specific replies', level=DEBUG)
+                    rsp = CephBrokerRsp(rdata['broker_rsp'])
+                    if not rsp.exit_code:
+                        return True
+
+    return False
+
+
+def get_broker_rsp_key():
+    """Return broker response key for this unit
+
+    This is the key that ceph is going to use to pass request status
+    information back to this unit
+    """
+    return 'broker-rsp-' + local_unit().replace('/', '-')
+
+
+def send_request_if_needed(request, relation='ceph'):
+    """Send broker request if an equivalent request has not already been sent
+
+    @param request: A CephBrokerRq object
+    """
+    if is_request_sent(request, relation=relation):
+        log('Request already sent but not complete, not sending new request',
+            level=DEBUG)
+    else:
+        for rid in relation_ids(relation):
+            log('Sending request {}'.format(request.request_id), level=DEBUG)
+            relation_set(relation_id=rid, broker_req=request.request)
 
=== modified file 'hooks/charmhelpers/contrib/storage/linux/loopback.py'
--- hooks/charmhelpers/contrib/storage/linux/loopback.py	2015-06-10 15:45:48 +0000
+++ hooks/charmhelpers/contrib/storage/linux/loopback.py	2015-12-01 15:05:49 +0000
@@ -76,3 +76,13 @@
         check_call(cmd)
 
     return create_loopback(path)
+
+
+def is_mapped_loopback_device(device):
+    """
+    Checks if a given device name is an existing/mapped loopback device.
+    :param device: str: Full path to the device (eg, /dev/loop1).
+    :returns: str: Path to the backing file if is a loopback device
+    empty string otherwise
+    """
+    return loopback_devices().get(device, "")
 
=== modified file 'hooks/charmhelpers/contrib/storage/linux/utils.py'
--- hooks/charmhelpers/contrib/storage/linux/utils.py	2015-06-10 15:45:48 +0000
+++ hooks/charmhelpers/contrib/storage/linux/utils.py	2015-12-01 15:05:49 +0000
@@ -43,9 +43,10 @@
 
     :param block_device: str: Full path of block device to clean.
     '''
+    # https://github.com/ceph/ceph/commit/fdd7f8d83afa25c4e09aaedd90ab93f3b64a677b
     # sometimes sgdisk exits non-zero; this is OK, dd will clean up
-    call(['sgdisk', '--zap-all', '--mbrtogpt',
-          '--clear', block_device])
+    call(['sgdisk', '--zap-all', '--', block_device])
+    call(['sgdisk', '--clear', '--mbrtogpt', '--', block_device])
     dev_end = check_output(['blockdev', '--getsz',
                             block_device]).decode('UTF-8')
     gpt_end = int(dev_end.split()[0]) - 100
@@ -67,4 +68,4 @@
     out = check_output(['mount']).decode('UTF-8')
     if is_partition:
         return bool(re.search(device + r"\b", out))
-    return bool(re.search(device + r"[0-9]+\b", out))
+    return bool(re.search(device + r"[0-9]*\b", out))
 
=== added file 'hooks/charmhelpers/core/files.py'
--- hooks/charmhelpers/core/files.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/files.py	2015-12-01 15:05:49 +0000
@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = 'Jorge Niedbalski <[e-mail]>'
+
+import os
+import subprocess
+
+
+def sed(filename, before, after, flags='g'):
+    """
+    Search and replaces the given pattern on filename.
+
+    :param filename: relative or absolute file path.
+    :param before: expression to be replaced (see 'man sed')
+    :param after: expression to replace with (see 'man sed')
+    :param flags: sed-compatible regex flags in example, to make
+    the  search and replace case insensitive, specify ``flags="i"``.
+    The ``g`` flag is always specified regardless, so you do not
+    need to remember to include it when overriding this parameter.
+    :returns: If the sed command exit code was zero then return,
+    otherwise raise CalledProcessError.
+    """
+    expression = r's/{0}/{1}/{2}'.format(before,
+                                         after, flags)
+
+    return subprocess.check_call(["sed", "-i", "-r", "-e",
+                                  expression,
+                                  os.path.expanduser(filename)])
 
=== modified file 'hooks/charmhelpers/core/hookenv.py'
--- hooks/charmhelpers/core/hookenv.py	2015-06-10 07:35:12 +0000
+++ hooks/charmhelpers/core/hookenv.py	2015-12-01 15:05:49 +0000
@@ -21,7 +21,10 @@
 #  Charm Helpers Developers <[e-mail]>
 
 from __future__ import print_function
+import copy
+from distutils.version import LooseVersion
 from functools import wraps
+import glob
 import os
 import json
 import yaml
@@ -71,6 +74,7 @@
         res = func(*args, **kwargs)
         cache[key] = res
         return res
+    wrapper._wrapped = func
     return wrapper
 
 
@@ -170,9 +174,19 @@
     return os.environ.get('JUJU_RELATION', None)
 
 
-def relation_id():
-    """The relation ID for the current relation hook"""
-    return os.environ.get('JUJU_RELATION_ID', None)
+@cached
+def relation_id(relation_name=None, service_or_unit=None):
+    """The relation ID for the current or a specified relation"""
+    if not relation_name and not service_or_unit:
+        return os.environ.get('JUJU_RELATION_ID', None)
+    elif relation_name and service_or_unit:
+        service_name = service_or_unit.split('/')[0]
+        for relid in relation_ids(relation_name):
+            remote_service = remote_service_name(relid)
+            if remote_service == service_name:
+                return relid
+    else:
+        raise ValueError('Must specify neither or both of relation_name and service_or_unit')
 
 
 def local_unit():
@@ -190,9 +204,20 @@
     return local_unit().split('/')[0]
 
 
+@cached
+def remote_service_name(relid=None):
+    """The remote service name for a given relation-id (or the current relation)"""
+    if relid is None:
+        unit = remote_unit()
+    else:
+        units = related_units(relid)
+        unit = units[0] if units else None
+    return unit.split('/')[0] if unit else None
+
+
 def hook_name():
     """The name of the currently executing hook"""
-    return os.path.basename(sys.argv[0])
+    return os.environ.get('JUJU_HOOK_NAME', os.path.basename(sys.argv[0]))
 
 
 class Config(dict):
@@ -242,29 +267,7 @@
         self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
         if os.path.exists(self.path):
             self.load_previous()
-
-    def __getitem__(self, key):
-        """For regular dict lookups, check the current juju config first,
-        then the previous (saved) copy. This ensures that user-saved values
-        will be returned by a dict lookup.
-
-        """
-        try:
-            return dict.__getitem__(self, key)
-        except KeyError:
-            return (self._prev_dict or {})[key]
-
-    def get(self, key, default=None):
-        try:
-            return self[key]
-        except KeyError:
-            return default
-
-    def keys(self):
-        prev_keys = []
-        if self._prev_dict is not None:
-            prev_keys = self._prev_dict.keys()
-        return list(set(prev_keys + list(dict.keys(self))))
+        atexit(self._implicit_save)
 
     def load_previous(self, path=None):
         """Load previous copy of config from disk.
@@ -283,6 +286,9 @@
         self.path = path or self.path
         with open(self.path) as f:
             self._prev_dict = json.load(f)
+        for k, v in copy.deepcopy(self._prev_dict).items():
+            if k not in self:
+                self[k] = v
 
     def changed(self, key):
         """Return True if the current value for this key is different from
@@ -314,13 +320,13 @@
         instance.
 
         """
-        if self._prev_dict:
-            for k, v in six.iteritems(self._prev_dict):
-                if k not in self:
-                    self[k] = v
         with open(self.path, 'w') as f:
             json.dump(self, f)
 
+    def _implicit_save(self):
+        if self.implicit_save:
+            self.save()
+
 
 @cached
 def config(scope=None):
@@ -485,6 +491,76 @@
 
 
 @cached
+def peer_relation_id():
+    '''Get a peer relation id if a peer relation has been joined, else None.'''
+    md = metadata()
+    section = md.get('peers')
+    if section:
+        for key in section:
+            relids = relation_ids(key)
+            if relids:
+                return relids[0]
+    return None
+
+
+@cached
+def relation_to_interface(relation_name):
+    """
+    Given the name of a relation, return the interface that relation uses.
+
+    :returns: The interface name, or ``None``.
+    """
+    return relation_to_role_and_interface(relation_name)[1]
+
+
+@cached
+def relation_to_role_and_interface(relation_name):
+    """
+    Given the name of a relation, return the role and the name of the interface
+    that relation uses (where role is one of ``provides``, ``requires``, or ``peer``).
+
+    :returns: A tuple containing ``(role, interface)``, or ``(None, None)``.
+    """
+    _metadata = metadata()
+    for role in ('provides', 'requires', 'peer'):
+        interface = _metadata.get(role, {}).get(relation_name, {}).get('interface')
+        if interface:
+            return role, interface
+    return None, None
+
+
+@cached
+def role_and_interface_to_relations(role, interface_name):
+    """
+    Given a role and interface name, return a list of relation names for the
+    current charm that use that interface under that role (where role is one
+    of ``provides``, ``requires``, or ``peer``).
+
+    :returns: A list of relation names.
+    """
+    _metadata = metadata()
+    results = []
+    for relation_name, relation in _metadata.get(role, {}).items():
+        if relation['interface'] == interface_name:
+            results.append(relation_name)
+    return results
+
+
+@cached
+def interface_to_relations(interface_name):
+    """
+    Given an interface, return a list of relation names for the current
+    charm that use that interface.
+
+    :returns: A list of relation names.
+    """
+    results = []
+    for role in ('provides', 'requires', 'peer'):
+        results.extend(role_and_interface_to_relations(role, interface_name))
+    return results
+
+
+@cached
 def charm_name():
     """Get the name of the current charm as is specified on metadata.yaml"""
     return metadata().get('name')
@@ -560,6 +636,38 @@
     return unit_get('private-address')
 
 
+@cached
+def storage_get(attribute="", storage_id=""):
+    """Get storage attributes"""
+    _args = ['storage-get', '--format=json']
+    if storage_id:
+        _args.extend(('-s', storage_id))
+    if attribute:
+        _args.append(attribute)
+    try:
+        return json.loads(subprocess.check_output(_args).decode('UTF-8'))
+    except ValueError:
+        return None
+
+
+@cached
+def storage_list(storage_name=""):
+    """List the storage IDs for the unit"""
+    _args = ['storage-list', '--format=json']
+    if storage_name:
+        _args.append(storage_name)
+    try:
+        return json.loads(subprocess.check_output(_args).decode('UTF-8'))
+    except ValueError:
+        return None
+    except OSError as e:
+        import errno
+        if e.errno == errno.ENOENT:
+            # storage-list does not exist
+            return []
+        raise
+
+
 class UnregisteredHookError(Exception):
     """Raised when an undefined hook is called"""
     pass
@@ -587,10 +695,14 @@
             hooks.execute(sys.argv)
     """
 
-    def __init__(self, config_save=True):
+    def __init__(self, config_save=None):
         super(Hooks, self).__init__()
         self._hooks = {}
-        self._config_save = config_save
+
+        # For unknown reasons, we allow the Hooks constructor to override
+        # config().implicit_save.
+        if config_save is not None:
+            config().implicit_save = config_save
 
     def register(self, name, function):
         """Register a hook"""
@@ -598,13 +710,16 @@
 
     def execute(self, args):
         """Execute a registered hook based on args[0]"""
+        _run_atstart()
         hook_name = os.path.basename(args[0])
         if hook_name in self._hooks:
-            self._hooks[hook_name]()
-            if self._config_save:
-                cfg = config()
-                if cfg.implicit_save:
-                    cfg.save()
+            try:
+                self._hooks[hook_name]()
+            except SystemExit as x:
+                if x.code is None or x.code == 0:
+                    _run_atexit()
+                raise
+            _run_atexit()
         else:
             raise UnregisteredHookError(hook_name)
 
@@ -653,6 +768,21 @@
     subprocess.check_call(['action-fail', message])
 
 
+def action_name():
+    """Get the name of the currently executing action."""
+    return os.environ.get('JUJU_ACTION_NAME')
+
+
+def action_uuid():
+    """Get the UUID of the currently executing action."""
+    return os.environ.get('JUJU_ACTION_UUID')
+
+
+def action_tag():
+    """Get the tag for the currently executing action."""
+    return os.environ.get('JUJU_ACTION_TAG')
+
+
 def status_set(workload_state, message):
     """Set the workload state with a message
 
@@ -682,25 +812,28 @@
 
 
 def status_get():
-    """Retrieve the previously set juju workload state
-
-    If the status-set command is not found then assume this is juju < 1.23 and
-    return 'unknown'
+    """Retrieve the previously set juju workload state and message
+
+    If the status-get command is not found then assume this is juju < 1.23 and
+    return 'unknown', ""
+
     """
-    cmd = ['status-get']
+    cmd = ['status-get', "--format=json", "--include-data"]
     try:
-        raw_status = subprocess.check_output(cmd, universal_newlines=True)
-        status = raw_status.rstrip()
-        return status
+        raw_status = subprocess.check_output(cmd)
     except OSError as e:
         if e.errno == errno.ENOENT:
-            return 'unknown'
+            return ('unknown', "")
         else:
             raise
+    else:
+        status = json.loads(raw_status.decode("UTF-8"))
+        return (status["status"], status["message"])
 
 
 def translate_exc(from_exc, to_exc):
     def inner_translate_exc1(f):
+        @wraps(f)
         def inner_translate_exc2(*args, **kwargs):
             try:
                 return f(*args, **kwargs)
@@ -732,13 +865,80 @@
 @translate_exc(from_exc=OSError, to_exc=NotImplementedError)
 def leader_set(settings=None, **kwargs):
     """Juju leader set value(s)"""
-    log("Juju leader-set '%s'" % (settings), level=DEBUG)
+    # Don't log secrets.
+    # log("Juju leader-set '%s'" % (settings), level=DEBUG)
     cmd = ['leader-set']
     settings = settings or {}
     settings.update(kwargs)
-    for k, v in settings.iteritems():
+    for k, v in settings.items():
         if v is None:
             cmd.append('{}='.format(k))
         else:
             cmd.append('{}={}'.format(k, v))
     subprocess.check_call(cmd)
+
+
+@cached
+def juju_version():
+    """Full version string (eg. '1.23.3.1-trusty-amd64')"""
+    # Per https://bugs.launchpad.net/juju-core/+bug/1455368/comments/1
+    jujud = glob.glob('/var/lib/juju/tools/machine-*/jujud')[0]
+    return subprocess.check_output([jujud, 'version'],
+                                   universal_newlines=True).strip()
+
+
+@cached
+def has_juju_version(minimum_version):
+    """Return True if the Juju version is at least the provided version"""
+    return LooseVersion(juju_version()) >= LooseVersion(minimum_version)
+
+
+_atexit = []
+_atstart = []
+
+
+def atstart(callback, *args, **kwargs):
+    '''Schedule a callback to run before the main hook.
+
+    Callbacks are run in the order they were added.
+
+    This is useful for modules and classes to perform initialization
+    and inject behavior. In particular:
+
+        - Run common code before all of your hooks, such as logging
+          the hook name or interesting relation data.
+        - Defer object or module initialization that requires a hook
+          context until we know there actually is a hook context,
+          making testing easier.
+        - Rather than requiring charm authors to include boilerplate to
+          invoke your helper's behavior, have it run automatically if
+          your object is instantiated or module imported.
+
+    This is not at all useful after your hook framework as been launched.
+    '''
+    global _atstart
+    _atstart.append((callback, args, kwargs))
+
+
+def atexit(callback, *args, **kwargs):
+    '''Schedule a callback to run on successful hook completion.
+
+    Callbacks are run in the reverse order that they were added.'''
+    _atexit.append((callback, args, kwargs))
+
+
+def _run_atstart():
+    '''Hook frameworks must invoke this before running the main hook body.'''
+    global _atstart
+    for callback, args, kwargs in _atstart:
+        callback(*args, **kwargs)
+    del _atstart[:]
+
+
+def _run_atexit():
+    '''Hook frameworks must invoke this after the main hook body has
+    successfully completed. Do not invoke it if the hook fails.'''
+    global _atexit
+    for callback, args, kwargs in reversed(_atexit):
+        callback(*args, **kwargs)
+    del _atexit[:]
 
=== modified file 'hooks/charmhelpers/core/host.py'
--- hooks/charmhelpers/core/host.py	2015-07-01 13:35:47 +0000
+++ hooks/charmhelpers/core/host.py	2015-12-01 15:05:49 +0000
@@ -63,6 +63,56 @@
     return service_result
 
 
+def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d"):
+    """Pause a system service.
+
+    Stop it, and prevent it from starting again at boot."""
+    stopped = True
+    if service_running(service_name):
+        stopped = service_stop(service_name)
+    upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
+    sysv_file = os.path.join(initd_dir, service_name)
+    if os.path.exists(upstart_file):
+        override_path = os.path.join(
+            init_dir, '{}.override'.format(service_name))
+        with open(override_path, 'w') as fh:
+            fh.write("manual\n")
+    elif os.path.exists(sysv_file):
+        subprocess.check_call(["update-rc.d", service_name, "disable"])
+    else:
+        # XXX: Support SystemD too
+        raise ValueError(
+            "Unable to detect {0} as either Upstart {1} or SysV {2}".format(
+                service_name, upstart_file, sysv_file))
+    return stopped
+
+
+def service_resume(service_name, init_dir="/etc/init",
+                   initd_dir="/etc/init.d"):
+    """Resume a system service.
+
+    Reenable starting again at boot. Start the service"""
+    upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
+    sysv_file = os.path.join(initd_dir, service_name)
+    if os.path.exists(upstart_file):
+        override_path = os.path.join(
+            init_dir, '{}.override'.format(service_name))
+        if os.path.exists(override_path):
+            os.unlink(override_path)
+    elif os.path.exists(sysv_file):
+        subprocess.check_call(["update-rc.d", service_name, "enable"])
+    else:
+        # XXX: Support SystemD too
+        raise ValueError(
+            "Unable to detect {0} as either Upstart {1} or SysV {2}".format(
+                service_name, upstart_file, sysv_file))
+
+    started = service_running(service_name)
+    if not started:
+        started = service_start(service_name)
+    return started
+
+
 def service(action, service_name):
     """Control a system service"""
     cmd = ['service', service_name, action]
@@ -119,8 +169,9 @@
 
 
 def user_exists(username):
+    """Check if a user exists"""
     try:
-        user_info = pwd.getpwnam(username)
+        pwd.getpwnam(username)
         user_exists = True
     except KeyError:
         user_exists = False
@@ -149,11 +200,7 @@
 
 def add_user_to_group(username, group):
     """Add a user to a group"""
-    cmd = [
-        'gpasswd', '-a',
-        username,
-        group
-    ]
+    cmd = ['gpasswd', '-a', username, group]
     log("Adding user {} to group {}".format(username, group))
     subprocess.check_call(cmd)
 
@@ -263,8 +310,8 @@
     return system_mounts
 
 
-
 def fstab_mount(mountpoint):
+    """Mount filesystem using fstab"""
     cmd_args = ['mount', mountpoint]
     try:
         subprocess.check_output(cmd_args)
@@ -390,25 +437,80 @@
     return(''.join(random_chars))
 
 
-def list_nics(nic_type):
+def is_phy_iface(interface):
+    """Returns True if interface is not virtual, otherwise False."""
+    if interface:
+        sys_net = '/sys/class/net'
+        if os.path.isdir(sys_net):
+            for iface in glob.glob(os.path.join(sys_net, '*')):
+                if '/virtual/' in os.path.realpath(iface):
+                    continue
+
+                if interface == os.path.basename(iface):
+                    return True
+
+    return False
+
+
+def get_bond_master(interface):
+    """Returns bond master if interface is bond slave otherwise None.
+
+    NOTE: the provided interface is expected to be physical
+    """
+    if interface:
+        iface_path = '/sys/class/net/%s' % (interface)
+        if os.path.exists(iface_path):
+            if '/virtual/' in os.path.realpath(iface_path):
+                return None
+
+            master = os.path.join(iface_path, 'master')
+            if os.path.exists(master):
+                master = os.path.realpath(master)
+                # make sure it is a bond master
+                if os.path.exists(os.path.join(master, 'bonding')):
+                    return os.path.basename(master)
+
+    return None
+
+
+def list_nics(nic_type=None):
     '''Return a list of nics of given type(s)'''
     if isinstance(nic_type, six.string_types):
         int_types = [nic_type]
     else:
         int_types = nic_type
+
     interfaces = []
-    for int_type in int_types:
-        cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
+    if nic_type:
+        for int_type in int_types:
+            cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
+            ip_output = subprocess.check_output(cmd).decode('UTF-8')
+            ip_output = ip_output.split('\n')
+            ip_output = (line for line in ip_output if line)
+            for line in ip_output:
+                if line.split()[1].startswith(int_type):
+                    matched = re.search('.*: (' + int_type +
+                                        r'[0-9]+\.[0-9]+)@.*', line)
+                    if matched:
+                        iface = matched.groups()[0]
+                    else:
+                        iface = line.split()[1].replace(":", "")
+
+                    if iface not in interfaces:
+                        interfaces.append(iface)
+    else:
+        cmd = ['ip', 'a']
         ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
-        ip_output = (line for line in ip_output if line)
+        ip_output = (line.strip() for line in ip_output if line)
+
+        key = re.compile('^[0-9]+:\s+(.+):')
         for line in ip_output:
-            if line.split()[1].startswith(int_type):
-                matched = re.search('.*: (' + int_type + r'[0-9]+\.[0-9]+)@.*', line)
-                if matched:
-                    interface = matched.groups()[0]
-                else:
-                    interface = line.split()[1].replace(":", "")
-                interfaces.append(interface)
+            matched = re.search(key, line)
+            if matched:
+                iface = matched.group(1)
+                iface = iface.partition("@")[0]
+                if iface not in interfaces:
+                    interfaces.append(iface)
 
     return interfaces
 
@@ -440,23 +542,6 @@
     return hwaddr
 
 
-def get_mac_nic_map():
-    '''Return a dict of macs and their corresponding nics'''
-    cmd = ['ip', '-o', '-0', 'addr', 'list']
-    ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
-    mac_nic_map = {}
-    for line in ip_output:
-        columns = line.split()
-        if 'link/ether' in columns:
-             hwaddr = columns[columns.index('link/ether') + 1]
-             nic = columns[1].replace(':', '')
-             if mac_nic_map.get(hwaddr):
-                 mac_nic_map[hwaddr].append(nic)
-             else:
-                 mac_nic_map[hwaddr] = [nic]
-    return mac_nic_map
-
-
 def cmp_pkgrevno(package, revno, pkgcache=None):
     '''Compare supplied revno with the revno of the installed package
 
@@ -485,7 +570,14 @@
         os.chdir(cur)
 
 
-def chownr(path, owner, group, follow_links=True):
+def chownr(path, owner, group, follow_links=True, chowntopdir=False):
+    """
+    Recursively change user and group ownership of files and directories
+    in given path. Doesn't chown path itself by default, only its children.
+
+    :param bool follow_links: Also Chown links if True
+    :param bool chowntopdir: Also chown path itself if True
+    """
     uid = pwd.getpwnam(owner).pw_uid
     gid = grp.getgrnam(group).gr_gid
     if follow_links:
@@ -493,6 +585,10 @@
     else:
         chown = os.lchown
 
+    if chowntopdir:
+        broken_symlink = os.path.lexists(path) and not os.path.exists(path)
+        if not broken_symlink:
+            chown(path, uid, gid)
     for root, dirs, files in os.walk(path):
         for name in dirs + files:
             full = os.path.join(root, name)
@@ -503,3 +599,19 @@
 
 def lchownr(path, owner, group):
     chownr(path, owner, group, follow_links=False)
+
+
+def get_total_ram():
+    '''The total amount of system RAM in bytes.
+
+    This is what is reported by the OS, and may be overcommitted when
+    there are multiple containers hosted on the same machine.
+    '''
+    with open('/proc/meminfo', 'r') as f:
+        for line in f.readlines():
+            if line:
+                key, value, unit = line.split()
+                if key == 'MemTotal:':
+                    assert unit == 'kB', 'Unknown unit'
+                    return int(value) * 1024  # Classic, not KiB.
+        raise NotImplementedError()
 
=== modified file 'hooks/charmhelpers/core/hugepage.py'
--- hooks/charmhelpers/core/hugepage.py	2015-06-22 09:26:28 +0000
+++ hooks/charmhelpers/core/hugepage.py	2015-12-01 15:05:49 +0000
@@ -1,5 +1,3 @@
-
-#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 # Copyright 2014-2015 Canonical Limited.
@@ -19,36 +17,55 @@
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import yaml
-from charmhelpers.core.fstab import Fstab
-from charmhelpers.core.sysctl import (
-    create,
-)
+from charmhelpers.core import fstab
+from charmhelpers.core import sysctl
 from charmhelpers.core.host import (
     add_group,
     add_user_to_group,
     fstab_mount,
     mkdir,
 )
+from charmhelpers.core.strutils import bytes_from_string
+from subprocess import check_output
+
 
 def hugepage_support(user, group='hugetlb', nr_hugepages=256,
-                     max_map_count=65536, mnt_point='/hugepages',
-                     pagesize='2MB', mount=True):
+                     max_map_count=65536, mnt_point='/run/hugepages/kvm',
+                     pagesize='2MB', mount=True, set_shmmax=False):
+    """Enable hugepages on system.
+
+    Args:
+    user (str)  -- Username to allow access to hugepages to
+    group (str) -- Group name to own hugepages
+    nr_hugepages (int) -- Number of pages to reserve
+    max_map_count (int) -- Number of Virtual Memory Areas a process can own
+    mnt_point (str) -- Directory to mount hugepages on
+    pagesize (str) -- Size of hugepages
+    mount (bool) -- Whether to Mount hugepages
+    """
     group_info = add_group(group)
     gid = group_info.gr_gid
     add_user_to_group(user, group)
+    if max_map_count < 2 * nr_hugepages:
+        max_map_count = 2 * nr_hugepages
     sysctl_settings = {
         'vm.nr_hugepages': nr_hugepages,
-        'vm.max_map_count': max_map_count,  # 1GB
+        'vm.max_map_count': max_map_count,
         'vm.hugetlb_shm_group': gid,
     }
-    create(yaml.dump(sysctl_settings), '/etc/sysctl.d/10-hugepage.conf')
+    if set_shmmax:
+        shmmax_current = int(check_output(['sysctl', '-n', 'kernel.shmmax']))
+        shmmax_minsize = bytes_from_string(pagesize) * nr_hugepages
+        if shmmax_minsize > shmmax_current:
+            sysctl_settings['kernel.shmmax'] = shmmax_minsize
+    sysctl.create(yaml.dump(sysctl_settings), '/etc/sysctl.d/10-hugepage.conf')
     mkdir(mnt_point, owner='root', group='root', perms=0o755, force=False)
-    fstab = Fstab()
-    fstab_entry = fstab.get_entry_by_attr('mountpoint', mnt_point)
+    lfstab = fstab.Fstab()
+    fstab_entry = lfstab.get_entry_by_attr('mountpoint', mnt_point)
     if fstab_entry:
-        fstab.remove_entry(fstab_entry)
-    entry = fstab.Entry('nodev', mnt_point, 'hugetlbfs',
-                        'mode=1770,gid={},pagesize={}'.format(gid, pagesize), 0, 0)
-    fstab.add_entry(entry)
+        lfstab.remove_entry(fstab_entry)
+    entry = lfstab.Entry('nodev', mnt_point, 'hugetlbfs',
+                         'mode=1770,gid={},pagesize={}'.format(gid, pagesize), 0, 0)
+    lfstab.add_entry(entry)
     if mount:
         fstab_mount(mnt_point)
 
=== added file 'hooks/charmhelpers/core/kernel.py'
--- hooks/charmhelpers/core/kernel.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/kernel.py	2015-12-01 15:05:49 +0000
@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = "Jorge Niedbalski <[e-mail]>"
+
+from charmhelpers.core.hookenv import (
+    log,
+    INFO
+)
+
+from subprocess import check_call, check_output
+import re
+
+
+def modprobe(module, persist=True):
+    """Load a kernel module and configure for auto-load on reboot."""
+    cmd = ['modprobe', module]
+
+    log('Loading kernel module %s' % module, level=INFO)
+
+    check_call(cmd)
+    if persist:
+        with open('/etc/modules', 'r+') as modules:
+            if module not in modules.read():
+                modules.write(module)
+
+
+def rmmod(module, force=False):
+    """Remove a module from the linux kernel"""
+    cmd = ['rmmod']
+    if force:
+        cmd.append('-f')
+    cmd.append(module)
+    log('Removing kernel module %s' % module, level=INFO)
+    return check_call(cmd)
+
+
+def lsmod():
+    """Shows what kernel modules are currently loaded"""
+    return check_output(['lsmod'],
+                        universal_newlines=True)
+
+
+def is_module_loaded(module):
+    """Checks if a kernel module is already loaded"""
+    matches = re.findall('^%s[ ]+' % module, lsmod(), re.M)
+    return len(matches) > 0
+
+
+def update_initramfs(version='all'):
+    """Updates an initramfs image"""
+    return check_call(["update-initramfs", "-k", version, "-u"])
 
=== modified file 'hooks/charmhelpers/core/services/base.py'
--- hooks/charmhelpers/core/services/base.py	2015-06-10 07:35:12 +0000
+++ hooks/charmhelpers/core/services/base.py	2015-12-01 15:05:49 +0000
@@ -128,15 +128,18 @@
         """
         Handle the current hook by doing The Right Thing with the registered services.
         """
-        hook_name = hookenv.hook_name()
-        if hook_name == 'stop':
-            self.stop_services()
-        else:
-            self.reconfigure_services()
-            self.provide_data()
-        cfg = hookenv.config()
-        if cfg.implicit_save:
-            cfg.save()
+        hookenv._run_atstart()
+        try:
+            hook_name = hookenv.hook_name()
+            if hook_name == 'stop':
+                self.stop_services()
+            else:
+                self.reconfigure_services()
+                self.provide_data()
+        except SystemExit as x:
+            if x.code is None or x.code == 0:
+                hookenv._run_atexit()
+        hookenv._run_atexit()
 
     def provide_data(self):
         """
 
=== modified file 'hooks/charmhelpers/core/services/helpers.py'
--- hooks/charmhelpers/core/services/helpers.py	2015-06-12 12:22:51 +0000
+++ hooks/charmhelpers/core/services/helpers.py	2015-12-01 15:05:49 +0000
@@ -16,6 +16,7 @@
 
 import os
 import yaml
+
 from charmhelpers.core import hookenv
 from charmhelpers.core import host
 from charmhelpers.core import templating
@@ -240,42 +241,43 @@
     action.
 
     :param str source: The template source file, relative to
-    `$CHARM_DIR/templates`
+        `$CHARM_DIR/templates`
 
     :param str target: The target to write the rendered template to
     :param str owner: The owner of the rendered file
     :param str group: The group of the rendered file
     :param int perms: The permissions of the rendered file
-    :param list template_searchpath: List of paths to search for template in
     :param partial on_change_action: functools partial to be executed when
                                      rendered file changes
+    :param jinja2 loader template_loader: A jinja2 template loader
     """
     def __init__(self, source, target,
                  owner='root', group='root', perms=0o444,
-                 template_searchpath=None, on_change_action=None):
+                 on_change_action=None, template_loader=None):
         self.source = source
         self.target = target
         self.owner = owner
         self.group = group
         self.perms = perms
-        self.template_searchpath = template_searchpath
         self.on_change_action = on_change_action
+        self.template_loader = template_loader
 
     def __call__(self, manager, service_name, event_name):
         pre_checksum = ''
         if self.on_change_action and os.path.isfile(self.target):
             pre_checksum = host.file_hash(self.target)
-            print pre_checksum
         service = manager.get_service(service_name)
         context = {}
         for ctx in service.get('required_data', []):
             context.update(ctx)
         templating.render(self.source, self.target, context,
                           self.owner, self.group, self.perms,
-                          self.template_searchpath)
+                          template_loader=self.template_loader)
         if self.on_change_action:
             if pre_checksum == host.file_hash(self.target):
-                print "No change detected " + self.target
+                hookenv.log(
+                    'No change detected: {}'.format(self.target),
+                    hookenv.DEBUG)
             else:
                 self.on_change_action()
 
 
=== modified file 'hooks/charmhelpers/core/strutils.py'
--- hooks/charmhelpers/core/strutils.py	2015-06-10 07:35:12 +0000
+++ hooks/charmhelpers/core/strutils.py	2015-12-01 15:05:49 +0000
@@ -18,6 +18,7 @@
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import six
+import re
 
 
 def bool_from_string(value):
@@ -40,3 +41,32 @@
 
     msg = "Unable to interpret string value '%s' as boolean" % (value)
     raise ValueError(msg)
+
+
+def bytes_from_string(value):
+    """Interpret human readable string value as bytes.
+
+    Returns int
+    """
+    BYTE_POWER = {
+        'K': 1,
+        'KB': 1,
+        'M': 2,
+        'MB': 2,
+        'G': 3,
+        'GB': 3,
+        'T': 4,
+        'TB': 4,
+        'P': 5,
+        'PB': 5,
+    }
+    if isinstance(value, six.string_types):
+        value = six.text_type(value)
+    else:
+        msg = "Unable to interpret non-string value '%s' as boolean" % (value)
+        raise ValueError(msg)
+    matches = re.match("([0-9]+)([a-zA-Z]+)", value)
+    if not matches:
+        msg = "Unable to interpret string value '%s' as bytes" % (value)
+        raise ValueError(msg)
+    return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
 
=== modified file 'hooks/charmhelpers/core/templating.py'
--- hooks/charmhelpers/core/templating.py	2015-06-17 12:23:31 +0000
+++ hooks/charmhelpers/core/templating.py	2015-12-01 15:05:49 +0000
@@ -21,8 +21,7 @@
 
 
 def render(source, target, context, owner='root', group='root',
-           perms=0o444, templates_dir=None, encoding='UTF-8',
-           template_searchpath=None):
+           perms=0o444, templates_dir=None, encoding='UTF-8', template_loader=None):
     """
     Render a template.
 
@@ -41,7 +40,7 @@
     this will attempt to use charmhelpers.fetch.apt_install to install it.
     """
     try:
-        from jinja2 import ChoiceLoader, FileSystemLoader, Environment, exceptions
+        from jinja2 import FileSystemLoader, Environment, exceptions
     except ImportError:
         try:
             from charmhelpers.fetch import apt_install
@@ -51,25 +50,26 @@
                         level=hookenv.ERROR)
             raise
         apt_install('python-jinja2', fatal=True)
-        from jinja2 import ChoiceLoader, FileSystemLoader, Environment, exceptions
+        from jinja2 import FileSystemLoader, Environment, exceptions
 
-    if template_searchpath:
-        fs_loaders = []
-        for tmpl_dir in template_searchpath:
-            fs_loaders.append(FileSystemLoader(tmpl_dir))
-        loader = ChoiceLoader(fs_loaders) 
+    if template_loader:
+        template_env = Environment(loader=template_loader)
     else:
         if templates_dir is None:
             templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
-        loader = Environment(loader=FileSystemLoader(templates_dir))
+        template_env = Environment(loader=FileSystemLoader(templates_dir))
     try:
         source = source
-        template = loader.get_template(source)
+        template = template_env.get_template(source)
     except exceptions.TemplateNotFound as e:
         hookenv.log('Could not load template %s from %s.' %
                     (source, templates_dir),
                     level=hookenv.ERROR)
         raise e
     content = template.render(context)
-    host.mkdir(os.path.dirname(target), owner, group, perms=0o755)
+    target_dir = os.path.dirname(target)
+    if not os.path.exists(target_dir):
+        # This is a terrible default directory permission, as the file
+        # or its siblings will often contain secrets.
+        host.mkdir(os.path.dirname(target), owner, group, perms=0o755)
     host.write_file(target, content.encode(encoding), owner, group, perms)
 
=== modified file 'hooks/charmhelpers/core/unitdata.py'
--- hooks/charmhelpers/core/unitdata.py	2015-06-10 07:35:12 +0000
+++ hooks/charmhelpers/core/unitdata.py	2015-12-01 15:05:49 +0000
@@ -152,6 +152,7 @@
 import collections
 import contextlib
 import datetime
+import itertools
 import json
 import os
 import pprint
@@ -164,8 +165,7 @@
 class Storage(object):
     """Simple key value database for local unit state within charms.
 
-    Modifications are automatically committed at hook exit. That's
-    currently regardless of exit code.
+    Modifications are not persisted unless :meth:`flush` is called.
 
     To support dicts, lists, integer, floats, and booleans values
     are automatically json encoded/decoded.
@@ -173,8 +173,11 @@
     def __init__(self, path=None):
         self.db_path = path
         if path is None:
-            self.db_path = os.path.join(
-                os.environ.get('CHARM_DIR', ''), '.unit-state.db')
+            if 'UNIT_STATE_DB' in os.environ:
+                self.db_path = os.environ['UNIT_STATE_DB']
+            else:
+                self.db_path = os.path.join(
+                    os.environ.get('CHARM_DIR', ''), '.unit-state.db')
         self.conn = sqlite3.connect('%s' % self.db_path)
         self.cursor = self.conn.cursor()
         self.revision = None
@@ -189,15 +192,8 @@
         self.conn.close()
         self._closed = True
 
-    def _scoped_query(self, stmt, params=None):
-        if params is None:
-            params = []
-        return stmt, params
-
     def get(self, key, default=None, record=False):
-        self.cursor.execute(
-            *self._scoped_query(
-                'select data from kv where key=?', [key]))
+        self.cursor.execute('select data from kv where key=?', [key])
         result = self.cursor.fetchone()
         if not result:
             return default
@@ -206,33 +202,81 @@
         return json.loads(result[0])
 
     def getrange(self, key_prefix, strip=False):
-        stmt = "select key, data from kv where key like '%s%%'" % key_prefix
-        self.cursor.execute(*self._scoped_query(stmt))
+        """
+        Get a range of keys starting with a common prefix as a mapping of
+        keys to values.
+
+        :param str key_prefix: Common prefix among all keys
+        :param bool strip: Optionally strip the common prefix from the key
+            names in the returned dict
+        :return dict: A (possibly empty) dict of key-value mappings
+        """
+        self.cursor.execute("select key, data from kv where key like ?",
+                            ['%s%%' % key_prefix])
         result = self.cursor.fetchall()
 
         if not result:
-            return None
+            return {}
         if not strip:
             key_prefix = ''
         return dict([
             (k[len(key_prefix):], json.loads(v)) for k, v in result])
 
     def update(self, mapping, prefix=""):
+        """
+        Set the values of multiple keys at once.
+
+        :param dict mapping: Mapping of keys to values
+        :param str prefix: Optional prefix to apply to all keys in `mapping`
+            before setting
+        """
         for k, v in mapping.items():
             self.set("%s%s" % (prefix, k), v)
 
     def unset(self, key):
+        """
+        Remove a key from the database entirely.
+        """
         self.cursor.execute('delete from kv where key=?', [key])
         if self.revision and self.cursor.rowcount:
             self.cursor.execute(
                 'insert into kv_revisions values (?, ?, ?)',
                 [key, self.revision, json.dumps('DELETED')])
 
+    def unsetrange(self, keys=None, prefix=""):
+        """
+        Remove a range of keys starting with a common prefix, from the database
+        entirely.
+
+        :param list keys: List of keys to remove.
+        :param str prefix: Optional prefix to apply to all keys in ``keys``
+            before removing.
+        """
+        if keys is not None:
+            keys = ['%s%s' % (prefix, key) for key in keys]
+            self.cursor.execute('delete from kv where key in (%s)' % ','.join(['?'] * len(keys)), keys)
+            if self.revision and self.cursor.rowcount:
+                self.cursor.execute(
+                    'insert into kv_revisions values %s' % ','.join(['(?, ?, ?)'] * len(keys)),
+                    list(itertools.chain.from_iterable((key, self.revision, json.dumps('DELETED')) for key in keys)))
+        else:
+            self.cursor.execute('delete from kv where key like ?',
+                                ['%s%%' % prefix])
+            if self.revision and self.cursor.rowcount:
+                self.cursor.execute(
+                    'insert into kv_revisions values (?, ?, ?)',
+                    ['%s%%' % prefix, self.revision, json.dumps('DELETED')])
+
     def set(self, key, value):
+        """
+        Set a value in the database.
+
+        :param str key: Key to set the value for
+        :param value: Any JSON-serializable value to be set
+        """
         serialized = json.dumps(value)
 
-        self.cursor.execute(
-            'select data from kv where key=?', [key])
+        self.cursor.execute('select data from kv where key=?', [key])
         exists = self.cursor.fetchone()
 
         # Skip mutations to the same value
 
=== modified file 'hooks/charmhelpers/fetch/__init__.py'
--- hooks/charmhelpers/fetch/__init__.py	2015-06-10 07:35:12 +0000
+++ hooks/charmhelpers/fetch/__init__.py	2015-12-01 15:05:49 +0000
@@ -90,6 +90,14 @@
     'kilo/proposed': 'trusty-proposed/kilo',
     'trusty-kilo/proposed': 'trusty-proposed/kilo',
     'trusty-proposed/kilo': 'trusty-proposed/kilo',
+    # Liberty
+    'liberty': 'trusty-updates/liberty',
+    'trusty-liberty': 'trusty-updates/liberty',
+    'trusty-liberty/updates': 'trusty-updates/liberty',
+    'trusty-updates/liberty': 'trusty-updates/liberty',
+    'liberty/proposed': 'trusty-proposed/liberty',
+    'trusty-liberty/proposed': 'trusty-proposed/liberty',
+    'trusty-proposed/liberty': 'trusty-proposed/liberty',
 }
 
 # The order of this list is very important. Handlers should be listed in from
@@ -215,19 +223,27 @@
     _run_apt_command(cmd, fatal)
 
 
+def apt_mark(packages, mark, fatal=False):
+    """Flag one or more packages using apt-mark"""
+    log("Marking {} as {}".format(packages, mark))
+    cmd = ['apt-mark', mark]
+    if isinstance(packages, six.string_types):
+        cmd.append(packages)
+    else:
+        cmd.extend(packages)
+
+    if fatal:
+        subprocess.check_call(cmd, universal_newlines=True)
+    else:
+        subprocess.call(cmd, universal_newlines=True)
+
+
 def apt_hold(packages, fatal=False):
-    """Hold one or more packages"""
-    cmd = ['apt-mark', 'hold']
-    if isinstance(packages, six.string_types):
-        cmd.append(packages)
-    else:
-        cmd.extend(packages)
-    log("Holding {}".format(packages))
-
-    if fatal:
-        subprocess.check_call(cmd)
-    else:
-        subprocess.call(cmd)
+    return apt_mark(packages, 'hold', fatal=fatal)
+
+
+def apt_unhold(packages, fatal=False):
+    return apt_mark(packages, 'unhold', fatal=fatal)
 
 
 def add_source(source, key=None):
@@ -370,8 +386,9 @@
     for handler in handlers:
         try:
             installed_to = handler.install(source, *args, **kwargs)
-        except UnhandledSource:
-            pass
+        except UnhandledSource as e:
+            log('Install source attempt unsuccessful: {}'.format(e),
+                level='WARNING')
     if not installed_to:
         raise UnhandledSource("No handler found for source {}".format(source))
     return installed_to
 
=== modified file 'hooks/charmhelpers/fetch/archiveurl.py'
--- hooks/charmhelpers/fetch/archiveurl.py	2015-06-10 07:35:12 +0000
+++ hooks/charmhelpers/fetch/archiveurl.py	2015-12-01 15:05:49 +0000
@@ -77,6 +77,8 @@
     def can_handle(self, source):
         url_parts = self.parse_url(source)
         if url_parts.scheme not in ('http', 'https', 'ftp', 'file'):
+            # XXX: Why is this returning a boolean and a string? It's
+            # doomed to fail since "bool(can_handle('foo://'))"  will be True.
             return "Wrong source type"
         if get_archive_handler(self.base_url(source)):
             return True
@@ -155,7 +157,11 @@
             else:
                 algorithms = hashlib.algorithms_available
             if key in algorithms:
-                check_hash(dld_file, value, key)
+                if len(value) != 1:
+                    raise TypeError(
+                        "Expected 1 hash value, not %d" % len(value))
+                expected = value[0]
+                check_hash(dld_file, expected, key)
         if checksum:
             check_hash(dld_file, checksum, hash_type)
         return extract(dld_file, dest)
 
=== modified file 'hooks/charmhelpers/fetch/giturl.py'
--- hooks/charmhelpers/fetch/giturl.py	2015-06-10 07:35:12 +0000
+++ hooks/charmhelpers/fetch/giturl.py	2015-12-01 15:05:49 +0000
@@ -67,7 +67,7 @@
         try:
             self.clone(source, dest_dir, branch, depth)
         except GitCommandError as e:
-            raise UnhandledSource(e.message)
+            raise UnhandledSource(e)
         except OSError as e:
             raise UnhandledSource(e.strerror)
         return dest_dir
 
=== modified file 'hooks/services.py'
--- hooks/services.py	2015-09-14 16:44:47 +0000
+++ hooks/services.py	2015-12-01 15:05:49 +0000
@@ -2,6 +2,9 @@
 from charmhelpers.core import hookenv
 from charmhelpers.core.services.base import ServiceManager
 from charmhelpers.core.services import helpers
+from charmhelpers.contrib.openstack.templating import get_loader
+from charmhelpers.core.services.base import service_restart
+from charmhelpers.contrib.openstack.utils import os_release
 
 import vpp_utils
 import vpp_data
@@ -9,6 +12,7 @@
 
 def manage():
     config = hookenv.config()
+    release = os_release('neutron-common')
     manager = ServiceManager([
         # Actions which have no prerequisites and can be rerun
         {
@@ -18,6 +22,7 @@
             ],
             'provided_data': [
                 vpp_data.NeutronPluginRelation(),
+                vpp_data.AMQPRelation(),
             ],
         },
         # Install hugepages and components reliant on huge pages
@@ -34,12 +39,14 @@
         {
             'service': 'vpp-compute-render',
             'required_data': [
+                vpp_data.AMQPRelation(),
                 vpp_data.SystemResources(),
                 vpp_data.NeutronPluginRelation(),
                 vpp_data.ODLControllerRelation(),
                 config,
                 vpp_data.ConfigTranslation(),
                 vpp_data.PCIInfo(),
+                vpp_data.NeutronPluginAPIRelation(),
             ],
             'data_ready': [
                 vpp_utils.bind_orphaned_net_interfaces,
@@ -53,6 +60,27 @@
                     target='/etc/apparmor.d/libvirt/TEMPLATE.qemu',
                     on_change_action=(partial(vpp_utils.reload_apparmor)),
                 ),
+                helpers.render_template(
+                    source='neutron.conf',
+                    template_loader=get_loader('templates/', release),
+                    target='/etc/neutron/neutron.conf',
+                    on_change_action=(partial(service_restart,
+                                              'neutron-dhcp-agent')),
+                ),
+                helpers.render_template(
+                    source='dhcp_agent.ini',
+                    template_loader=get_loader('templates/', release),
+                    target='/etc/neutron/dhcp_agent.ini',
+                    on_change_action=(partial(service_restart,
+                                              'neutron-dhcp-agent')),
+                ),
+                helpers.render_template(
+                    source='metadata_agent.ini',
+                    template_loader=get_loader('templates/', release),
+                    target='/etc/neutron/metadata_agent.ini',
+                    on_change_action=(partial(service_restart,
+                                              'neutron-metadata-agent')),
+                ),
                 vpp_utils.odl_node_registration,
                 vpp_utils.odl_register_macs,
                 vpp_utils.bind_orphaned_net_interfaces,
 
=== modified file 'hooks/vpp_data.py'
--- hooks/vpp_data.py	2015-08-17 06:56:01 +0000
+++ hooks/vpp_data.py	2015-12-01 15:05:49 +0000
@@ -2,16 +2,47 @@
 import glob
 import os
 import json
+from charmhelpers.contrib.openstack import context
 from charmhelpers.core.services import helpers
 from charmhelpers.core.hookenv import(
     config,
     log,
 )
+import uuid
 
 VLAN = 'vlan'
 VXLAN = 'vxlan'
 GRE = 'gre'
 OVERLAY_NET_TYPES = [VXLAN, GRE]
+NEUTRON_CONF_DIR = "/etc/neutron"
+SHARED_SECRET_FILE = "/etc/neutron/secret.txt"
+
+class NeutronPluginAPIRelation(helpers.RelationContext):
+    name = 'neutron-plugin-api'
+    interface = 'neutron-plugin-api'
+
+    def get_first_data(self):
+        if self.get('neutron-plugin-api') and len(self['neutron-plugin-api']):
+            return self['neutron-plugin-api'][0]
+        else:
+            return {}
+
+    def get_data(self):
+        super(NeutronPluginAPIRelation, self).get_data()
+        api_server = self.get_first_data()
+        self['service_host'] = api_server.get('service_host')
+        self['service_protocol'] = api_server.get('service_protocol', 'http')
+        self['service_port'] = api_server.get('service_port')
+        self['admin_tenant_name'] = api_server.get('service_tenant')
+        self['admin_user'] = api_server.get('service_username')
+        self['admin_password'] = api_server.get('service_password')
+        self['region'] = api_server.get('region')
+
+    def is_ready(self):
+        if 'service_password' in self.get_first_data():
+            return True
+        else:
+            return False
 
 
 class ODLControllerRelation(helpers.RelationContext):
@@ -47,6 +78,22 @@
     name = 'neutron-plugin'
     interface = 'neutron-plugin-api-subordinate'
 
+    def __init__(self, *args, **kwargs):
+        super(NeutronPluginRelation, self).__init__(*args, **kwargs)
+        self['shared_secret'] = self.get_metadata_secret()
+
+    def get_metadata_secret(self):
+        secret = None
+        if os.path.exists(os.path.dirname(SHARED_SECRET_FILE)):
+            if os.path.exists(SHARED_SECRET_FILE):
+                with open(SHARED_SECRET_FILE, 'r') as secret_file:
+                    secret = secret_file.read().strip()
+            else:
+                secret = str(uuid.uuid4())
+                with open(SHARED_SECRET_FILE, 'w') as secret_file:
+                    secret_file.write(secret)
+        return secret
+
     def provide_data(self):
         # Add sections and tuples to insert values into neutron-server's
         # neutron.conf e.g.
@@ -83,6 +130,7 @@
         relation_info = {
             'neutron-plugin': 'odl',
             'subordinate_configuration': json.dumps(principle_config),
+            'metadata-shared-secret': self['shared_secret'],
         }
         return relation_info
 
@@ -179,3 +227,30 @@
                             'net': tmp_dict.get('net'),
                         }]
         return mac_net_config
+
+
+class AMQPRelation(helpers.RelationContext):
+    name = 'amqp'
+    interface = 'rabbitmq'
+
+    def __init__(self, *args, **kwargs):
+        self.ctxt = context.AMQPContext(ssl_dir=NEUTRON_CONF_DIR)()
+        super(AMQPRelation, self).__init__(*args, **kwargs)
+
+    def get_data(self):
+        super(AMQPRelation, self).get_data()
+        for key, value in self.ctxt.iteritems():
+            self[key] = value
+
+    def provide_data(self):
+        relation_info = {
+            'username': config('rabbit-user'),
+            'vhost': config('rabbit-vhost'),
+        }
+        return relation_info
+
+    def is_ready(self):
+        if self.ctxt.get('rabbitmq_password'):
+            return True
+        else:
+            return False
 
=== modified file 'hooks/vpp_utils.py'
--- hooks/vpp_utils.py	2015-09-14 16:44:47 +0000
+++ hooks/vpp_utils.py	2015-12-01 15:05:49 +0000
@@ -24,7 +24,8 @@
 ODL_MOUNT_PATH = ('/restconf/config/opendaylight-inventory:nodes/node/'
                   'controller-config/yang-ext:mount/config:modules')
 
-PACKAGES = ['python-psutil', 'gcc', 'make', 'python-crypto', 'qemu', 'seabios']
+PACKAGES = ['python-psutil', 'gcc', 'make', 'python-crypto', 'qemu', 'seabios',
+            'neutron-dhcp-agent']
 
 
 def install_packages(servicename):
 
=== modified file 'metadata.yaml'
--- metadata.yaml	2015-07-21 15:18:07 +0000
+++ metadata.yaml	2015-12-01 15:05:49 +0000
@@ -17,3 +17,7 @@
   container:
     interface: juju-info
     scope: container
+  amqp:
+    interface: rabbitmq
+  neutron-plugin-api:
+    interface: neutron-plugin-api
 
=== added directory 'templates/icehouse'
=== added file 'templates/icehouse/dhcp_agent.ini'
--- templates/icehouse/dhcp_agent.ini	1970-01-01 00:00:00 +0000
+++ templates/icehouse/dhcp_agent.ini	2015-12-01 15:05:49 +0000
@@ -0,0 +1,13 @@
+###############################################################################
+# [ WARNING ]
+# Configuration file maintained by Juju. Local changes may be overwritten.
+###############################################################################
+[DEFAULT]
+debug = {{ debug }}
+resync_interval = 5
+interface_driver = neutron.agent.linux.interface.NSNullDriver
+dhcp_driver = neutron.agent.linux.dhcp.Dnsmasq
+use_namespaces = True
+enable_isolated_metadata = True
+enable_metadata_network = True
+
 
=== added file 'templates/icehouse/metadata_agent.ini'
--- templates/icehouse/metadata_agent.ini	1970-01-01 00:00:00 +0000
+++ templates/icehouse/metadata_agent.ini	2015-12-01 15:05:49 +0000
@@ -0,0 +1,15 @@
+###############################################################################
+# [ WARNING ]
+# Configuration file maintained by Juju. Local changes may be overwritten.
+###############################################################################
+# Metadata service seems to cache neutron api url from keystone so trigger
+
+[DEFAULT]
+auth_url = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/v2.0
+auth_region = {{ region }}
+admin_tenant_name = {{ admin_tenant_name }}
+admin_user = {{ admin_user }}
+admin_password = {{ admin_password }}
+nova_metadata_port = 8775
+metadata_proxy_shared_secret = {{ shared_secret }}
+cache_url = memory://?default_ttl=5
 
=== added file 'templates/icehouse/neutron.conf'
--- templates/icehouse/neutron.conf	1970-01-01 00:00:00 +0000
+++ templates/icehouse/neutron.conf	2015-12-01 15:05:49 +0000
@@ -0,0 +1,31 @@
+# icehouse
+###############################################################################
+# [ WARNING ]
+# Configuration file maintained by Juju. Local changes may be overwritten.
+# Config managed by neutron-openvswitch charm
+###############################################################################
+[DEFAULT]
+verbose = {{ verbose }}
+debug = {{ debug }}
+state_path = /var/lib/neutron
+lock_path = $state_path/lock
+bind_host = 0.0.0.0
+bind_port = 9696
+
+api_paste_config = /etc/neutron/api-paste.ini
+auth_strategy = keystone
+default_notification_level = INFO
+notification_topics = notifications
+
+{% include "parts/rabbitmq" %}
+
+[QUOTAS]
+
+[DEFAULT_SERVICETYPE]
+
+[AGENT]
+root_helper = sudo neutron-rootwrap /etc/neutron/rootwrap.conf
+
+[keystone_authtoken]
+signing_dir = /var/lib/neutron/keystone-signing
+
 
=== added directory 'templates/parts'
=== added file 'templates/parts/rabbitmq'
--- templates/parts/rabbitmq	1970-01-01 00:00:00 +0000
+++ templates/parts/rabbitmq	2015-12-01 15:05:49 +0000
@@ -0,0 +1,21 @@
+{% if rabbitmq_host or rabbitmq_hosts -%}
+rabbit_userid = {{ rabbitmq_user }}
+rabbit_virtual_host = {{ rabbitmq_virtual_host }}
+rabbit_password = {{ rabbitmq_password }}
+{% if rabbitmq_hosts -%}
+rabbit_hosts = {{ rabbitmq_hosts }}
+{% if rabbitmq_ha_queues -%}
+rabbit_ha_queues = True
+rabbit_durable_queues = False
+{% endif -%}
+{% else -%}
+rabbit_host = {{ rabbitmq_host }}
+{% endif -%}
+{% if rabbit_ssl_port -%}
+rabbit_use_ssl = True
+rabbit_port = {{ rabbit_ssl_port }}
+{% if rabbit_ssl_ca -%}
+kombu_ssl_ca_certs = {{ rabbit_ssl_ca }}
+{% endif -%}
+{% endif -%}
+{% endif -%}
 \ No newline at end of file
 
=== modified file 'tests/charmhelpers/contrib/amulet/utils.py'
--- tests/charmhelpers/contrib/amulet/utils.py	2015-06-16 07:53:15 +0000
+++ tests/charmhelpers/contrib/amulet/utils.py	2015-12-01 15:05:49 +0000
@@ -14,14 +14,21 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
-import ConfigParser
 import io
 import logging
+import os
 import re
 import sys
 import time
 
+import amulet
+import distro_info
 import six
+from six.moves import configparser
+if six.PY3:
+    from urllib import parse as urlparse
+else:
+    import urlparse
 
 
 class AmuletUtils(object):
@@ -33,6 +40,7 @@
 
     def __init__(self, log_level=logging.ERROR):
         self.log = self.get_logger(level=log_level)
+        self.ubuntu_releases = self.get_ubuntu_releases()
 
     def get_logger(self, name="amulet-logger", level=logging.DEBUG):
         """Get a logger object that will log to stdout."""
@@ -70,12 +78,44 @@
         else:
             return False
 
+    def get_ubuntu_release_from_sentry(self, sentry_unit):
+        """Get Ubuntu release codename from sentry unit.
+
+        :param sentry_unit: amulet sentry/service unit pointer
+        :returns: list of strings - release codename, failure message
+        """
+        msg = None
+        cmd = 'lsb_release -cs'
+        release, code = sentry_unit.run(cmd)
+        if code == 0:
+            self.log.debug('{} lsb_release: {}'.format(
+                sentry_unit.info['unit_name'], release))
+        else:
+            msg = ('{} `{}` returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, release, code))
+        if release not in self.ubuntu_releases:
+            msg = ("Release ({}) not found in Ubuntu releases "
+                   "({})".format(release, self.ubuntu_releases))
+        return release, msg
+
     def validate_services(self, commands):
-        """Validate services.
-
-           Verify the specified services are running on the corresponding
+        """Validate that lists of commands succeed on service units.  Can be
+           used to verify system services are running on the corresponding
            service units.
-           """
+
+        :param commands: dict with sentry keys and arbitrary command list vals
+        :returns: None if successful, Failure string message otherwise
+        """
+        self.log.debug('Checking status of system services...')
+
+        # /!\ DEPRECATION WARNING (beisner):
+        # New and existing tests should be rewritten to use
+        # validate_services_by_name() as it is aware of init systems.
+        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+                      'validate_services_by_name instead of validate_services '
+                      'due to init system differences.')
+
         for k, v in six.iteritems(commands):
             for cmd in v:
                 output, code = k.run(cmd)
@@ -86,6 +126,45 @@
                     return "command `{}` returned {}".format(cmd, str(code))
         return None
 
+    def validate_services_by_name(self, sentry_services):
+        """Validate system service status by service name, automatically
+           detecting init system based on Ubuntu release codename.
+
+        :param sentry_services: dict with sentry keys and svc list values
+        :returns: None if successful, Failure string message otherwise
+        """
+        self.log.debug('Checking status of system services...')
+
+        # Point at which systemd became a thing
+        systemd_switch = self.ubuntu_releases.index('vivid')
+
+        for sentry_unit, services_list in six.iteritems(sentry_services):
+            # Get lsb_release codename from unit
+            release, ret = self.get_ubuntu_release_from_sentry(sentry_unit)
+            if ret:
+                return ret
+
+            for service_name in services_list:
+                if (self.ubuntu_releases.index(release) >= systemd_switch or
+                        service_name in ['rabbitmq-server', 'apache2']):
+                    # init is systemd (or regular sysv)
+                    cmd = 'sudo service {} status'.format(service_name)
+                    output, code = sentry_unit.run(cmd)
+                    service_running = code == 0
+                elif self.ubuntu_releases.index(release) < systemd_switch:
+                    # init is upstart
+                    cmd = 'sudo status {}'.format(service_name)
+                    output, code = sentry_unit.run(cmd)
+                    service_running = code == 0 and "start/running" in output
+
+                self.log.debug('{} `{}` returned '
+                               '{}'.format(sentry_unit.info['unit_name'],
+                                           cmd, code))
+                if not service_running:
+                    return u"command `{}` returned {} {}".format(
+                        cmd, output, str(code))
+        return None
+
     def _get_config(self, unit, filename):
         """Get a ConfigParser object for parsing a unit's config file."""
         file_contents = unit.file_contents(filename)
@@ -93,7 +172,7 @@
         # NOTE(beisner):  by default, ConfigParser does not handle options
         # with no value, such as the flags used in the mysql my.cnf file.
         # https://bugs.python.org/issue7005
-        config = ConfigParser.ConfigParser(allow_no_value=True)
+        config = configparser.ConfigParser(allow_no_value=True)
         config.readfp(io.StringIO(file_contents))
         return config
 
@@ -103,7 +182,15 @@
 
            Verify that the specified section of the config file contains
            the expected option key:value pairs.
+
+           Compare expected dictionary data vs actual dictionary data.
+           The values in the 'expected' dictionary can be strings, bools, ints,
+           longs, or can be a function that evaluates a variable and returns a
+           bool.
            """
+        self.log.debug('Validating config file data ({} in {} on {})'
+                       '...'.format(section, config_file,
+                                    sentry_unit.info['unit_name']))
         config = self._get_config(sentry_unit, config_file)
 
         if section != 'DEFAULT' and not config.has_section(section):
@@ -112,9 +199,20 @@
         for k in expected.keys():
             if not config.has_option(section, k):
                 return "section [{}] is missing option {}".format(section, k)
-            if config.get(section, k) != expected[k]:
+
+            actual = config.get(section, k)
+            v = expected[k]
+            if (isinstance(v, six.string_types) or
+                    isinstance(v, bool) or
+                    isinstance(v, six.integer_types)):
+                # handle explicit values
+                if actual != v:
+                    return "section [{}] {}:{} != expected {}:{}".format(
+                           section, k, actual, k, expected[k])
+            # handle function pointers, such as not_null or valid_ip
+            elif not v(actual):
                 return "section [{}] {}:{} != expected {}:{}".format(
-                       section, k, config.get(section, k), k, expected[k])
+                       section, k, actual, k, expected[k])
         return None
 
     def _validate_dict_data(self, expected, actual):
@@ -122,7 +220,7 @@
 
            Compare expected dictionary data vs actual dictionary data.
            The values in the 'expected' dictionary can be strings, bools, ints,
-           longs, or can be a function that evaluate a variable and returns a
+           longs, or can be a function that evaluates a variable and returns a
            bool.
            """
         self.log.debug('actual: {}'.format(repr(actual)))
@@ -133,8 +231,10 @@
                 if (isinstance(v, six.string_types) or
                         isinstance(v, bool) or
                         isinstance(v, six.integer_types)):
+                    # handle explicit values
                     if v != actual[k]:
                         return "{}:{}".format(k, actual[k])
+                # handle function pointers, such as not_null or valid_ip
                 elif not v(actual[k]):
                     return "{}:{}".format(k, actual[k])
             else:
@@ -321,3 +421,133 @@
 
     def endpoint_error(self, name, data):
         return 'unexpected endpoint data in {} - {}'.format(name, data)
+
+    def get_ubuntu_releases(self):
+        """Return a list of all Ubuntu releases in order of release."""
+        _d = distro_info.UbuntuDistroInfo()
+        _release_list = _d.all
+        self.log.debug('Ubuntu release list: {}'.format(_release_list))
+        return _release_list
+
+    def file_to_url(self, file_rel_path):
+        """Convert a relative file path to a file URL."""
+        _abs_path = os.path.abspath(file_rel_path)
+        return urlparse.urlparse(_abs_path, scheme='file').geturl()
+
+    def check_commands_on_units(self, commands, sentry_units):
+        """Check that all commands in a list exit zero on all
+        sentry units in a list.
+
+        :param commands:  list of bash commands
+        :param sentry_units:  list of sentry unit pointers
+        :returns: None if successful; Failure message otherwise
+        """
+        self.log.debug('Checking exit codes for {} commands on {} '
+                       'sentry units...'.format(len(commands),
+                                                len(sentry_units)))
+        for sentry_unit in sentry_units:
+            for cmd in commands:
+                output, code = sentry_unit.run(cmd)
+                if code == 0:
+                    self.log.debug('{} `{}` returned {} '
+                                   '(OK)'.format(sentry_unit.info['unit_name'],
+                                                 cmd, code))
+                else:
+                    return ('{} `{}` returned {} '
+                            '{}'.format(sentry_unit.info['unit_name'],
+                                        cmd, code, output))
+        return None
+
+    def get_process_id_list(self, sentry_unit, process_name):
+        """Get a list of process ID(s) from a single sentry juju unit
+        for a single process name.
+
+        :param sentry_unit: Pointer to amulet sentry instance (juju unit)
+        :param process_name: Process name
+        :returns: List of process IDs
+        """
+        cmd = 'pidof {}'.format(process_name)
+        output, code = sentry_unit.run(cmd)
+        if code != 0:
+            msg = ('{} `{}` returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, code, output))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+        return str(output).split()
+
+    def get_unit_process_ids(self, unit_processes):
+        """Construct a dict containing unit sentries, process names, and
+        process IDs."""
+        pid_dict = {}
+        for sentry_unit, process_list in unit_processes.iteritems():
+            pid_dict[sentry_unit] = {}
+            for process in process_list:
+                pids = self.get_process_id_list(sentry_unit, process)
+                pid_dict[sentry_unit].update({process: pids})
+        return pid_dict
+
+    def validate_unit_process_ids(self, expected, actual):
+        """Validate process id quantities for services on units."""
+        self.log.debug('Checking units for running processes...')
+        self.log.debug('Expected PIDs: {}'.format(expected))
+        self.log.debug('Actual PIDs: {}'.format(actual))
+
+        if len(actual) != len(expected):
+            return ('Unit count mismatch.  expected, actual: {}, '
+                    '{} '.format(len(expected), len(actual)))
+
+        for (e_sentry, e_proc_names) in expected.iteritems():
+            e_sentry_name = e_sentry.info['unit_name']
+            if e_sentry in actual.keys():
+                a_proc_names = actual[e_sentry]
+            else:
+                return ('Expected sentry ({}) not found in actual dict data.'
+                        '{}'.format(e_sentry_name, e_sentry))
+
+            if len(e_proc_names.keys()) != len(a_proc_names.keys()):
+                return ('Process name count mismatch.  expected, actual: {}, '
+                        '{}'.format(len(expected), len(actual)))
+
+            for (e_proc_name, e_pids_length), (a_proc_name, a_pids) in \
+                    zip(e_proc_names.items(), a_proc_names.items()):
+                if e_proc_name != a_proc_name:
+                    return ('Process name mismatch.  expected, actual: {}, '
+                            '{}'.format(e_proc_name, a_proc_name))
+
+                a_pids_length = len(a_pids)
+                fail_msg = ('PID count mismatch. {} ({}) expected, actual: '
+                            '{}, {} ({})'.format(e_sentry_name, e_proc_name,
+                                                 e_pids_length, a_pids_length,
+                                                 a_pids))
+
+                # If expected is not bool, ensure PID quantities match
+                if not isinstance(e_pids_length, bool) and \
+                        a_pids_length != e_pids_length:
+                    return fail_msg
+                # If expected is bool True, ensure 1 or more PIDs exist
+                elif isinstance(e_pids_length, bool) and \
+                        e_pids_length is True and a_pids_length < 1:
+                    return fail_msg
+                # If expected is bool False, ensure 0 PIDs exist
+                elif isinstance(e_pids_length, bool) and \
+                        e_pids_length is False and a_pids_length != 0:
+                    return fail_msg
+                else:
+                    self.log.debug('PID check OK: {} {} {}: '
+                                   '{}'.format(e_sentry_name, e_proc_name,
+                                               e_pids_length, a_pids))
+        return None
+
+    def validate_list_of_identical_dicts(self, list_of_dicts):
+        """Check that all dicts within a list are identical."""
+        hashes = []
+        for _dict in list_of_dicts:
+            hashes.append(hash(frozenset(_dict.items())))
+
+        self.log.debug('Hashes: {}'.format(hashes))
+        if len(set(hashes)) == 1:
+            self.log.debug('Dicts within list are identical')
+        else:
+            return 'Dicts within list are not identical'
+
+        return None
 
=== modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py'
--- tests/charmhelpers/contrib/openstack/amulet/deployment.py	2015-06-24 08:25:28 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/deployment.py	2015-12-01 15:05:49 +0000
@@ -44,7 +44,7 @@
            Determine if the local branch being tested is derived from its
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
-        base_charms = ['mysql', 'mongodb']
+        base_charms = ['mysql', 'mongodb', 'nrpe']
 
         if self.series in ['precise', 'trusty']:
             base_series = self.series
@@ -83,9 +83,10 @@
         services.append(this_service)
         use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',