Merge lp:~gnuoy/charms/trusty/swift-storage/next-charm-sync into lp:~openstack-charmers-archive/charms/trusty/swift-storage/next

[Liam Young]

[Liam Young (gnuoy)]

Approved by jamespage

review: Approve

=== added file '.bzrignore'
--- .bzrignore	1970-01-01 00:00:00 +0000
+++ .bzrignore	2014-08-13 13:52:53 +0000
@@ -0,0 +1,2 @@
+bin
+.coverage

=== modified file 'Makefile'
--- Makefile	2014-07-11 16:41:12 +0000
+++ Makefile	2014-08-13 13:52:53 +0000
@@ -17,9 +17,14 @@
 	#   https://bugs.launchpad.net/amulet/+bug/1320357
 	@juju test -v -p AMULET_HTTP_PROXY
 
-sync:
-	@charm-helper-sync -c charm-helpers-hooks.yaml
-	@charm-helper-sync -c charm-helpers-tests.yaml
+bin/charm_helpers_sync.py:
+	@mkdir -p bin
+	@bzr cat lp:charm-helpers/tools/charm_helpers_sync/charm_helpers_sync.py \
+	> bin/charm_helpers_sync.py
+
+sync: bin/charm_helpers_sync.py
+	@$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-hooks.yaml
+	@$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-tests.yaml
 
 publish: lint test
 	bzr push lp:charms/swift-storage

=== modified file 'hooks/charmhelpers/contrib/hahelpers/cluster.py'
--- hooks/charmhelpers/contrib/hahelpers/cluster.py	2014-07-28 11:54:32 +0000
+++ hooks/charmhelpers/contrib/hahelpers/cluster.py	2014-08-13 13:52:53 +0000
@@ -6,6 +6,11 @@
 #  Adam Gandelman <[e-mail]>
 #
 
+"""
+Helpers for clustering and determining "cluster leadership" and other
+clustering-related helpers.
+"""
+
 import subprocess
 import os
 
@@ -19,6 +24,7 @@
     config as config_get,
     INFO,
     ERROR,
+    WARNING,
     unit_get,
 )
 
@@ -27,6 +33,29 @@
     pass
 
 
+def is_elected_leader(resource):
+    """
+    Returns True if the charm executing this is the elected cluster leader.
+
+    It relies on two mechanisms to determine leadership:
+        1. If the charm is part of a corosync cluster, call corosync to
+        determine leadership.
+        2. If the charm is not part of a corosync cluster, the leader is
+        determined as being "the alive unit with the lowest unit numer". In
+        other words, the oldest surviving unit.
+    """
+    if is_clustered():
+        if not is_crm_leader(resource):
+            log('Deferring action to CRM leader.', level=INFO)
+            return False
+    else:
+        peers = peer_units()
+        if peers and not oldest_peer(peers):
+            log('Deferring action to oldest service unit.', level=INFO)
+            return False
+    return True
+
+
 def is_clustered():
     for r_id in (relation_ids('ha') or []):
         for unit in (relation_list(r_id) or []):
@@ -38,7 +67,11 @@
     return False
 
 
-def is_leader(resource):
+def is_crm_leader(resource):
+    """
+    Returns True if the charm calling this is the elected corosync leader,
+    as returned by calling the external "crm" command.
+    """
     cmd = [
         "crm", "resource",
         "show", resource
@@ -54,15 +87,31 @@
             return False
 
 
-def peer_units():
+def is_leader(resource):
+    log("is_leader is deprecated. Please consider using is_crm_leader "
+        "instead.", level=WARNING)
+    return is_crm_leader(resource)
+
+
+def peer_units(peer_relation="cluster"):
     peers = []
-    for r_id in (relation_ids('cluster') or []):
+    for r_id in (relation_ids(peer_relation) or []):
         for unit in (relation_list(r_id) or []):
             peers.append(unit)
     return peers
 
 
+def peer_ips(peer_relation='cluster', addr_key='private-address'):
+    '''Return a dict of peers and their private-address'''
+    peers = {}
+    for r_id in relation_ids(peer_relation):
+        for unit in relation_list(r_id):
+            peers[unit] = relation_get(addr_key, rid=r_id, unit=unit)
+    return peers
+
+
 def oldest_peer(peers):
+    """Determines who the oldest peer is by comparing unit numbers."""
     local_unit_no = int(os.getenv('JUJU_UNIT_NAME').split('/')[1])
     for peer in peers:
         remote_unit_no = int(peer.split('/')[1])
@@ -72,16 +121,9 @@
 
 
 def eligible_leader(resource):
-    if is_clustered():
-        if not is_leader(resource):
-            log('Deferring action to CRM leader.', level=INFO)
-            return False
-    else:
-        peers = peer_units()
-        if peers and not oldest_peer(peers):
-            log('Deferring action to oldest service unit.', level=INFO)
-            return False
-    return True
+    log("eligible_leader is deprecated. Please consider using "
+        "is_elected_leader instead.", level=WARNING)
+    return is_elected_leader(resource)
 
 
 def https():

=== modified file 'hooks/charmhelpers/contrib/network/ip.py'
--- hooks/charmhelpers/contrib/network/ip.py	2014-07-28 12:08:48 +0000
+++ hooks/charmhelpers/contrib/network/ip.py	2014-08-13 13:52:53 +0000
@@ -4,7 +4,7 @@
 
 from charmhelpers.fetch import apt_install
 from charmhelpers.core.hookenv import (
-    ERROR, log,
+    ERROR, log, config,
 )
 
 try:
@@ -154,3 +154,21 @@
 get_iface_for_address = partial(_get_for_address, key='iface')
 
 get_netmask_for_address = partial(_get_for_address, key='netmask')
+
+
+def get_ipv6_addr(iface="eth0"):
+    try:
+        iface_addrs = netifaces.ifaddresses(iface)
+        if netifaces.AF_INET6 not in iface_addrs:
+            raise Exception("Interface '%s' doesn't have an ipv6 address." % iface)
+
+        addresses = netifaces.ifaddresses(iface)[netifaces.AF_INET6]
+        ipv6_addr = [a['addr'] for a in addresses if not a['addr'].startswith('fe80')
+                     and config('vip') != a['addr']]
+        if not ipv6_addr:
+            raise Exception("Interface '%s' doesn't have global ipv6 address." % iface)
+
+        return ipv6_addr[0]
+
+    except ValueError:
+        raise ValueError("Invalid interface '%s'" % iface)

=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/deployment.py'
--- hooks/charmhelpers/contrib/openstack/amulet/deployment.py	2014-07-28 11:54:32 +0000
+++ hooks/charmhelpers/contrib/openstack/amulet/deployment.py	2014-08-13 13:52:53 +0000
@@ -4,8 +4,11 @@
 
 
 class OpenStackAmuletDeployment(AmuletDeployment):
-    """This class inherits from AmuletDeployment and has additional support
-       that is specifically for use by OpenStack charms."""
+    """OpenStack amulet deployment.
+
+       This class inherits from AmuletDeployment and has additional support
+       that is specifically for use by OpenStack charms.
+       """
 
     def __init__(self, series=None, openstack=None, source=None):
         """Initialize the deployment environment."""
@@ -40,11 +43,14 @@
             self.d.configure(service, config)
 
     def _get_openstack_release(self):
-        """Return an integer representing the enum value of the openstack
-           release."""
-        self.precise_essex, self.precise_folsom, self.precise_grizzly, \
-            self.precise_havana, self.precise_icehouse, \
-            self.trusty_icehouse = range(6)
+        """Get openstack release.
+
+           Return an integer representing the enum value of the openstack
+           release.
+           """
+        (self.precise_essex, self.precise_folsom, self.precise_grizzly,
+         self.precise_havana, self.precise_icehouse,
+         self.trusty_icehouse) = range(6)
         releases = {
             ('precise', None): self.precise_essex,
             ('precise', 'cloud:precise-folsom'): self.precise_folsom,

=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/utils.py'
--- hooks/charmhelpers/contrib/openstack/amulet/utils.py	2014-07-28 11:54:32 +0000
+++ hooks/charmhelpers/contrib/openstack/amulet/utils.py	2014-08-13 13:52:53 +0000
@@ -16,8 +16,11 @@
 
 
 class OpenStackAmuletUtils(AmuletUtils):
-    """This class inherits from AmuletUtils and has additional support
-       that is specifically for use by OpenStack charms."""
+    """OpenStack amulet utilities.
+
+       This class inherits from AmuletUtils and has additional support
+       that is specifically for use by OpenStack charms.
+       """
 
     def __init__(self, log_level=ERROR):
         """Initialize the deployment environment."""
@@ -25,13 +28,17 @@
 
     def validate_endpoint_data(self, endpoints, admin_port, internal_port,
                                public_port, expected):
-        """Validate actual endpoint data vs expected endpoint data. The ports
-           are used to find the matching endpoint."""
+        """Validate endpoint data.
+
+           Validate actual endpoint data vs expected endpoint data. The ports
+           are used to find the matching endpoint.
+           """
         found = False
         for ep in endpoints:
             self.log.debug('endpoint: {}'.format(repr(ep)))
-            if admin_port in ep.adminurl and internal_port in ep.internalurl \
-               and public_port in ep.publicurl:
+            if (admin_port in ep.adminurl and
+                    internal_port in ep.internalurl and
+                    public_port in ep.publicurl):
                 found = True
                 actual = {'id': ep.id,
                           'region': ep.region,
@@ -47,8 +54,11 @@
             return 'endpoint not found'
 
     def validate_svc_catalog_endpoint_data(self, expected, actual):
-        """Validate a list of actual service catalog endpoints vs a list of
-           expected service catalog endpoints."""
+        """Validate service catalog endpoint data.
+
+           Validate a list of actual service catalog endpoints vs a list of
+           expected service catalog endpoints.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for k, v in expected.iteritems():
             if k in actual:
@@ -60,8 +70,11 @@
         return ret
 
     def validate_tenant_data(self, expected, actual):
-        """Validate a list of actual tenant data vs list of expected tenant
-           data."""
+        """Validate tenant data.
+
+           Validate a list of actual tenant data vs list of expected tenant
+           data.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -78,8 +91,11 @@
         return ret
 
     def validate_role_data(self, expected, actual):
-        """Validate a list of actual role data vs a list of expected role
-           data."""
+        """Validate role data.
+
+           Validate a list of actual role data vs a list of expected role
+           data.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -95,8 +111,11 @@
         return ret
 
     def validate_user_data(self, expected, actual):
-        """Validate a list of actual user data vs a list of expected user
-           data."""
+        """Validate user data.
+
+           Validate a list of actual user data vs a list of expected user
+           data.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -114,21 +133,24 @@
         return ret
 
     def validate_flavor_data(self, expected, actual):
-        """Validate a list of actual flavors vs a list of expected flavors."""
+        """Validate flavor data.
+
+           Validate a list of actual flavors vs a list of expected flavors.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         act = [a.name for a in actual]
         return self._validate_list_data(expected, act)
 
     def tenant_exists(self, keystone, tenant):
-        """Return True if tenant exists"""
+        """Return True if tenant exists."""
         return tenant in [t.name for t in keystone.tenants.list()]
 
     def authenticate_keystone_admin(self, keystone_sentry, user, password,
                                     tenant):
         """Authenticates admin user with the keystone admin endpoint."""
-        service_ip = \
-            keystone_sentry.relation('shared-db',
-                                     'mysql:shared-db')['private-address']
+        unit = keystone_sentry
+        service_ip = unit.relation('shared-db',
+                                   'mysql:shared-db')['private-address']
         ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8'))
         return keystone_client.Client(username=user, password=password,
                                       tenant_name=tenant, auth_url=ep)
@@ -177,12 +199,40 @@
             image = glance.images.create(name=image_name, is_public=True,
                                          disk_format='qcow2',
                                          container_format='bare', data=f)
+        count = 1
+        status = image.status
+        while status != 'active' and count < 10:
+            time.sleep(3)
+            image = glance.images.get(image.id)
+            status = image.status
+            self.log.debug('image status: {}'.format(status))
+            count += 1
+
+        if status != 'active':
+            self.log.error('image creation timed out')
+            return None
+
         return image
 
     def delete_image(self, glance, image):
         """Delete the specified image."""
+        num_before = len(list(glance.images.list()))
         glance.images.delete(image)
 
+        count = 1
+        num_after = len(list(glance.images.list()))
+        while num_after != (num_before - 1) and count < 10:
+            time.sleep(3)
+            num_after = len(list(glance.images.list()))
+            self.log.debug('number of images: {}'.format(num_after))
+            count += 1
+
+        if num_after != (num_before - 1):
+            self.log.error('image deletion timed out')
+            return False
+
+        return True
+
     def create_instance(self, nova, image_name, instance_name, flavor):
         """Create the specified instance."""
         image = nova.images.find(name=image_name)
@@ -199,11 +249,27 @@
             self.log.debug('instance status: {}'.format(status))
             count += 1
 
-        if status == 'BUILD':
+        if status != 'ACTIVE':
+            self.log.error('instance creation timed out')
             return None
 
         return instance
 
     def delete_instance(self, nova, instance):
         """Delete the specified instance."""
+        num_before = len(list(nova.servers.list()))
         nova.servers.delete(instance)
+
+        count = 1
+        num_after = len(list(nova.servers.list()))
+        while num_after != (num_before - 1) and count < 10:
+            time.sleep(3)
+            num_after = len(list(nova.servers.list()))
+            self.log.debug('number of instances: {}'.format(num_after))
+            count += 1
+
+        if num_after != (num_before - 1):
+            self.log.error('instance deletion timed out')
+            return False
+
+        return True

=== modified file 'hooks/charmhelpers/contrib/openstack/context.py'
--- hooks/charmhelpers/contrib/openstack/context.py	2014-07-28 11:54:32 +0000
+++ hooks/charmhelpers/contrib/openstack/context.py	2014-08-13 13:52:53 +0000
@@ -44,7 +44,10 @@
     neutron_plugin_attribute,
 )
 
-from charmhelpers.contrib.network.ip import get_address_in_network
+from charmhelpers.contrib.network.ip import (
+    get_address_in_network,
+    get_ipv6_addr,
+)
 
 CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
 
@@ -401,9 +404,12 @@
 
         cluster_hosts = {}
         l_unit = local_unit().replace('/', '-')
-        cluster_hosts[l_unit] = \
-            get_address_in_network(config('os-internal-network'),
-                                   unit_get('private-address'))
+        if config('prefer-ipv6'):
+            addr = get_ipv6_addr()
+        else:
+            addr = unit_get('private-address')
+        cluster_hosts[l_unit] = get_address_in_network(config('os-internal-network'),
+                                                       addr)
 
         for rid in relation_ids('cluster'):
             for unit in related_units(rid):
@@ -414,6 +420,16 @@
         ctxt = {
             'units': cluster_hosts,
         }
+
+        if config('prefer-ipv6'):
+            ctxt['local_host'] = 'ip6-localhost'
+            ctxt['haproxy_host'] = '::'
+            ctxt['stat_port'] = ':::8888'
+        else:
+            ctxt['local_host'] = '127.0.0.1'
+            ctxt['haproxy_host'] = '0.0.0.0'
+            ctxt['stat_port'] = ':8888'
+
         if len(cluster_hosts.keys()) > 1:
             # Enable haproxy when we have enough peers.
             log('Ensuring haproxy enabled in /etc/default/haproxy.')

=== modified file 'hooks/charmhelpers/contrib/openstack/ip.py'
--- hooks/charmhelpers/contrib/openstack/ip.py	2014-07-28 11:54:32 +0000
+++ hooks/charmhelpers/contrib/openstack/ip.py	2014-08-13 13:52:53 +0000
@@ -7,6 +7,7 @@
     get_address_in_network,
     is_address_in_network,
     is_ipv6,
+    get_ipv6_addr,
 )
 
 from charmhelpers.contrib.hahelpers.cluster import is_clustered
@@ -64,10 +65,13 @@
                         vip):
                     resolved_address = vip
     else:
+        if config('prefer-ipv6'):
+            fallback_addr = get_ipv6_addr()
+        else:
+            fallback_addr = unit_get(_address_map[endpoint_type]['fallback'])
         resolved_address = get_address_in_network(
-            config(_address_map[endpoint_type]['config']),
-            unit_get(_address_map[endpoint_type]['fallback'])
-        )
+            config(_address_map[endpoint_type]['config']), fallback_addr)
+
     if resolved_address is None:
         raise ValueError('Unable to resolve a suitable IP address'
                          ' based on charm state and configuration')

=== modified file 'hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg'
--- hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg	2014-07-28 11:54:32 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg	2014-08-13 13:52:53 +0000
@@ -1,6 +1,6 @@
 global
-    log 127.0.0.1 local0
-    log 127.0.0.1 local1 notice
+    log {{ local_host }} local0
+    log {{ local_host }} local1 notice
     maxconn 20000
     user haproxy
     group haproxy
@@ -17,7 +17,7 @@
     timeout client 30000
     timeout server 30000
 
-listen stats :8888
+listen stats {{ stat_port }}
     mode http
     stats enable
     stats hide-version

=== modified file 'hooks/charmhelpers/contrib/storage/linux/utils.py'
--- hooks/charmhelpers/contrib/storage/linux/utils.py	2014-07-10 18:04:35 +0000
+++ hooks/charmhelpers/contrib/storage/linux/utils.py	2014-08-13 13:52:53 +0000
@@ -46,5 +46,8 @@
     :returns: boolean: True if the path represents a mounted device, False if
         it doesn't.
     '''
+    is_partition = bool(re.search(r".*[0-9]+\b", device))
     out = check_output(['mount'])
+    if is_partition:
+        return bool(re.search(device + r"\b", out))
     return bool(re.search(device + r"[0-9]+\b", out))

=== modified file 'hooks/charmhelpers/core/host.py'
--- hooks/charmhelpers/core/host.py	2014-07-28 11:54:32 +0000
+++ hooks/charmhelpers/core/host.py	2014-08-13 13:52:53 +0000
@@ -12,6 +12,8 @@
 import string
 import subprocess
 import hashlib
+import shutil
+from contextlib import contextmanager
 
 from collections import OrderedDict
 
@@ -52,7 +54,7 @@
 def service_running(service):
     """Determine whether a system service is running"""
     try:
-        output = subprocess.check_output(['service', service, 'status'])
+        output = subprocess.check_output(['service', service, 'status'], stderr=subprocess.STDOUT)
     except subprocess.CalledProcessError:
         return False
     else:
@@ -62,6 +64,16 @@
             return False
 
 
+def service_available(service_name):
+    """Determine whether a system service is available"""
+    try:
+        subprocess.check_output(['service', service_name, 'status'], stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError:
+        return False
+    else:
+        return True
+
+
 def adduser(username, password=None, shell='/bin/bash', system_user=False):
     """Add a user to the system"""
     try:
@@ -329,3 +341,24 @@
         pkgcache = apt_pkg.Cache()
     pkg = pkgcache[package]
     return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)
+
+
+@contextmanager
+def chdir(d):
+    cur = os.getcwd()
+    try:
+        yield os.chdir(d)
+    finally:
+        os.chdir(cur)
+
+
+def chownr(path, owner, group):
+    uid = pwd.getpwnam(owner).pw_uid
+    gid = grp.getgrnam(group).gr_gid
+
+    for root, dirs, files in os.walk(path):
+        for name in dirs + files:
+            full = os.path.join(root, name)
+            broken_symlink = os.path.lexists(full) and not os.path.exists(full)
+            if not broken_symlink:
+                os.chown(full, uid, gid)

=== added directory 'hooks/charmhelpers/core/services'
=== added file 'hooks/charmhelpers/core/services/__init__.py'
--- hooks/charmhelpers/core/services/__init__.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/services/__init__.py	2014-08-13 13:52:53 +0000
@@ -0,0 +1,2 @@
+from .base import *
+from .helpers import *

=== added file 'hooks/charmhelpers/core/services/base.py'
--- hooks/charmhelpers/core/services/base.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/services/base.py	2014-08-13 13:52:53 +0000
@@ -0,0 +1,305 @@
+import os
+import re
+import json
+from collections import Iterable
+
+from charmhelpers.core import host
+from charmhelpers.core import hookenv
+
+
+__all__ = ['ServiceManager', 'ManagerCallback',
+           'PortManagerCallback', 'open_ports', 'close_ports', 'manage_ports',
+           'service_restart', 'service_stop']
+
+
+class ServiceManager(object):
+    def __init__(self, services=None):
+        """
+        Register a list of services, given their definitions.
+
+        Traditional charm authoring is focused on implementing hooks.  That is,
+        the charm author is thinking in terms of "What hook am I handling; what
+        does this hook need to do?"  However, in most cases, the real question
+        should be "Do I have the information I need to configure and start this
+        piece of software and, if so, what are the steps for doing so?"  The
+        ServiceManager framework tries to bring the focus to the data and the
+        setup tasks, in the most declarative way possible.
+
+        Service definitions are dicts in the following formats (all keys except
+        'service' are optional)::
+
+            {
+                "service": <service name>,
+                "required_data": <list of required data contexts>,
+                "data_ready": <one or more callbacks>,
+                "data_lost": <one or more callbacks>,
+                "start": <one or more callbacks>,
+                "stop": <one or more callbacks>,
+                "ports": <list of ports to manage>,
+            }
+
+        The 'required_data' list should contain dicts of required data (or
+        dependency managers that act like dicts and know how to collect the data).
+        Only when all items in the 'required_data' list are populated are the list
+        of 'data_ready' and 'start' callbacks executed.  See `is_ready()` for more
+        information.
+
+        The 'data_ready' value should be either a single callback, or a list of
+        callbacks, to be called when all items in 'required_data' pass `is_ready()`.
+        Each callback will be called with the service name as the only parameter.
+        After all of the 'data_ready' callbacks are called, the 'start' callbacks
+        are fired.
+
+        The 'data_lost' value should be either a single callback, or a list of
+        callbacks, to be called when a 'required_data' item no longer passes
+        `is_ready()`.  Each callback will be called with the service name as the
+        only parameter.  After all of the 'data_lost' callbacks are called,
+        the 'stop' callbacks are fired.
+
+        The 'start' value should be either a single callback, or a list of
+        callbacks, to be called when starting the service, after the 'data_ready'
+        callbacks are complete.  Each callback will be called with the service
+        name as the only parameter.  This defaults to
+        `[host.service_start, services.open_ports]`.
+
+        The 'stop' value should be either a single callback, or a list of
+        callbacks, to be called when stopping the service.  If the service is
+        being stopped because it no longer has all of its 'required_data', this
+        will be called after all of the 'data_lost' callbacks are complete.
+        Each callback will be called with the service name as the only parameter.
+        This defaults to `[services.close_ports, host.service_stop]`.
+
+        The 'ports' value should be a list of ports to manage.  The default
+        'start' handler will open the ports after the service is started,
+        and the default 'stop' handler will close the ports prior to stopping
+        the service.
+
+
+        Examples:
+
+        The following registers an Upstart service called bingod that depends on
+        a mongodb relation and which runs a custom `db_migrate` function prior to
+        restarting the service, and a Runit service called spadesd::
+
+            manager = services.ServiceManager([
+                {
+                    'service': 'bingod',
+                    'ports': [80, 443],
+                    'required_data': [MongoRelation(), config(), {'my': 'data'}],
+                    'data_ready': [
+                        services.template(source='bingod.conf'),
+                        services.template(source='bingod.ini',
+                                          target='/etc/bingod.ini',
+                                          owner='bingo', perms=0400),
+                    ],
+                },
+                {
+                    'service': 'spadesd',
+                    'data_ready': services.template(source='spadesd_run.j2',
+                                                    target='/etc/sv/spadesd/run',
+                                                    perms=0555),
+                    'start': runit_start,
+                    'stop': runit_stop,
+                },
+            ])
+            manager.manage()
+        """
+        self._ready_file = os.path.join(hookenv.charm_dir(), 'READY-SERVICES.json')
+        self._ready = None
+        self.services = {}
+        for service in services or []:
+            service_name = service['service']
+            self.services[service_name] = service
+
+    def manage(self):
+        """
+        Handle the current hook by doing The Right Thing with the registered services.
+        """
+        hook_name = hookenv.hook_name()
+        if hook_name == 'stop':
+            self.stop_services()
+        else:
+            self.provide_data()
+            self.reconfigure_services()
+
+    def provide_data(self):
+        hook_name = hookenv.hook_name()
+        for service in self.services.values():
+            for provider in service.get('provided_data', []):
+                if re.match(r'{}-relation-(joined|changed)'.format(provider.name), hook_name):
+                    data = provider.provide_data()
+                    if provider._is_ready(data):
+                        hookenv.relation_set(None, data)
+
+    def reconfigure_services(self, *service_names):
+        """
+        Update all files for one or more registered services, and,
+        if ready, optionally restart them.
+
+        If no service names are given, reconfigures all registered services.
+        """
+        for service_name in service_names or self.services.keys():
+            if self.is_ready(service_name):
+                self.fire_event('data_ready', service_name)
+                self.fire_event('start', service_name, default=[
+                    service_restart,
+                    manage_ports])
+                self.save_ready(service_name)
+            else:
+                if self.was_ready(service_name):
+                    self.fire_event('data_lost', service_name)
+                self.fire_event('stop', service_name, default=[
+                    manage_ports,
+                    service_stop])
+                self.save_lost(service_name)
+
+    def stop_services(self, *service_names):
+        """
+        Stop one or more registered services, by name.
+
+        If no service names are given, stops all registered services.
+        """
+        for service_name in service_names or self.services.keys():
+            self.fire_event('stop', service_name, default=[
+                manage_ports,
+                service_stop])
+
+    def get_service(self, service_name):
+        """
+        Given the name of a registered service, return its service definition.
+        """
+        service = self.services.get(service_name)
+        if not service:
+            raise KeyError('Service not registered: %s' % service_name)
+        return service
+
+    def fire_event(self, event_name, service_name, default=None):
+        """
+        Fire a data_ready, data_lost, start, or stop event on a given service.
+        """
+        service = self.get_service(service_name)
+        callbacks = service.get(event_name, default)
+        if not callbacks:
+            return
+        if not isinstance(callbacks, Iterable):
+            callbacks = [callbacks]
+        for callback in callbacks:
+            if isinstance(callback, ManagerCallback):
+                callback(self, service_name, event_name)
+            else:
+                callback(service_name)
+
+    def is_ready(self, service_name):
+        """
+        Determine if a registered service is ready, by checking its 'required_data'.
+
+        A 'required_data' item can be any mapping type, and is considered ready
+        if `bool(item)` evaluates as True.
+        """
+        service = self.get_service(service_name)
+        reqs = service.get('required_data', [])
+        return all(bool(req) for req in reqs)
+
+    def _load_ready_file(self):
+        if self._ready is not None:
+            return
+        if os.path.exists(self._ready_file):
+            with open(self._ready_file) as fp:
+                self._ready = set(json.load(fp))
+        else:
+            self._ready = set()
+
+    def _save_ready_file(self):
+        if self._ready is None:
+            return
+        with open(self._ready_file, 'w') as fp:
+            json.dump(list(self._ready), fp)
+
+    def save_ready(self, service_name):
+        """
+        Save an indicator that the given service is now data_ready.
+        """
+        self._load_ready_file()
+        self._ready.add(service_name)
+        self._save_ready_file()
+
+    def save_lost(self, service_name):
+        """
+        Save an indicator that the given service is no longer data_ready.
+        """
+        self._load_ready_file()
+        self._ready.discard(service_name)
+        self._save_ready_file()
+
+    def was_ready(self, service_name):
+        """
+        Determine if the given service was previously data_ready.
+        """
+        self._load_ready_file()
+        return service_name in self._ready
+
+
+class ManagerCallback(object):
+    """
+    Special case of a callback that takes the `ServiceManager` instance
+    in addition to the service name.
+
+    Subclasses should implement `__call__` which should accept three parameters:
+
+        * `manager`       The `ServiceManager` instance
+        * `service_name`  The name of the service it's being triggered for
+        * `event_name`    The name of the event that this callback is handling
+    """
+    def __call__(self, manager, service_name, event_name):
+        raise NotImplementedError()
+
+
+class PortManagerCallback(ManagerCallback):
+    """
+    Callback class that will open or close ports, for use as either
+    a start or stop action.
+    """
+    def __call__(self, manager, service_name, event_name):
+        service = manager.get_service(service_name)
+        new_ports = service.get('ports', [])
+        port_file = os.path.join(hookenv.charm_dir(), '.{}.ports'.format(service_name))
+        if os.path.exists(port_file):
+            with open(port_file) as fp:
+                old_ports = fp.read().split(',')
+            for old_port in old_ports:
+                if bool(old_port):
+                    old_port = int(old_port)
+                    if old_port not in new_ports:
+                        hookenv.close_port(old_port)
+        with open(port_file, 'w') as fp:
+            fp.write(','.join(str(port) for port in new_ports))
+        for port in new_ports:
+            if event_name == 'start':
+                hookenv.open_port(port)
+            elif event_name == 'stop':
+                hookenv.close_port(port)
+
+
+def service_stop(service_name):
+    """
+    Wrapper around host.service_stop to prevent spurious "unknown service"
+    messages in the logs.
+    """
+    if host.service_running(service_name):
+        host.service_stop(service_name)
+
+
+def service_restart(service_name):
+    """
+    Wrapper around host.service_restart to prevent spurious "unknown service"
+    messages in the logs.
+    """
+    if host.service_available(service_name):
+        if host.service_running(service_name):
+            host.service_restart(service_name)
+        else:
+            host.service_start(service_name)
+
+
+# Convenience aliases
+open_ports = close_ports = manage_ports = PortManagerCallback()

=== added file 'hooks/charmhelpers/core/services/helpers.py'
--- hooks/charmhelpers/core/services/helpers.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/services/helpers.py	2014-08-13 13:52:53 +0000
@@ -0,0 +1,125 @@
+from charmhelpers.core import hookenv
+from charmhelpers.core import templating
+
+from charmhelpers.core.services.base import ManagerCallback
+
+
+__all__ = ['RelationContext', 'TemplateCallback',
+           'render_template', 'template']
+
+
+class RelationContext(dict):
+    """
+    Base class for a context generator that gets relation data from juju.
+
+    Subclasses must provide the attributes `name`, which is the name of the
+    interface of interest, `interface`, which is the type of the interface of
+    interest, and `required_keys`, which is the set of keys required for the
+    relation to be considered complete.  The data for all interfaces matching
+    the `name` attribute that are complete will used to populate the dictionary
+    values (see `get_data`, below).
+
+    The generated context will be namespaced under the interface type, to prevent
+    potential naming conflicts.
+    """
+    name = None
+    interface = None
+    required_keys = []
+
+    def __init__(self, *args, **kwargs):
+        super(RelationContext, self).__init__(*args, **kwargs)
+        self.get_data()
+
+    def __bool__(self):
+        """
+        Returns True if all of the required_keys are available.
+        """
+        return self.is_ready()
+
+    __nonzero__ = __bool__
+
+    def __repr__(self):
+        return super(RelationContext, self).__repr__()
+
+    def is_ready(self):
+        """
+        Returns True if all of the `required_keys` are available from any units.
+        """
+        ready = len(self.get(self.name, [])) > 0
+        if not ready:
+            hookenv.log('Incomplete relation: {}'.format(self.__class__.__name__), hookenv.DEBUG)
+        return ready
+
+    def _is_ready(self, unit_data):
+        """
+        Helper method that tests a set of relation data and returns True if
+        all of the `required_keys` are present.
+        """
+        return set(unit_data.keys()).issuperset(set(self.required_keys))
+
+    def get_data(self):
+        """
+        Retrieve the relation data for each unit involved in a relation and,
+        if complete, store it in a list under `self[self.name]`.  This
+        is automatically called when the RelationContext is instantiated.
+
+        The units are sorted lexographically first by the service ID, then by
+        the unit ID.  Thus, if an interface has two other services, 'db:1'
+        and 'db:2', with 'db:1' having two units, 'wordpress/0' and 'wordpress/1',
+        and 'db:2' having one unit, 'mediawiki/0', all of which have a complete
+        set of data, the relation data for the units will be stored in the
+        order: 'wordpress/0', 'wordpress/1', 'mediawiki/0'.
+
+        If you only care about a single unit on the relation, you can just
+        access it as `{{ interface[0]['key'] }}`.  However, if you can at all
+        support multiple units on a relation, you should iterate over the list,
+        like::
+
+            {% for unit in interface -%}
+                {{ unit['key'] }}{% if not loop.last %},{% endif %}
+            {%- endfor %}
+
+        Note that since all sets of relation data from all related services and
+        units are in a single list, if you need to know which service or unit a
+        set of data came from, you'll need to extend this class to preserve
+        that information.
+        """
+        if not hookenv.relation_ids(self.name):
+            return
+
+        ns = self.setdefault(self.name, [])
+        for rid in sorted(hookenv.relation_ids(self.name)):
+            for unit in sorted(hookenv.related_units(rid)):
+                reldata = hookenv.relation_get(rid=rid, unit=unit)
+                if self._is_ready(reldata):
+                    ns.append(reldata)
+
+    def provide_data(self):
+        """
+        Return data to be relation_set for this interface.
+        """
+        return {}
+
+
+class TemplateCallback(ManagerCallback):
+    """
+    Callback class that will render a template, for use as a ready action.
+    """
+    def __init__(self, source, target, owner='root', group='root', perms=0444):
+        self.source = source
+        self.target = target
+        self.owner = owner
+        self.group = group
+        self.perms = perms
+
+    def __call__(self, manager, service_name, event_name):
+        service = manager.get_service(service_name)
+        context = {}
+        for ctx in service.get('required_data', []):
+            context.update(ctx)
+        templating.render(self.source, self.target, context,
+                          self.owner, self.group, self.perms)
+
+
+# Convenience aliases for templates
+render_template = template = TemplateCallback

=== added file 'hooks/charmhelpers/core/templating.py'
--- hooks/charmhelpers/core/templating.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/templating.py	2014-08-13 13:52:53 +0000
@@ -0,0 +1,51 @@
+import os
+
+from charmhelpers.core import host
+from charmhelpers.core import hookenv
+
+
+def render(source, target, context, owner='root', group='root', perms=0444, templates_dir=None):
+    """
+    Render a template.
+
+    The `source` path, if not absolute, is relative to the `templates_dir`.
+
+    The `target` path should be absolute.
+
+    The context should be a dict containing the values to be replaced in the
+    template.
+
+    The `owner`, `group`, and `perms` options will be passed to `write_file`.
+
+    If omitted, `templates_dir` defaults to the `templates` folder in the charm.
+
+    Note: Using this requires python-jinja2; if it is not installed, calling
+    this will attempt to use charmhelpers.fetch.apt_install to install it.
+    """
+    try:
+        from jinja2 import FileSystemLoader, Environment, exceptions
+    except ImportError:
+        try:
+            from charmhelpers.fetch import apt_install
+        except ImportError:
+            hookenv.log('Could not import jinja2, and could not import '
+                        'charmhelpers.fetch to install it',
+                        level=hookenv.ERROR)
+            raise
+        apt_install('python-jinja2', fatal=True)
+        from jinja2 import FileSystemLoader, Environment, exceptions
+
+    if templates_dir is None:
+        templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
+    loader = Environment(loader=FileSystemLoader(templates_dir))
+    try:
+        source = source
+        template = loader.get_template(source)
+    except exceptions.TemplateNotFound as e:
+        hookenv.log('Could not load template %s from %s.' %
+                    (source, templates_dir),
+                    level=hookenv.ERROR)
+        raise e
+    content = template.render(context)
+    host.mkdir(os.path.dirname(target))
+    host.write_file(target, content, owner, group, perms)

=== modified file 'hooks/charmhelpers/fetch/__init__.py'
--- hooks/charmhelpers/fetch/__init__.py	2014-07-10 18:04:35 +0000
+++ hooks/charmhelpers/fetch/__init__.py	2014-08-13 13:52:53 +0000
@@ -122,6 +122,7 @@
     # Tell apt to build an in-memory cache to prevent race conditions (if
     # another process is already building the cache).
     apt_pkg.config.set("Dir::Cache::pkgcache", "")
+    apt_pkg.config.set("Dir::Cache::srcpkgcache", "")
 
     cache = apt_pkg.Cache()
     _pkgs = []

=== modified file 'tests/charmhelpers/contrib/amulet/deployment.py'
--- tests/charmhelpers/contrib/amulet/deployment.py	2014-07-28 11:54:32 +0000
+++ tests/charmhelpers/contrib/amulet/deployment.py	2014-08-13 13:52:53 +0000
@@ -1,9 +1,14 @@
 import amulet
 
+import os
+
 
 class AmuletDeployment(object):
-    """This class provides generic Amulet deployment and test runner
-       methods."""
+    """Amulet deployment.
+
+       This class provides generic Amulet deployment and test runner
+       methods.
+       """
 
     def __init__(self, series=None):
         """Initialize the deployment environment."""
@@ -16,11 +21,19 @@
             self.d = amulet.Deployment()
 
     def _add_services(self, this_service, other_services):
-        """Add services to the deployment where this_service is the local charm
+        """Add services.
+
+           Add services to the deployment where this_service is the local charm
            that we're focused on testing and other_services are the other
-           charms that come from the charm store."""
+           charms that come from the charm store.
+           """
         name, units = range(2)
-        self.this_service = this_service[name]
+
+        if this_service[name] != os.path.basename(os.getcwd()):
+            s = this_service[name]
+            msg = "The charm's root directory name needs to be {}".format(s)
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
         self.d.add(this_service[name], units=this_service[units])
 
         for svc in other_services:
@@ -45,10 +58,10 @@
         """Deploy environment and wait for all hooks to finish executing."""
         try:
             self.d.setup()
-            self.d.sentry.wait()
+            self.d.sentry.wait(timeout=900)
         except amulet.helpers.TimeoutError:
             amulet.raise_status(amulet.FAIL, msg="Deployment timed out")
-        except:
+        except Exception:
             raise
 
     def run_tests(self):

=== modified file 'tests/charmhelpers/contrib/amulet/utils.py'
--- tests/charmhelpers/contrib/amulet/utils.py	2014-07-28 11:54:32 +0000
+++ tests/charmhelpers/contrib/amulet/utils.py	2014-08-13 13:52:53 +0000
@@ -3,12 +3,15 @@
 import logging
 import re
 import sys
-from time import sleep
+import time
 
 
 class AmuletUtils(object):
-    """This class provides common utility functions that are used by Amulet
-       tests."""
+    """Amulet utilities.
+
+       This class provides common utility functions that are used by Amulet
+       tests.
+       """
 
     def __init__(self, log_level=logging.ERROR):
         self.log = self.get_logger(level=log_level)
@@ -17,8 +20,8 @@
         """Get a logger object that will log to stdout."""
         log = logging
         logger = log.getLogger(name)
-        fmt = \
-            log.Formatter("%(asctime)s %(funcName)s %(levelname)s: %(message)s")
+        fmt = log.Formatter("%(asctime)s %(funcName)s "
+                            "%(levelname)s: %(message)s")
 
         handler = log.StreamHandler(stream=sys.stdout)
         handler.setLevel(level)
@@ -38,7 +41,7 @@
     def valid_url(self, url):
         p = re.compile(
             r'^(?:http|ftp)s?://'
-            r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # flake8: noqa
+            r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'  # noqa
             r'localhost|'
             r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'
             r'(?::\d+)?'
@@ -50,8 +53,11 @@
             return False
 
     def validate_services(self, commands):
-        """Verify the specified services are running on the corresponding
-           service units."""
+        """Validate services.
+
+           Verify the specified services are running on the corresponding
+           service units.
+           """
         for k, v in commands.iteritems():
             for cmd in v:
                 output, code = k.run(cmd)
@@ -66,9 +72,13 @@
         config.readfp(io.StringIO(file_contents))
         return config
 
-    def validate_config_data(self, sentry_unit, config_file, section, expected):
-        """Verify that the specified section of the config file contains
-           the expected option key:value pairs."""
+    def validate_config_data(self, sentry_unit, config_file, section,
+                             expected):
+        """Validate config file data.
+
+           Verify that the specified section of the config file contains
+           the expected option key:value pairs.
+           """
         config = self._get_config(sentry_unit, config_file)
 
         if section != 'DEFAULT' and not config.has_section(section):
@@ -78,20 +88,23 @@
             if not config.has_option(section, k):
                 return "section [{}] is missing option {}".format(section, k)
             if config.get(section, k) != expected[k]:
-                return "section [{}] {}:{} != expected {}:{}".format(section,
-                       k, config.get(section, k), k, expected[k])
+                return "section [{}] {}:{} != expected {}:{}".format(
+                       section, k, config.get(section, k), k, expected[k])
         return None
 
     def _validate_dict_data(self, expected, actual):
-        """Compare expected dictionary data vs actual dictionary data.
+        """Validate dictionary data.
+
+           Compare expected dictionary data vs actual dictionary data.
            The values in the 'expected' dictionary can be strings, bools, ints,
            longs, or can be a function that evaluate a variable and returns a
-           bool."""
+           bool.
+           """
         for k, v in expected.iteritems():
             if k in actual:
-                if isinstance(v, basestring) or \
-                   isinstance(v, bool) or \
-                   isinstance(v, (int, long)):
+                if (isinstance(v, basestring) or
+                        isinstance(v, bool) or
+                        isinstance(v, (int, long))):
                     if v != actual[k]:
                         return "{}:{}".format(k, actual[k])
                 elif not v(actual[k]):
@@ -114,7 +127,7 @@
         return None
 
     def not_null(self, string):
-        if string != None:
+        if string is not None:
             return True
         else:
             return False
@@ -128,9 +141,12 @@
         return sentry_unit.directory_stat(directory)['mtime']
 
     def _get_proc_start_time(self, sentry_unit, service, pgrep_full=False):
-        """Determine start time of the process based on the last modification
+        """Get process' start time.
+
+           Determine start time of the process based on the last modification
            time of the /proc/pid directory. If pgrep_full is True, the process
-           name is matched against the full command line."""
+           name is matched against the full command line.
+           """
         if pgrep_full:
             cmd = 'pgrep -o -f {}'.format(service)
         else:
@@ -139,13 +155,16 @@
         return self._get_dir_mtime(sentry_unit, proc_dir)
 
     def service_restarted(self, sentry_unit, service, filename,
-                          pgrep_full=False):
-        """Compare a service's start time vs a file's last modification time
+                          pgrep_full=False, sleep_time=20):
+        """Check if service was restarted.
+
+           Compare a service's start time vs a file's last modification time
            (such as a config file for that service) to determine if the service
-           has been restarted."""
-        sleep(10)
-        if self._get_proc_start_time(sentry_unit, service, pgrep_full) >= \
-           self._get_file_mtime(sentry_unit, filename):
+           has been restarted.
+           """
+        time.sleep(sleep_time)
+        if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >=
+                self._get_file_mtime(sentry_unit, filename)):
             return True
         else:
             return False

=== modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py'
--- tests/charmhelpers/contrib/openstack/amulet/deployment.py	2014-07-28 11:54:32 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/deployment.py	2014-08-13 13:52:53 +0000
@@ -4,8 +4,11 @@
 
 
 class OpenStackAmuletDeployment(AmuletDeployment):
-    """This class inherits from AmuletDeployment and has additional support
-       that is specifically for use by OpenStack charms."""
+    """OpenStack amulet deployment.
+
+       This class inherits from AmuletDeployment and has additional support
+       that is specifically for use by OpenStack charms.
+       """
 
     def __init__(self, series=None, openstack=None, source=None):
         """Initialize the deployment environment."""
@@ -40,11 +43,14 @@
             self.d.configure(service, config)
 
     def _get_openstack_release(self):
-        """Return an integer representing the enum value of the openstack
-           release."""
-        self.precise_essex, self.precise_folsom, self.precise_grizzly, \
-            self.precise_havana, self.precise_icehouse, \
-            self.trusty_icehouse = range(6)
+        """Get openstack release.
+
+           Return an integer representing the enum value of the openstack
+           release.
+           """
+        (self.precise_essex, self.precise_folsom, self.precise_grizzly,
+         self.precise_havana, self.precise_icehouse,
+         self.trusty_icehouse) = range(6)
         releases = {
             ('precise', None): self.precise_essex,
             ('precise', 'cloud:precise-folsom'): self.precise_folsom,

=== modified file 'tests/charmhelpers/contrib/openstack/amulet/utils.py'
--- tests/charmhelpers/contrib/openstack/amulet/utils.py	2014-07-28 11:54:32 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/utils.py	2014-08-13 13:52:53 +0000
@@ -16,8 +16,11 @@
 
 
 class OpenStackAmuletUtils(AmuletUtils):
-    """This class inherits from AmuletUtils and has additional support
-       that is specifically for use by OpenStack charms."""
+    """OpenStack amulet utilities.
+
+       This class inherits from AmuletUtils and has additional support
+       that is specifically for use by OpenStack charms.
+       """
 
     def __init__(self, log_level=ERROR):
         """Initialize the deployment environment."""
@@ -25,13 +28,17 @@
 
     def validate_endpoint_data(self, endpoints, admin_port, internal_port,
                                public_port, expected):
-        """Validate actual endpoint data vs expected endpoint data. The ports
-           are used to find the matching endpoint."""
+        """Validate endpoint data.
+
+           Validate actual endpoint data vs expected endpoint data. The ports
+           are used to find the matching endpoint.
+           """
         found = False
         for ep in endpoints:
             self.log.debug('endpoint: {}'.format(repr(ep)))
-            if admin_port in ep.adminurl and internal_port in ep.internalurl \
-               and public_port in ep.publicurl:
+            if (admin_port in ep.adminurl and
+                    internal_port in ep.internalurl and
+                    public_port in ep.publicurl):
                 found = True
                 actual = {'id': ep.id,
                           'region': ep.region,
@@ -47,8 +54,11 @@
             return 'endpoint not found'
 
     def validate_svc_catalog_endpoint_data(self, expected, actual):
-        """Validate a list of actual service catalog endpoints vs a list of
-           expected service catalog endpoints."""
+        """Validate service catalog endpoint data.
+
+           Validate a list of actual service catalog endpoints vs a list of
+           expected service catalog endpoints.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for k, v in expected.iteritems():
             if k in actual:
@@ -60,8 +70,11 @@
         return ret
 
     def validate_tenant_data(self, expected, actual):
-        """Validate a list of actual tenant data vs list of expected tenant
-           data."""
+        """Validate tenant data.
+
+           Validate a list of actual tenant data vs list of expected tenant
+           data.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -78,8 +91,11 @@
         return ret
 
     def validate_role_data(self, expected, actual):
-        """Validate a list of actual role data vs a list of expected role
-           data."""
+        """Validate role data.
+
+           Validate a list of actual role data vs a list of expected role
+           data.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -95,8 +111,11 @@
         return ret
 
     def validate_user_data(self, expected, actual):
-        """Validate a list of actual user data vs a list of expected user
-           data."""
+        """Validate user data.
+
+           Validate a list of actual user data vs a list of expected user
+           data.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -114,21 +133,24 @@
         return ret
 
     def validate_flavor_data(self, expected, actual):
-        """Validate a list of actual flavors vs a list of expected flavors."""
+        """Validate flavor data.
+
+           Validate a list of actual flavors vs a list of expected flavors.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         act = [a.name for a in actual]
         return self._validate_list_data(expected, act)
 
     def tenant_exists(self, keystone, tenant):
-        """Return True if tenant exists"""
+        """Return True if tenant exists."""
         return tenant in [t.name for t in keystone.tenants.list()]
 
     def authenticate_keystone_admin(self, keystone_sentry, user, password,
                                     tenant):
         """Authenticates admin user with the keystone admin endpoint."""
-        service_ip = \
-            keystone_sentry.relation('shared-db',
-                                     'mysql:shared-db')['private-address']
+        unit = keystone_sentry
+        service_ip = unit.relation('shared-db',
+                                   'mysql:shared-db')['private-address']
         ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8'))
         return keystone_client.Client(username=user, password=password,
                                       tenant_name=tenant, auth_url=ep)
@@ -177,12 +199,40 @@
             image = glance.images.create(name=image_name, is_public=True,
                                          disk_format='qcow2',
                                          container_format='bare', data=f)
+        count = 1
+        status = image.status
+        while status != 'active' and count < 10:
+            time.sleep(3)
+            image = glance.images.get(image.id)
+            status = image.status
+            self.log.debug('image status: {}'.format(status))
+            count += 1
+
+        if status != 'active':
+            self.log.error('image creation timed out')
+            return None
+
         return image
 
     def delete_image(self, glance, image):
         """Delete the specified image."""
+        num_before = len(list(glance.images.list()))
         glance.images.delete(image)
 
+        count = 1
+        num_after = len(list(glance.images.list()))
+        while num_after != (num_before - 1) and count < 10:
+            time.sleep(3)
+            num_after = len(list(glance.images.list()))
+            self.log.debug('number of images: {}'.format(num_after))
+            count += 1
+
+        if num_after != (num_before - 1):
+            self.log.error('image deletion timed out')
+            return False
+
+        return True
+
     def create_instance(self, nova, image_name, instance_name, flavor):
         """Create the specified instance."""
         image = nova.images.find(name=image_name)
@@ -199,11 +249,27 @@
             self.log.debug('instance status: {}'.format(status))
             count += 1
 
-        if status == 'BUILD':
+        if status != 'ACTIVE':
+            self.log.error('instance creation timed out')
             return None
 
         return instance
 
     def delete_instance(self, nova, instance):
         """Delete the specified instance."""
+        num_before = len(list(nova.servers.list()))
         nova.servers.delete(instance)
+
+        count = 1
+        num_after = len(list(nova.servers.list()))
+        while num_after != (num_before - 1) and count < 10:
+            time.sleep(3)
+            num_after = len(list(nova.servers.list()))
+            self.log.debug('number of instances: {}'.format(num_after))
+            count += 1
+
+        if num_after != (num_before - 1):
+            self.log.error('instance deletion timed out')
+            return False
+
+        return True