Merge lp:~1chb1n/charms/trusty/ceilometer/next-amulet-kilo into lp:~openstack-charmers-archive/charms/trusty/ceilometer/next

[Ryan Beisner]

Update amulet tests for Kilo; Prep for Wily, Liberty; sync tests/charmhelpers; sync hooks/charmhelpers.

Update amulet tests for Kilo; Prep for Wily, Liberty; sync tests/charmhelpers; sync hooks/charmhelpers.

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #5349 ceilometer-next for 1chb1n mp261850
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/5349/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #4981 ceilometer-next for 1chb1n mp261850
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/4981/

[lp:~1chb1n/charms/trusty/ceilometer/next-amulet-kilo]

89. By Ryan Beisner on 2015-06-12

update Makefile

90. By Ryan Beisner on 2015-06-12

update categories to tags in metadata.yaml

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #4982 ceilometer-next for 1chb1n mp261850
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/4982/

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #5350 ceilometer-next for 1chb1n mp261850
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/5350/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #4591 ceilometer-next for 1chb1n mp261850
    AMULET FAIL: amulet-test failed

AMULET Results (max last 2 lines):
make: *** [test] Error 1
ERROR:root:Make target returned non-zero.

Full amulet test output: http://paste.ubuntu.com/11702550/
Build: http://10.245.162.77:8080/job/charm_amulet_test/4591/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #4592 ceilometer-next for 1chb1n mp261850
    AMULET FAIL: amulet-test failed

AMULET Results (max last 2 lines):
make: *** [test] Error 1
ERROR:root:Make target returned non-zero.

Full amulet test output: http://paste.ubuntu.com/11702555/
Build: http://10.245.162.77:8080/job/charm_amulet_test/4592/

[lp:~1chb1n/charms/trusty/ceilometer/next-amulet-kilo]

91. By Ryan Beisner on 2015-06-12

fix dependency package name typo

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #5351 ceilometer-next for 1chb1n mp261850
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/5351/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #4983 ceilometer-next for 1chb1n mp261850
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/4983/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #4594 ceilometer-next for 1chb1n mp261850
    AMULET OK: passed

Build: http://10.245.162.77:8080/job/charm_amulet_test/4594/

[Corey Bryant (corey.bryant)]

review: Approve

=== modified file 'Makefile'
--- Makefile	2015-04-16 21:31:59 +0000
+++ Makefile	2015-06-12 17:20:17 +0000
@@ -2,7 +2,8 @@
 PYTHON := /usr/bin/env python
 
 lint:
-	@flake8 --exclude hooks/charmhelpers hooks unit_tests tests
+	@flake8 --exclude hooks/charmhelpers,tests/charmhelpers \
+        hooks unit_tests tests
 	@charm proof
 
 unit_test:

=== modified file 'hooks/charmhelpers/contrib/hahelpers/cluster.py'
--- hooks/charmhelpers/contrib/hahelpers/cluster.py	2015-06-11 12:57:33 +0000
+++ hooks/charmhelpers/contrib/hahelpers/cluster.py	2015-06-12 17:20:17 +0000
@@ -64,6 +64,10 @@
     pass
 
 
+class CRMDCNotFound(Exception):
+    pass
+
+
 def is_elected_leader(resource):
     """
     Returns True if the charm executing this is the elected cluster leader.
@@ -116,8 +120,9 @@
         status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
         if not isinstance(status, six.text_type):
             status = six.text_type(status, "utf-8")
-    except subprocess.CalledProcessError:
-        return False
+    except subprocess.CalledProcessError as ex:
+        raise CRMDCNotFound(str(ex))
+
     current_dc = ''
     for line in status.split('\n'):
         if line.startswith('Current DC'):
@@ -125,10 +130,14 @@
             current_dc = line.split(':')[1].split()[0]
     if current_dc == get_unit_hostname():
         return True
+    elif current_dc == 'NONE':
+        raise CRMDCNotFound('Current DC: NONE')
+
     return False
 
 
-@retry_on_exception(5, base_delay=2, exc_type=CRMResourceNotFound)
+@retry_on_exception(5, base_delay=2,
+                    exc_type=(CRMResourceNotFound, CRMDCNotFound))
 def is_crm_leader(resource, retry=False):
     """
     Returns True if the charm calling this is the elected corosync leader,

=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/deployment.py'
--- hooks/charmhelpers/contrib/openstack/amulet/deployment.py	2015-05-11 07:26:16 +0000
+++ hooks/charmhelpers/contrib/openstack/amulet/deployment.py	2015-06-12 17:20:17 +0000
@@ -110,7 +110,8 @@
         (self.precise_essex, self.precise_folsom, self.precise_grizzly,
          self.precise_havana, self.precise_icehouse,
          self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
-         self.trusty_kilo, self.vivid_kilo) = range(10)
+         self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
+         self.wily_liberty) = range(12)
 
         releases = {
             ('precise', None): self.precise_essex,
@@ -121,8 +122,10 @@
             ('trusty', None): self.trusty_icehouse,
             ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
             ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
+            ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
             ('utopic', None): self.utopic_juno,
-            ('vivid', None): self.vivid_kilo}
+            ('vivid', None): self.vivid_kilo,
+            ('wily', None): self.wily_liberty}
         return releases[(self.series, self.openstack)]
 
     def _get_openstack_release_string(self):
@@ -138,6 +141,7 @@
             ('trusty', 'icehouse'),
             ('utopic', 'juno'),
             ('vivid', 'kilo'),
+            ('wily', 'liberty'),
         ])
         if self.openstack:
             os_origin = self.openstack.split(':')[1]

=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/utils.py'
--- hooks/charmhelpers/contrib/openstack/amulet/utils.py	2015-01-26 09:48:14 +0000
+++ hooks/charmhelpers/contrib/openstack/amulet/utils.py	2015-06-12 17:20:17 +0000
@@ -16,15 +16,15 @@
 
 import logging
 import os
+import six
 import time
 import urllib
 
 import glanceclient.v1.client as glance_client
+import heatclient.v1.client as heat_client
 import keystoneclient.v2_0 as keystone_client
 import novaclient.v1_1.client as nova_client
 
-import six
-
 from charmhelpers.contrib.amulet.utils import (
     AmuletUtils
 )
@@ -37,7 +37,7 @@
     """OpenStack amulet utilities.
 
        This class inherits from AmuletUtils and has additional support
-       that is specifically for use by OpenStack charms.
+       that is specifically for use by OpenStack charm tests.
        """
 
     def __init__(self, log_level=ERROR):
@@ -51,6 +51,8 @@
            Validate actual endpoint data vs expected endpoint data. The ports
            are used to find the matching endpoint.
            """
+        self.log.debug('Validating endpoint data...')
+        self.log.debug('actual: {}'.format(repr(endpoints)))
         found = False
         for ep in endpoints:
             self.log.debug('endpoint: {}'.format(repr(ep)))
@@ -77,6 +79,7 @@
            Validate a list of actual service catalog endpoints vs a list of
            expected service catalog endpoints.
            """
+        self.log.debug('Validating service catalog endpoint data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for k, v in six.iteritems(expected):
             if k in actual:
@@ -93,6 +96,7 @@
            Validate a list of actual tenant data vs list of expected tenant
            data.
            """
+        self.log.debug('Validating tenant data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -114,6 +118,7 @@
            Validate a list of actual role data vs a list of expected role
            data.
            """
+        self.log.debug('Validating role data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -134,6 +139,7 @@
            Validate a list of actual user data vs a list of expected user
            data.
            """
+        self.log.debug('Validating user data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -155,17 +161,20 @@
 
            Validate a list of actual flavors vs a list of expected flavors.
            """
+        self.log.debug('Validating flavor data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         act = [a.name for a in actual]
         return self._validate_list_data(expected, act)
 
     def tenant_exists(self, keystone, tenant):
         """Return True if tenant exists."""
+        self.log.debug('Checking if tenant exists ({})...'.format(tenant))
         return tenant in [t.name for t in keystone.tenants.list()]
 
     def authenticate_keystone_admin(self, keystone_sentry, user, password,
                                     tenant):
         """Authenticates admin user with the keystone admin endpoint."""
+        self.log.debug('Authenticating keystone admin...')
         unit = keystone_sentry
         service_ip = unit.relation('shared-db',
                                    'mysql:shared-db')['private-address']
@@ -175,6 +184,7 @@
 
     def authenticate_keystone_user(self, keystone, user, password, tenant):
         """Authenticates a regular user with the keystone public endpoint."""
+        self.log.debug('Authenticating keystone user ({})...'.format(user))
         ep = keystone.service_catalog.url_for(service_type='identity',
                                               endpoint_type='publicURL')
         return keystone_client.Client(username=user, password=password,
@@ -182,12 +192,21 @@
 
     def authenticate_glance_admin(self, keystone):
         """Authenticates admin user with glance."""
+        self.log.debug('Authenticating glance admin...')
         ep = keystone.service_catalog.url_for(service_type='image',
                                               endpoint_type='adminURL')
         return glance_client.Client(ep, token=keystone.auth_token)
 
+    def authenticate_heat_admin(self, keystone):
+        """Authenticates the admin user with heat."""
+        self.log.debug('Authenticating heat admin...')
+        ep = keystone.service_catalog.url_for(service_type='orchestration',
+                                              endpoint_type='publicURL')
+        return heat_client.Client(endpoint=ep, token=keystone.auth_token)
+
     def authenticate_nova_user(self, keystone, user, password, tenant):
         """Authenticates a regular user with nova-api."""
+        self.log.debug('Authenticating nova user ({})...'.format(user))
         ep = keystone.service_catalog.url_for(service_type='identity',
                                               endpoint_type='publicURL')
         return nova_client.Client(username=user, api_key=password,
@@ -195,6 +214,7 @@
 
     def create_cirros_image(self, glance, image_name):
         """Download the latest cirros image and upload it to glance."""
+        self.log.debug('Creating glance image ({})...'.format(image_name))
         http_proxy = os.getenv('AMULET_HTTP_PROXY')
         self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
         if http_proxy:
@@ -235,6 +255,11 @@
 
     def delete_image(self, glance, image):
         """Delete the specified image."""
+
+        # /!\ DEPRECATION WARNING
+        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+                      'delete_resource instead of delete_image.')
+        self.log.debug('Deleting glance image ({})...'.format(image))
         num_before = len(list(glance.images.list()))
         glance.images.delete(image)
 
@@ -254,6 +279,8 @@
 
     def create_instance(self, nova, image_name, instance_name, flavor):
         """Create the specified instance."""
+        self.log.debug('Creating instance '
+                       '({}|{}|{})'.format(instance_name, image_name, flavor))
         image = nova.images.find(name=image_name)
         flavor = nova.flavors.find(name=flavor)
         instance = nova.servers.create(name=instance_name, image=image,
@@ -276,6 +303,11 @@
 
     def delete_instance(self, nova, instance):
         """Delete the specified instance."""
+
+        # /!\ DEPRECATION WARNING
+        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+                      'delete_resource instead of delete_instance.')
+        self.log.debug('Deleting instance ({})...'.format(instance))
         num_before = len(list(nova.servers.list()))
         nova.servers.delete(instance)
 
@@ -292,3 +324,90 @@
             return False
 
         return True
+
+    def create_or_get_keypair(self, nova, keypair_name="testkey"):
+        """Create a new keypair, or return pointer if it already exists."""
+        try:
+            _keypair = nova.keypairs.get(keypair_name)
+            self.log.debug('Keypair ({}) already exists, '
+                           'using it.'.format(keypair_name))
+            return _keypair
+        except:
+            self.log.debug('Keypair ({}) does not exist, '
+                           'creating it.'.format(keypair_name))
+
+        _keypair = nova.keypairs.create(name=keypair_name)
+        return _keypair
+
+    def delete_resource(self, resource, resource_id,
+                        msg="resource", max_wait=120):
+        """Delete one openstack resource, such as one instance, keypair,
+        image, volume, stack, etc., and confirm deletion within max wait time.
+
+        :param resource: pointer to os resource type, ex:glance_client.images
+        :param resource_id: unique name or id for the openstack resource
+        :param msg: text to identify purpose in logging
+        :param max_wait: maximum wait time in seconds
+        :returns: True if successful, otherwise False
+        """
+        num_before = len(list(resource.list()))
+        resource.delete(resource_id)
+
+        tries = 0
+        num_after = len(list(resource.list()))
+        while num_after != (num_before - 1) and tries < (max_wait / 4):
+            self.log.debug('{} delete check: '
+                           '{} [{}:{}] {}'.format(msg, tries,
+                                                  num_before,
+                                                  num_after,
+                                                  resource_id))
+            time.sleep(4)
+            num_after = len(list(resource.list()))
+            tries += 1
+
+        self.log.debug('{}:  expected, actual count = {}, '
+                       '{}'.format(msg, num_before - 1, num_after))
+
+        if num_after == (num_before - 1):
+            return True
+        else:
+            self.log.error('{} delete timed out'.format(msg))
+            return False
+
+    def resource_reaches_status(self, resource, resource_id,
+                                expected_stat='available',
+                                msg='resource', max_wait=120):
+        """Wait for an openstack resources status to reach an
+           expected status within a specified time.  Useful to confirm that
+           nova instances, cinder vols, snapshots, glance images, heat stacks
+           and other resources eventually reach the expected status.
+
+        :param resource: pointer to os resource type, ex: heat_client.stacks
+        :param resource_id: unique id for the openstack resource
+        :param expected_stat: status to expect resource to reach
+        :param msg: text to identify purpose in logging
+        :param max_wait: maximum wait time in seconds
+        :returns: True if successful, False if status is not reached
+        """
+
+        tries = 0
+        resource_stat = resource.get(resource_id).status
+        while resource_stat != expected_stat and tries < (max_wait / 4):
+            self.log.debug('{} status check: '
+                           '{} [{}:{}] {}'.format(msg, tries,
+                                                  resource_stat,
+                                                  expected_stat,
+                                                  resource_id))
+            time.sleep(4)
+            resource_stat = resource.get(resource_id).status
+            tries += 1
+
+        self.log.debug('{}:  expected, actual status = {}, '
+                       '{}'.format(msg, resource_stat, expected_stat))
+
+        if resource_stat == expected_stat:
+            return True
+        else:
+            self.log.debug('{} never reached expected status: '
+                           '{}'.format(resource_id, expected_stat))
+            return False

=== modified file 'hooks/charmhelpers/contrib/openstack/context.py'
--- hooks/charmhelpers/contrib/openstack/context.py	2015-05-11 07:26:16 +0000
+++ hooks/charmhelpers/contrib/openstack/context.py	2015-06-12 17:20:17 +0000
@@ -240,7 +240,7 @@
         if self.relation_prefix:
             password_setting = self.relation_prefix + '_password'
 
-        for rid in relation_ids('shared-db'):
+        for rid in relation_ids(self.interfaces[0]):
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
                 host = rdata.get('db_host')

=== modified file 'hooks/charmhelpers/contrib/openstack/neutron.py'
--- hooks/charmhelpers/contrib/openstack/neutron.py	2015-06-11 12:57:33 +0000
+++ hooks/charmhelpers/contrib/openstack/neutron.py	2015-06-12 17:20:17 +0000
@@ -172,14 +172,16 @@
             'services': ['calico-felix',
                          'bird',
                          'neutron-dhcp-agent',
-                         'nova-api-metadata'],
+                         'nova-api-metadata',
+                         'etcd'],
             'packages': [[headers_package()] + determine_dkms_package(),
                          ['calico-compute',
                           'bird',
                           'neutron-dhcp-agent',
-                          'nova-api-metadata']],
-            'server_packages': ['neutron-server', 'calico-control'],
-            'server_services': ['neutron-server']
+                          'nova-api-metadata',
+                          'etcd']],
+            'server_packages': ['neutron-server', 'calico-control', 'etcd'],
+            'server_services': ['neutron-server', 'etcd']
         },
         'vsp': {
             'config': '/etc/neutron/plugins/nuage/nuage_plugin.ini',

=== modified file 'hooks/charmhelpers/contrib/openstack/utils.py'
--- hooks/charmhelpers/contrib/openstack/utils.py	2015-06-11 12:57:33 +0000
+++ hooks/charmhelpers/contrib/openstack/utils.py	2015-06-12 17:20:17 +0000
@@ -79,6 +79,7 @@
     ('trusty', 'icehouse'),
     ('utopic', 'juno'),
     ('vivid', 'kilo'),
+    ('wily', 'liberty'),
 ])
 
 
@@ -91,6 +92,7 @@
     ('2014.1', 'icehouse'),
     ('2014.2', 'juno'),
     ('2015.1', 'kilo'),
+    ('2015.2', 'liberty'),
 ])
 
 # The ugly duckling
@@ -113,6 +115,7 @@
     ('2.2.0', 'juno'),
     ('2.2.1', 'kilo'),
     ('2.2.2', 'kilo'),
+    ('2.3.0', 'liberty'),
 ])
 
 DEFAULT_LOOPBACK_SIZE = '5G'
@@ -321,6 +324,9 @@
             'kilo': 'trusty-updates/kilo',
             'kilo/updates': 'trusty-updates/kilo',
             'kilo/proposed': 'trusty-proposed/kilo',
+            'liberty': 'trusty-updates/liberty',
+            'liberty/updates': 'trusty-updates/liberty',
+            'liberty/proposed': 'trusty-proposed/liberty',
         }
 
         try:
@@ -641,7 +647,8 @@
         subprocess.check_call(cmd)
     except subprocess.CalledProcessError:
         package = os.path.basename(package_dir)
-        error_out("Error updating {} from global-requirements.txt".format(package))
+        error_out("Error updating {} from "
+                  "global-requirements.txt".format(package))
     os.chdir(orig_dir)
 
 

=== modified file 'hooks/charmhelpers/core/host.py'
--- hooks/charmhelpers/core/host.py	2015-06-11 12:57:33 +0000
+++ hooks/charmhelpers/core/host.py	2015-06-12 17:20:17 +0000
@@ -24,6 +24,7 @@
 import os
 import re
 import pwd
+import glob
 import grp
 import random
 import string
@@ -269,6 +270,21 @@
         return None
 
 
+def path_hash(path):
+    """
+    Generate a hash checksum of all files matching 'path'. Standard wildcards
+    like '*' and '?' are supported, see documentation for the 'glob' module for
+    more information.
+
+    :return: dict: A { filename: hash } dictionary for all matched files.
+                   Empty if none found.
+    """
+    return {
+        filename: file_hash(filename)
+        for filename in glob.iglob(path)
+    }
+
+
 def check_hash(path, checksum, hash_type='md5'):
     """
     Validate a file using a cryptographic checksum.
@@ -296,23 +312,25 @@
 
         @restart_on_change({
             '/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ]
+            '/etc/apache/sites-enabled/*': [ 'apache2' ]
             })
-        def ceph_client_changed():
+        def config_changed():
             pass  # your code here
 
     In this example, the cinder-api and cinder-volume services
     would be restarted if /etc/ceph/ceph.conf is changed by the
-    ceph_client_changed function.
+    ceph_client_changed function. The apache2 service would be
+    restarted if any file matching the pattern got changed, created
+    or removed. Standard wildcards are supported, see documentation
+    for the 'glob' module for more information.
     """
     def wrap(f):
         def wrapped_f(*args, **kwargs):
-            checksums = {}
-            for path in restart_map:
-                checksums[path] = file_hash(path)
+            checksums = {path: path_hash(path) for path in restart_map}
             f(*args, **kwargs)
             restarts = []
             for path in restart_map:
-                if checksums[path] != file_hash(path):
+                if path_hash(path) != checksums[path]:
                     restarts += restart_map[path]
             services_list = list(OrderedDict.fromkeys(restarts))
             if not stopstart:

=== modified file 'metadata.yaml'
--- metadata.yaml	2015-01-09 16:27:03 +0000
+++ metadata.yaml	2015-06-12 17:20:17 +0000
@@ -8,7 +8,7 @@
  framework should be easily expandable to collect for other needs. To that
  effect, Ceilometer should be able to share collected data with a variety
  of consumers.
-categories:
+tags:
   - miscellaneous
   - openstack
 provides:

=== modified file 'tests/00-setup'
--- tests/00-setup	2015-02-17 14:09:16 +0000
+++ tests/00-setup	2015-06-12 17:20:17 +0000
@@ -5,7 +5,6 @@
 sudo add-apt-repository --yes ppa:juju/stable
 sudo apt-get update --yes
 sudo apt-get install --yes python-amulet \
-                           python-neutronclient \
+                           python-distro-info \
                            python-keystoneclient \
-                           python-novaclient \
-                           python-glanceclient
+                           python-ceilometerclient

=== modified file 'tests/019-basic-vivid-kilo' (properties changed: -x to +x)
=== modified file 'tests/README'
--- tests/README	2015-02-17 14:09:16 +0000
+++ tests/README	2015-06-12 17:20:17 +0000
@@ -1,12 +1,42 @@
 This directory provides Amulet tests that focus on verification of
 ceilometer deployments.
 
+test_* methods are called in lexical sort order.
+
+Test name convention to ensure desired test order:
+    1xx service and endpoint checks
+    2xx relation checks
+    3xx config checks
+    4xx functional checks
+    9xx restarts and other final checks
+
 In order to run tests, you'll need charm-tools installed (in addition to
 juju, of course):
     sudo add-apt-repository ppa:juju/stable
     sudo apt-get update
     sudo apt-get install charm-tools
 
+Common uses of ceilometer relations in bundles:
+    - - ceilometer
+      - keystone:identity-service
+    - [ ceilometer, rabbitmq-server ]
+    - [ ceilometer, mongodb ]
+    - [ ceilometer-agent, nova-compute ]
+    - [ ceilometer-agent, ceilometer ]
+
+More detailed relations of ceilometer in a common deployment:
+    relations:
+      amqp:
+      - rabbitmq-server
+      ceilometer-service:
+      - ceilometer-agent
+      cluster:
+      - ceilometer
+      identity-service:
+      - keystone
+      shared-db:
+      - mongodb
+
 If you use a web proxy server to access the web, you'll need to set the
 AMULET_HTTP_PROXY environment variable to the http URL of the proxy server.
 

=== modified file 'tests/basic_deployment.py'
--- tests/basic_deployment.py	2015-06-04 15:59:29 +0000
+++ tests/basic_deployment.py	2015-06-12 17:20:17 +0000
@@ -10,8 +10,8 @@
 
 from charmhelpers.contrib.openstack.amulet.utils import (
     OpenStackAmuletUtils,
-    DEBUG, # flake8: noqa
-    ERROR
+    DEBUG,
+    #ERROR
 )
 
 # Use DEBUG to turn on debug logging
@@ -82,6 +82,10 @@
         self.rabbitmq_sentry = self.d.sentry.unit['rabbitmq-server/0']
         self.mongodb_sentry = self.d.sentry.unit['mongodb/0']
         self.compute_sentry = self.d.sentry.unit['nova-compute/0']
+        u.log.debug('openstack release val: {}'.format(
+            self._get_openstack_release()))
+        u.log.debug('openstack release str: {}'.format(
+            self._get_openstack_release_string()))
 
         # Let things settle a bit before moving forward
         time.sleep(30)
@@ -97,34 +101,32 @@
                                                    endpoint_type='publicURL')
         self.ceil = ceilclient.Client(endpoint=ep, token=self._get_token)
 
-    def test_api_connection(self):
-        """Simple api call to check service is up and responding"""
-        assert(self.ceil.meters.list() == [])
-
-    def test_services(self):
+    def test_100_services(self):
         """Verify the expected services are running on the corresponding
            service units."""
-        ceilometer_cmds = [
-            'status ceilometer-agent-central',
-            'status ceilometer-collector',
-            'status ceilometer-api',
-            'status ceilometer-alarm-evaluator',
-            'status ceilometer-alarm-notifier',
-            'status ceilometer-agent-notification',
+        ceilometer_svcs = [
+            'ceilometer-agent-central',
+            'ceilometer-collector',
+            'ceilometer-api',
+            'ceilometer-alarm-evaluator',
+            'ceilometer-alarm-notifier',
+            'ceilometer-agent-notification',
         ]
-        commands = {
-            self.ceil_sentry: ceilometer_cmds,
-            self.mysql_sentry: ['status mysql'],
-            self.keystone_sentry: ['status keystone'],
-            self.rabbitmq_sentry: ['service rabbitmq-server status'],
-            self.mongodb_sentry: ['status mongodb'],
+        service_names = {
+            self.ceil_sentry: ceilometer_svcs,
+            self.mysql_sentry: ['mysql'],
+            self.keystone_sentry: ['keystone'],
+            self.rabbitmq_sentry: ['rabbitmq-server'],
+            self.mongodb_sentry: ['mongodb'],
         }
-        ret = u.validate_services(commands)
+
+        ret = u.validate_services_by_name(service_names)
         if ret:
             amulet.raise_status(amulet.FAIL, msg=ret)
 
-    def test_ceilometer_identity_relation(self):
+    def test_200_ceilometer_identity_relation(self):
         """Verify the ceilometer to keystone identity-service relation data"""
+        u.log.debug('Checking service catalog endpoint data...')
         unit = self.ceil_sentry
         relation = ['identity-service', 'keystone:identity-service']
         ceil_ip = unit.relation('identity-service',
@@ -146,8 +148,9 @@
             message = u.relation_error('ceilometer identity-service', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_keystone_ceilometer_identity_relation(self):
+    def test_201_keystone_ceilometer_identity_relation(self):
         """Verify the keystone to ceilometer identity-service relation data"""
+        u.log.debug('Checking keystone:ceilometer identity relation data...')
         unit = self.keystone_sentry
         relation = ['identity-service', 'ceilometer:identity-service']
         id_relation = unit.relation('identity-service',
@@ -172,8 +175,10 @@
             message = u.relation_error('keystone identity-service', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_keystone_ceilometer_identity_notes_relation(self):
+    def test_202_keystone_ceilometer_identity_notes_relation(self):
         """Verify ceilometer to keystone identity-notifications relation"""
+        u.log.debug('Checking keystone:ceilometer '
+                    'identity-notifications relation data...')
         unit = self.keystone_sentry
         relation = ['identity-service', 'ceilometer:identity-notifications']
         expected = {
@@ -184,8 +189,9 @@
             message = u.relation_error('keystone identity-notifications', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_ceilometer_amqp_relation(self):
+    def test_203_ceilometer_amqp_relation(self):
         """Verify the ceilometer to rabbitmq-server amqp relation data"""
+        u.log.debug('Checking ceilometer:rabbitmq amqp relation data...')
         unit = self.ceil_sentry
         relation = ['amqp', 'rabbitmq-server:amqp']
         expected = {
@@ -199,8 +205,9 @@
             message = u.relation_error('ceilometer amqp', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_amqp_ceilometer_relation(self):
+    def test_204_amqp_ceilometer_relation(self):
         """Verify the rabbitmq-server to ceilometer amqp relation data"""
+        u.log.debug('Checking rabbitmq:ceilometer amqp relation data...')
         unit = self.rabbitmq_sentry
         relation = ['amqp', 'ceilometer:amqp']
         expected = {
@@ -214,8 +221,9 @@
             message = u.relation_error('rabbitmq amqp', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_ceilometer_to_mongodb_relation(self):
+    def test_205_ceilometer_to_mongodb_relation(self):
         """Verify the ceilometer to mongodb relation data"""
+        u.log.debug('Checking ceilometer:mongodb relation data...')
         unit = self.ceil_sentry
         relation = ['shared-db', 'mongodb:database']
         expected = {
@@ -228,8 +236,9 @@
             message = u.relation_error('ceilometer shared-db', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_mongodb_to_ceilometer_relation(self):
+    def test_206_mongodb_to_ceilometer_relation(self):
         """Verify the mongodb to ceilometer relation data"""
+        u.log.debug('Checking mongodb:ceilometer relation data...')
         unit = self.mongodb_sentry
         relation = ['database', 'ceilometer:shared-db']
         expected = {
@@ -247,38 +256,9 @@
             message = u.relation_error('mongodb database', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_z_restart_on_config_change(self):
-        """Verify that the specified services are restarted when the config
-           is changed.
-
-           Note(coreycb): The method name with the _z_ is a little odd
-           but it forces the test to run last.  It just makes things
-           easier because restarting services requires re-authorization.
-           """
-        conf = '/etc/ceilometer/ceilometer.conf'
-        self.d.configure('ceilometer', {'debug': 'True'})
-        services = [
-            'ceilometer-agent-central',
-            'ceilometer-collector',
-            'ceilometer-api',
-            'ceilometer-alarm-evaluator',
-            'ceilometer-alarm-notifier',
-            'ceilometer-agent-notification',
-        ]
-
-        time = 20
-        for s in services:
-            if not u.service_restarted(self.ceil_sentry, s, conf,
-                                       pgrep_full=True, sleep_time=time):
-                self.d.configure('ceilometer', {'debug': 'False'})
-                msg = "service {} didn't restart after config change".format(s)
-                amulet.raise_status(amulet.FAIL, msg=msg)
-            time = 0
-
-        self.d.configure('ceilometer', {'debug': 'False'})
-
-    def test_ceilometer_config(self):
+    def test_300_ceilometer_config(self):
         """Verify the data in the ceilometer config file."""
+        u.log.debug('Checking ceilometer config file data...')
         unit = self.ceil_sentry
         rabbitmq_relation = self.rabbitmq_sentry.relation('amqp',
                                                           'ceilometer:amqp')
@@ -330,3 +310,50 @@
             if ret:
                 message = "ceilometer config error: {}".format(ret)
                 amulet.raise_status(amulet.FAIL, msg=message)
+
+    def test_400_api_connection(self):
+        """Simple api calls to check service is up and responding"""
+        u.log.debug('Checking api functionality...')
+        assert(self.ceil.samples.list() == [])
+        assert(self.ceil.meters.list() == [])
+
+    def test_900_restart_on_config_change(self):
+        """Verify that the specified services are restarted when the config
+           is changed.
+           """
+        sentry = self.ceil_sentry
+        juju_service = 'ceilometer'
+
+        # Expected default and alternate values
+        set_default = {'debug': 'False'}
+        set_alternate = {'debug': 'True'}
+
+        # Config file affected by juju set config change
+        conf_file = '/etc/ceilometer/ceilometer.conf'
+
+        # Services which are expected to restart upon config change
+        services = [
+            'ceilometer-agent-central',
+            'ceilometer-collector',
+            'ceilometer-api',
+            'ceilometer-alarm-evaluator',
+            'ceilometer-alarm-notifier',
+            'ceilometer-agent-notification',
+        ]
+
+        # Make config change, check for service restarts
+        u.log.debug('Making config change on {}...'.format(juju_service))
+        self.d.configure(juju_service, set_alternate)
+
+        sleep_time = 40
+        for s in services:
+            u.log.debug("Checking that service restarted: {}".format(s))
+            if not u.service_restarted(sentry, s,
+                                       conf_file, sleep_time=sleep_time,
+                                       pgrep_full=True):
+                self.d.configure(juju_service, set_default)
+                msg = "service {} didn't restart after config change".format(s)
+                amulet.raise_status(amulet.FAIL, msg=msg)
+            sleep_time = 0
+
+        self.d.configure(juju_service, set_default)

=== modified file 'tests/charmhelpers/contrib/amulet/utils.py'
--- tests/charmhelpers/contrib/amulet/utils.py	2015-04-23 14:55:32 +0000
+++ tests/charmhelpers/contrib/amulet/utils.py	2015-06-12 17:20:17 +0000
@@ -15,13 +15,15 @@
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import ConfigParser
+import distro_info
 import io
 import logging
+import os
 import re
+import six
 import sys
 import time
-
-import six
+import urlparse
 
 
 class AmuletUtils(object):
@@ -33,6 +35,7 @@
 
     def __init__(self, log_level=logging.ERROR):
         self.log = self.get_logger(level=log_level)
+        self.ubuntu_releases = self.get_ubuntu_releases()
 
     def get_logger(self, name="amulet-logger", level=logging.DEBUG):
         """Get a logger object that will log to stdout."""
@@ -70,12 +73,44 @@
         else:
             return False
 
+    def get_ubuntu_release_from_sentry(self, sentry_unit):
+        """Get Ubuntu release codename from sentry unit.
+
+        :param sentry_unit: amulet sentry/service unit pointer
+        :returns: list of strings - release codename, failure message
+        """
+        msg = None
+        cmd = 'lsb_release -cs'
+        release, code = sentry_unit.run(cmd)
+        if code == 0:
+            self.log.debug('{} lsb_release: {}'.format(
+                sentry_unit.info['unit_name'], release))
+        else:
+            msg = ('{} `{}` returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, release, code))
+        if release not in self.ubuntu_releases:
+            msg = ("Release ({}) not found in Ubuntu releases "
+                   "({})".format(release, self.ubuntu_releases))
+        return release, msg
+
     def validate_services(self, commands):
-        """Validate services.
-
-           Verify the specified services are running on the corresponding
+        """Validate that lists of commands succeed on service units.  Can be
+           used to verify system services are running on the corresponding
            service units.
-           """
+
+        :param commands: dict with sentry keys and arbitrary command list vals
+        :returns: None if successful, Failure string message otherwise
+        """
+        self.log.debug('Checking status of system services...')
+
+        # /!\ DEPRECATION WARNING (beisner):
+        # New and existing tests should be rewritten to use
+        # validate_services_by_name() as it is aware of init systems.
+        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+                      'validate_services_by_name instead of validate_services '
+                      'due to init system differences.')
+
         for k, v in six.iteritems(commands):
             for cmd in v:
                 output, code = k.run(cmd)
@@ -86,6 +121,41 @@
                     return "command `{}` returned {}".format(cmd, str(code))
         return None
 
+    def validate_services_by_name(self, sentry_services):
+        """Validate system service status by service name, automatically
+           detecting init system based on Ubuntu release codename.
+
+        :param sentry_services: dict with sentry keys and svc list values
+        :returns: None if successful, Failure string message otherwise
+        """
+        self.log.debug('Checking status of system services...')
+
+        # Point at which systemd became a thing
+        systemd_switch = self.ubuntu_releases.index('vivid')
+
+        for sentry_unit, services_list in six.iteritems(sentry_services):
+            # Get lsb_release codename from unit
+            release, ret = self.get_ubuntu_release_from_sentry(sentry_unit)
+            if ret:
+                return ret
+
+            for service_name in services_list:
+                if (self.ubuntu_releases.index(release) >= systemd_switch or
+                        service_name == "rabbitmq-server"):
+                    # init is systemd
+                    cmd = 'sudo service {} status'.format(service_name)
+                elif self.ubuntu_releases.index(release) < systemd_switch:
+                    # init is upstart
+                    cmd = 'sudo status {}'.format(service_name)
+
+                output, code = sentry_unit.run(cmd)
+                self.log.debug('{} `{}` returned '
+                               '{}'.format(sentry_unit.info['unit_name'],
+                                           cmd, code))
+                if code != 0:
+                    return "command `{}` returned {}".format(cmd, str(code))
+        return None
+
     def _get_config(self, unit, filename):
         """Get a ConfigParser object for parsing a unit's config file."""
         file_contents = unit.file_contents(filename)
@@ -104,6 +174,9 @@
            Verify that the specified section of the config file contains
            the expected option key:value pairs.
            """
+        self.log.debug('Validating config file data ({} in {} on {})'
+                       '...'.format(section, config_file,
+                                    sentry_unit.info['unit_name']))
         config = self._get_config(sentry_unit, config_file)
 
         if section != 'DEFAULT' and not config.has_section(section):
@@ -321,3 +394,15 @@
 
     def endpoint_error(self, name, data):
         return 'unexpected endpoint data in {} - {}'.format(name, data)
+
+    def get_ubuntu_releases(self):
+        """Return a list of all Ubuntu releases in order of release."""
+        _d = distro_info.UbuntuDistroInfo()
+        _release_list = _d.all
+        self.log.debug('Ubuntu release list: {}'.format(_release_list))
+        return _release_list
+
+    def file_to_url(self, file_rel_path):
+        """Convert a relative file path to a file URL."""
+        _abs_path = os.path.abspath(file_rel_path)
+        return urlparse.urlparse(_abs_path, scheme='file').geturl()

=== modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py'
--- tests/charmhelpers/contrib/openstack/amulet/deployment.py	2015-05-11 07:26:16 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/deployment.py	2015-06-12 17:20:17 +0000
@@ -110,7 +110,8 @@
         (self.precise_essex, self.precise_folsom, self.precise_grizzly,
          self.precise_havana, self.precise_icehouse,
          self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
-         self.trusty_kilo, self.vivid_kilo) = range(10)
+         self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
+         self.wily_liberty) = range(12)
 
         releases = {
             ('precise', None): self.precise_essex,
@@ -121,8 +122,10 @@
             ('trusty', None): self.trusty_icehouse,
             ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
             ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
+            ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
             ('utopic', None): self.utopic_juno,
-            ('vivid', None): self.vivid_kilo}
+            ('vivid', None): self.vivid_kilo,
+            ('wily', None): self.wily_liberty}
         return releases[(self.series, self.openstack)]
 
     def _get_openstack_release_string(self):
@@ -138,6 +141,7 @@
             ('trusty', 'icehouse'),
             ('utopic', 'juno'),
             ('vivid', 'kilo'),
+            ('wily', 'liberty'),
         ])
         if self.openstack:
             os_origin = self.openstack.split(':')[1]

=== modified file 'tests/charmhelpers/contrib/openstack/amulet/utils.py'
--- tests/charmhelpers/contrib/openstack/amulet/utils.py	2015-02-17 14:09:16 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/utils.py	2015-06-12 17:20:17 +0000
@@ -16,15 +16,15 @@
 
 import logging
 import os
+import six
 import time
 import urllib
 
 import glanceclient.v1.client as glance_client
+import heatclient.v1.client as heat_client
 import keystoneclient.v2_0 as keystone_client
 import novaclient.v1_1.client as nova_client
 
-import six
-
 from charmhelpers.contrib.amulet.utils import (
     AmuletUtils
 )
@@ -37,7 +37,7 @@
     """OpenStack amulet utilities.
 
        This class inherits from AmuletUtils and has additional support
-       that is specifically for use by OpenStack charms.
+       that is specifically for use by OpenStack charm tests.
        """
 
     def __init__(self, log_level=ERROR):
@@ -51,6 +51,8 @@
            Validate actual endpoint data vs expected endpoint data. The ports
            are used to find the matching endpoint.
            """
+        self.log.debug('Validating endpoint data...')
+        self.log.debug('actual: {}'.format(repr(endpoints)))
         found = False
         for ep in endpoints:
             self.log.debug('endpoint: {}'.format(repr(ep)))
@@ -77,6 +79,7 @@
            Validate a list of actual service catalog endpoints vs a list of
            expected service catalog endpoints.
            """
+        self.log.debug('Validating service catalog endpoint data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for k, v in six.iteritems(expected):
             if k in actual:
@@ -93,6 +96,7 @@
            Validate a list of actual tenant data vs list of expected tenant
            data.
            """
+        self.log.debug('Validating tenant data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -114,6 +118,7 @@
            Validate a list of actual role data vs a list of expected role
            data.
            """
+        self.log.debug('Validating role data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -134,6 +139,7 @@
            Validate a list of actual user data vs a list of expected user
            data.
            """
+        self.log.debug('Validating user data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -155,17 +161,20 @@
 
            Validate a list of actual flavors vs a list of expected flavors.
            """
+        self.log.debug('Validating flavor data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         act = [a.name for a in actual]
         return self._validate_list_data(expected, act)
 
     def tenant_exists(self, keystone, tenant):
         """Return True if tenant exists."""
+        self.log.debug('Checking if tenant exists ({})...'.format(tenant))
         return tenant in [t.name for t in keystone.tenants.list()]
 
     def authenticate_keystone_admin(self, keystone_sentry, user, password,
                                     tenant):
         """Authenticates admin user with the keystone admin endpoint."""
+        self.log.debug('Authenticating keystone admin...')
         unit = keystone_sentry
         service_ip = unit.relation('shared-db',
                                    'mysql:shared-db')['private-address']
@@ -175,6 +184,7 @@
 
     def authenticate_keystone_user(self, keystone, user, password, tenant):
         """Authenticates a regular user with the keystone public endpoint."""
+        self.log.debug('Authenticating keystone user ({})...'.format(user))
         ep = keystone.service_catalog.url_for(service_type='identity',
                                               endpoint_type='publicURL')
         return keystone_client.Client(username=user, password=password,
@@ -182,12 +192,21 @@
 
     def authenticate_glance_admin(self, keystone):
         """Authenticates admin user with glance."""
+        self.log.debug('Authenticating glance admin...')
         ep = keystone.service_catalog.url_for(service_type='image',
                                               endpoint_type='adminURL')
         return glance_client.Client(ep, token=keystone.auth_token)
 
+    def authenticate_heat_admin(self, keystone):
+        """Authenticates the admin user with heat."""
+        self.log.debug('Authenticating heat admin...')
+        ep = keystone.service_catalog.url_for(service_type='orchestration',
+                                              endpoint_type='publicURL')
+        return heat_client.Client(endpoint=ep, token=keystone.auth_token)
+
     def authenticate_nova_user(self, keystone, user, password, tenant):
         """Authenticates a regular user with nova-api."""
+        self.log.debug('Authenticating nova user ({})...'.format(user))
         ep = keystone.service_catalog.url_for(service_type='identity',
                                               endpoint_type='publicURL')
         return nova_client.Client(username=user, api_key=password,
@@ -195,6 +214,7 @@
 
     def create_cirros_image(self, glance, image_name):
         """Download the latest cirros image and upload it to glance."""
+        self.log.debug('Creating glance image ({})...'.format(image_name))
         http_proxy = os.getenv('AMULET_HTTP_PROXY')
         self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
         if http_proxy:
@@ -235,6 +255,11 @@
 
     def delete_image(self, glance, image):
         """Delete the specified image."""
+
+        # /!\ DEPRECATION WARNING
+        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+                      'delete_resource instead of delete_image.')
+        self.log.debug('Deleting glance image ({})...'.format(image))
         num_before = len(list(glance.images.list()))
         glance.images.delete(image)
 
@@ -254,6 +279,8 @@
 
     def create_instance(self, nova, image_name, instance_name, flavor):
         """Create the specified instance."""
+        self.log.debug('Creating instance '
+                       '({}|{}|{})'.format(instance_name, image_name, flavor))
         image = nova.images.find(name=image_name)
         flavor = nova.flavors.find(name=flavor)
         instance = nova.servers.create(name=instance_name, image=image,
@@ -276,6 +303,11 @@
 
     def delete_instance(self, nova, instance):
         """Delete the specified instance."""
+
+        # /!\ DEPRECATION WARNING
+        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+                      'delete_resource instead of delete_instance.')
+        self.log.debug('Deleting instance ({})...'.format(instance))
         num_before = len(list(nova.servers.list()))
         nova.servers.delete(instance)
 
@@ -292,3 +324,90 @@
             return False
 
         return True
+
+    def create_or_get_keypair(self, nova, keypair_name="testkey"):
+        """Create a new keypair, or return pointer if it already exists."""
+        try:
+            _keypair = nova.keypairs.get(keypair_name)
+            self.log.debug('Keypair ({}) already exists, '
+                           'using it.'.format(keypair_name))
+            return _keypair
+        except:
+            self.log.debug('Keypair ({}) does not exist, '
+                           'creating it.'.format(keypair_name))
+
+        _keypair = nova.keypairs.create(name=keypair_name)
+        return _keypair
+
+    def delete_resource(self, resource, resource_id,
+                        msg="resource", max_wait=120):
+        """Delete one openstack resource, such as one instance, keypair,
+        image, volume, stack, etc., and confirm deletion within max wait time.
+
+        :param resource: pointer to os resource type, ex:glance_client.images
+        :param resource_id: unique name or id for the openstack resource
+        :param msg: text to identify purpose in logging
+        :param max_wait: maximum wait time in seconds
+        :returns: True if successful, otherwise False
+        """
+        num_before = len(list(resource.list()))
+        resource.delete(resource_id)
+
+        tries = 0
+        num_after = len(list(resource.list()))
+        while num_after != (num_before - 1) and tries < (max_wait / 4):
+            self.log.debug('{} delete check: '
+                           '{} [{}:{}] {}'.format(msg, tries,
+                                                  num_before,
+                                                  num_after,
+                                                  resource_id))
+            time.sleep(4)
+            num_after = len(list(resource.list()))
+            tries += 1
+
+        self.log.debug('{}:  expected, actual count = {}, '
+                       '{}'.format(msg, num_before - 1, num_after))
+
+        if num_after == (num_before - 1):
+            return True
+        else:
+            self.log.error('{} delete timed out'.format(msg))
+            return False
+
+    def resource_reaches_status(self, resource, resource_id,
+                                expected_stat='available',
+                                msg='resource', max_wait=120):
+        """Wait for an openstack resources status to reach an
+           expected status within a specified time.  Useful to confirm that
+           nova instances, cinder vols, snapshots, glance images, heat stacks
+           and other resources eventually reach the expected status.
+
+        :param resource: pointer to os resource type, ex: heat_client.stacks
+        :param resource_id: unique id for the openstack resource
+        :param expected_stat: status to expect resource to reach
+        :param msg: text to identify purpose in logging
+        :param max_wait: maximum wait time in seconds
+        :returns: True if successful, False if status is not reached
+        """
+
+        tries = 0
+        resource_stat = resource.get(resource_id).status
+        while resource_stat != expected_stat and tries < (max_wait / 4):
+            self.log.debug('{} status check: '
+                           '{} [{}:{}] {}'.format(msg, tries,
+                                                  resource_stat,
+                                                  expected_stat,
+                                                  resource_id))
+            time.sleep(4)
+            resource_stat = resource.get(resource_id).status
+            tries += 1
+
+        self.log.debug('{}:  expected, actual status = {}, '
+                       '{}'.format(msg, resource_stat, expected_stat))
+
+        if resource_stat == expected_stat:
+            return True
+        else:
+            self.log.debug('{} never reached expected status: '
+                           '{}'.format(resource_id, expected_stat))
+            return False