Merge lp:~1chb1n/charms/trusty/glance/next-amulet-update into lp:~openstack-charmers-archive/charms/trusty/glance/next

[Ryan Beisner]

Update amulet tests for Kilo, prep for wily. Sync hooks/charmhelpers; Sync tests/charmhelpers.

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #5714 glance-next for 1chb1n mp263413
    LINT FAIL: lint-test failed

LINT Results (max last 2 lines):
make: *** [lint] Error 1
ERROR:root:Make target returned non-zero.

Full lint test output: http://paste.ubuntu.com/11808187/
Build: http://10.245.162.77:8080/job/charm_lint_check/5714/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #5346 glance-next for 1chb1n mp263413
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/5346/

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #5716 glance-next for 1chb1n mp263413
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/5716/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #5348 glance-next for 1chb1n mp263413
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/5348/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #4906 glance-next for 1chb1n mp263413
    AMULET FAIL: amulet-test failed

AMULET Results (max last 2 lines):
make: *** [functional_test] Error 1
ERROR:root:Make target returned non-zero.

Full amulet test output: http://paste.ubuntu.com/11808398/
Build: http://10.245.162.77:8080/job/charm_amulet_test/4906/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #4908 glance-next for 1chb1n mp263413
    AMULET FAIL: amulet-test failed

AMULET Results (max last 2 lines):
make: *** [functional_test] Error 1
ERROR:root:Make target returned non-zero.

Full amulet test output: http://paste.ubuntu.com/11808501/
Build: http://10.245.162.77:8080/job/charm_amulet_test/4908/

[lp:~1chb1n/charms/trusty/glance/next-amulet-update]

126. By Ryan Beisner on 2015-07-02

update tests for vivid-kilo

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #5718 glance-next for 1chb1n mp263413
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/5718/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #5350 glance-next for 1chb1n mp263413
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/5350/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #4910 glance-next for 1chb1n mp263413
    AMULET OK: passed

Build: http://10.245.162.77:8080/job/charm_amulet_test/4910/

=== modified file 'Makefile'
--- Makefile	2015-04-16 21:32:02 +0000
+++ Makefile	2015-07-02 12:52:15 +0000
@@ -2,16 +2,18 @@
 PYTHON := /usr/bin/env python
 
 lint:
-	@echo "Running flake8 tests: "
-	@flake8 --exclude hooks/charmhelpers actions hooks unit_tests tests
-	@echo "OK"
-	@echo "Running charm proof: "
+	@flake8 --exclude hooks/charmhelpers,tests/charmhelpers \
+        actions hooks unit_tests tests
 	@charm proof
-	@echo "OK"
 
-unit_test:
+test:
+	@# Bundletester expects unit tests here.
 	@$(PYTHON) /usr/bin/nosetests --nologcapture --with-coverage unit_tests
 
+functional_test:
+	@echo Starting Amulet tests...
+	@juju test -v -p AMULET_HTTP_PROXY,AMULET_OS_VIP --timeout 2700
+
 bin/charm_helpers_sync.py:
 	@mkdir -p bin
 	@bzr cat lp:charm-helpers/tools/charm_helpers_sync/charm_helpers_sync.py \
@@ -21,15 +23,8 @@
 	@$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-hooks.yaml
 	@$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-tests.yaml
 
-test:
-	@echo Starting Amulet tests...
-	# /!\ Note: The -v should only be temporary until Amulet sends
-	# raise_status() messages to stderr:
-	#   https://bugs.launchpad.net/amulet/+bug/1320357
-	@juju test -v -p AMULET_HTTP_PROXY,AMULET_OS_VIP --timeout 2700
-
-publish: lint unit_test
+publish: lint test
 	bzr push lp:charms/glance
 	bzr push lp:charms/trusty/glance
 
-all: unit_test lint
+all: test lint

=== modified file 'hooks/charmhelpers/core/hookenv.py'
--- hooks/charmhelpers/core/hookenv.py	2015-06-10 20:31:46 +0000
+++ hooks/charmhelpers/core/hookenv.py	2015-07-02 12:52:15 +0000
@@ -21,7 +21,9 @@
 #  Charm Helpers Developers <[e-mail]>
 
 from __future__ import print_function
+from distutils.version import LooseVersion
 from functools import wraps
+import glob
 import os
 import json
 import yaml
@@ -242,29 +244,7 @@
         self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
         if os.path.exists(self.path):
             self.load_previous()
-
-    def __getitem__(self, key):
-        """For regular dict lookups, check the current juju config first,
-        then the previous (saved) copy. This ensures that user-saved values
-        will be returned by a dict lookup.
-
-        """
-        try:
-            return dict.__getitem__(self, key)
-        except KeyError:
-            return (self._prev_dict or {})[key]
-
-    def get(self, key, default=None):
-        try:
-            return self[key]
-        except KeyError:
-            return default
-
-    def keys(self):
-        prev_keys = []
-        if self._prev_dict is not None:
-            prev_keys = self._prev_dict.keys()
-        return list(set(prev_keys + list(dict.keys(self))))
+        atexit(self._implicit_save)
 
     def load_previous(self, path=None):
         """Load previous copy of config from disk.
@@ -283,6 +263,9 @@
         self.path = path or self.path
         with open(self.path) as f:
             self._prev_dict = json.load(f)
+        for k, v in self._prev_dict.items():
+            if k not in self:
+                self[k] = v
 
     def changed(self, key):
         """Return True if the current value for this key is different from
@@ -314,13 +297,13 @@
         instance.
 
         """
-        if self._prev_dict:
-            for k, v in six.iteritems(self._prev_dict):
-                if k not in self:
-                    self[k] = v
         with open(self.path, 'w') as f:
             json.dump(self, f)
 
+    def _implicit_save(self):
+        if self.implicit_save:
+            self.save()
+
 
 @cached
 def config(scope=None):
@@ -587,10 +570,14 @@
             hooks.execute(sys.argv)
     """
 
-    def __init__(self, config_save=True):
+    def __init__(self, config_save=None):
         super(Hooks, self).__init__()
         self._hooks = {}
-        self._config_save = config_save
+
+        # For unknown reasons, we allow the Hooks constructor to override
+        # config().implicit_save.
+        if config_save is not None:
+            config().implicit_save = config_save
 
     def register(self, name, function):
         """Register a hook"""
@@ -598,13 +585,16 @@
 
     def execute(self, args):
         """Execute a registered hook based on args[0]"""
+        _run_atstart()
         hook_name = os.path.basename(args[0])
         if hook_name in self._hooks:
-            self._hooks[hook_name]()
-            if self._config_save:
-                cfg = config()
-                if cfg.implicit_save:
-                    cfg.save()
+            try:
+                self._hooks[hook_name]()
+            except SystemExit as x:
+                if x.code is None or x.code == 0:
+                    _run_atexit()
+                raise
+            _run_atexit()
         else:
             raise UnregisteredHookError(hook_name)
 
@@ -732,13 +722,79 @@
 @translate_exc(from_exc=OSError, to_exc=NotImplementedError)
 def leader_set(settings=None, **kwargs):
     """Juju leader set value(s)"""
-    log("Juju leader-set '%s'" % (settings), level=DEBUG)
+    # Don't log secrets.
+    # log("Juju leader-set '%s'" % (settings), level=DEBUG)
     cmd = ['leader-set']
     settings = settings or {}
     settings.update(kwargs)
-    for k, v in settings.iteritems():
+    for k, v in settings.items():
         if v is None:
             cmd.append('{}='.format(k))
         else:
             cmd.append('{}={}'.format(k, v))
     subprocess.check_call(cmd)
+
+
+@cached
+def juju_version():
+    """Full version string (eg. '1.23.3.1-trusty-amd64')"""
+    # Per https://bugs.launchpad.net/juju-core/+bug/1455368/comments/1
+    jujud = glob.glob('/var/lib/juju/tools/machine-*/jujud')[0]
+    return subprocess.check_output([jujud, 'version'],
+                                   universal_newlines=True).strip()
+
+
+@cached
+def has_juju_version(minimum_version):
+    """Return True if the Juju version is at least the provided version"""
+    return LooseVersion(juju_version()) >= LooseVersion(minimum_version)
+
+
+_atexit = []
+_atstart = []
+
+
+def atstart(callback, *args, **kwargs):
+    '''Schedule a callback to run before the main hook.
+
+    Callbacks are run in the order they were added.
+
+    This is useful for modules and classes to perform initialization
+    and inject behavior. In particular:
+        - Run common code before all of your hooks, such as logging
+          the hook name or interesting relation data.
+        - Defer object or module initialization that requires a hook
+          context until we know there actually is a hook context,
+          making testing easier.
+        - Rather than requiring charm authors to include boilerplate to
+          invoke your helper's behavior, have it run automatically if
+          your object is instantiated or module imported.
+
+    This is not at all useful after your hook framework as been launched.
+    '''
+    global _atstart
+    _atstart.append((callback, args, kwargs))
+
+
+def atexit(callback, *args, **kwargs):
+    '''Schedule a callback to run on successful hook completion.
+
+    Callbacks are run in the reverse order that they were added.'''
+    _atexit.append((callback, args, kwargs))
+
+
+def _run_atstart():
+    '''Hook frameworks must invoke this before running the main hook body.'''
+    global _atstart
+    for callback, args, kwargs in _atstart:
+        callback(*args, **kwargs)
+    del _atstart[:]
+
+
+def _run_atexit():
+    '''Hook frameworks must invoke this after the main hook body has
+    successfully completed. Do not invoke it if the hook fails.'''
+    global _atexit
+    for callback, args, kwargs in reversed(_atexit):
+        callback(*args, **kwargs)
+    del _atexit[:]

=== modified file 'hooks/charmhelpers/core/services/base.py'
--- hooks/charmhelpers/core/services/base.py	2015-06-10 20:31:46 +0000
+++ hooks/charmhelpers/core/services/base.py	2015-07-02 12:52:15 +0000
@@ -128,15 +128,18 @@
         """
         Handle the current hook by doing The Right Thing with the registered services.
         """
-        hook_name = hookenv.hook_name()
-        if hook_name == 'stop':
-            self.stop_services()
-        else:
-            self.reconfigure_services()
-            self.provide_data()
-        cfg = hookenv.config()
-        if cfg.implicit_save:
-            cfg.save()
+        hookenv._run_atstart()
+        try:
+            hook_name = hookenv.hook_name()
+            if hook_name == 'stop':
+                self.stop_services()
+            else:
+                self.reconfigure_services()
+                self.provide_data()
+        except SystemExit as x:
+            if x.code is None or x.code == 0:
+                hookenv._run_atexit()
+        hookenv._run_atexit()
 
     def provide_data(self):
         """

=== modified file 'metadata.yaml'
--- metadata.yaml	2014-10-30 03:30:35 +0000
+++ metadata.yaml	2015-07-02 12:52:15 +0000
@@ -6,8 +6,10 @@
  (Parallax) and an image delivery service (Teller). These services are used
  in conjunction by Nova to deliver images from object stores, such as
  OpenStack's Swift service, to Nova's compute nodes.
-categories:
-  - miscellaneous
+tags:
+  - openstack
+  - storage
+  - misc
 provides:
   nrpe-external-master:
     interface: nrpe-external-master

=== modified file 'tests/00-setup'
--- tests/00-setup	2014-10-08 20:18:38 +0000
+++ tests/00-setup	2015-07-02 12:52:15 +0000
@@ -5,6 +5,10 @@
 sudo add-apt-repository --yes ppa:juju/stable
 sudo apt-get update --yes
 sudo apt-get install --yes python-amulet \
+                           python-cinderclient \
+                           python-distro-info \
+                           python-glanceclient \
+                           python-heatclient \
                            python-keystoneclient \
-                           python-glanceclient \
-                           python-novaclient
+                           python-novaclient \
+                           python-swiftclient

=== modified file 'tests/017-basic-trusty-kilo' (properties changed: -x to +x)
=== modified file 'tests/019-basic-vivid-kilo' (properties changed: -x to +x)
=== added file 'tests/020-basic-trusty-liberty'
--- tests/020-basic-trusty-liberty	1970-01-01 00:00:00 +0000
+++ tests/020-basic-trusty-liberty	2015-07-02 12:52:15 +0000
@@ -0,0 +1,11 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic glance deployment on trusty-liberty."""
+
+from basic_deployment import GlanceBasicDeployment
+
+if __name__ == '__main__':
+    deployment = GlanceBasicDeployment(series='trusty',
+                                       openstack='cloud:trusty-liberty',
+                                       source='cloud:trusty-updates/liberty')
+    deployment.run_tests()

=== added file 'tests/021-basic-wily-liberty'
--- tests/021-basic-wily-liberty	1970-01-01 00:00:00 +0000
+++ tests/021-basic-wily-liberty	2015-07-02 12:52:15 +0000
@@ -0,0 +1,9 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic glance deployment on wily-liberty."""
+
+from basic_deployment import GlanceBasicDeployment
+
+if __name__ == '__main__':
+    deployment = GlanceBasicDeployment(series='wily')
+    deployment.run_tests()

=== modified file 'tests/README'
--- tests/README	2014-10-08 20:18:38 +0000
+++ tests/README	2015-07-02 12:52:15 +0000
@@ -1,6 +1,15 @@
 This directory provides Amulet tests that focus on verification of Glance
 deployments.
 
+test_* methods are called in lexical sort order.
+
+Test name convention to ensure desired test order:
+    1xx service and endpoint checks
+    2xx relation checks
+    3xx config checks
+    4xx functional checks
+    9xx restarts and other final checks
+
 In order to run tests, you'll need charm-tools installed (in addition to
 juju, of course):
     sudo add-apt-repository ppa:juju/stable

=== modified file 'tests/basic_deployment.py'
--- tests/basic_deployment.py	2015-05-12 14:49:27 +0000
+++ tests/basic_deployment.py	2015-07-02 12:52:15 +0000
@@ -1,7 +1,12 @@
 #!/usr/bin/python
 
+"""
+Basic glance amulet functional tests.
+"""
+
 import amulet
 import os
+import time
 import yaml
 
 from charmhelpers.contrib.openstack.amulet.deployment import (
@@ -10,25 +15,24 @@
 
 from charmhelpers.contrib.openstack.amulet.utils import (
     OpenStackAmuletUtils,
-    DEBUG, # flake8: noqa
-    ERROR
+    DEBUG,
+    # ERROR
 )
 
 # Use DEBUG to turn on debug logging
 u = OpenStackAmuletUtils(DEBUG)
 
+
 class GlanceBasicDeployment(OpenStackAmuletDeployment):
-    '''Amulet tests on a basic file-backed glance deployment.  Verify relations,
-       service status, endpoint service catalog, create and delete new image.'''
-
-#  TO-DO(beisner): 
-#    * Add tests with different storage back ends
-#    * Resolve Essex->Havana juju set charm bug
+    """Amulet tests on a basic file-backed glance deployment.  Verify
+    relations, service status, endpoint service catalog, create and
+    delete new image."""
 
     def __init__(self, series=None, openstack=None, source=None, git=False,
                  stable=False):
-        '''Deploy the entire test environment.'''
-        super(GlanceBasicDeployment, self).__init__(series, openstack, source, stable)
+        """Deploy the entire test environment."""
+        super(GlanceBasicDeployment, self).__init__(series, openstack,
+                                                    source, stable)
         self.git = git
         self._add_services()
         self._add_relations()
@@ -37,20 +41,21 @@
         self._initialize_tests()
 
     def _add_services(self):
-        '''Add services
+        """Add services
 
            Add the services that we're testing, where glance is local,
            and the rest of the service are from lp branches that are
            compatible with the local charm (e.g. stable or next).
-           '''
+           """
         this_service = {'name': 'glance'}
-        other_services = [{'name': 'mysql'}, {'name': 'rabbitmq-server'},
+        other_services = [{'name': 'mysql'},
+                          {'name': 'rabbitmq-server'},
                           {'name': 'keystone'}]
         super(GlanceBasicDeployment, self)._add_services(this_service,
                                                          other_services)
 
     def _add_relations(self):
-        '''Add relations for the services.'''
+        """Add relations for the services."""
         relations = {'glance:identity-service': 'keystone:identity-service',
                      'glance:shared-db': 'mysql:shared-db',
                      'keystone:shared-db': 'mysql:shared-db',
@@ -58,7 +63,7 @@
         super(GlanceBasicDeployment, self)._add_relations(relations)
 
     def _configure_services(self):
-        '''Configure all of the services.'''
+        """Configure all of the services."""
         glance_config = {}
         if self.git:
             branch = 'stable/' + self._get_openstack_release_string()
@@ -76,7 +81,8 @@
                 'http_proxy': amulet_http_proxy,
                 'https_proxy': amulet_http_proxy,
             }
-            glance_config['openstack-origin-git'] = yaml.dump(openstack_origin_git)
+            glance_config['openstack-origin-git'] = \
+                yaml.dump(openstack_origin_git)
 
         keystone_config = {'admin-password': 'openstack',
                            'admin-token': 'ubuntutesting'}
@@ -87,12 +93,19 @@
         super(GlanceBasicDeployment, self)._configure_services(configs)
 
     def _initialize_tests(self):
-        '''Perform final initialization before tests get run.'''
+        """Perform final initialization before tests get run."""
         # Access the sentries for inspecting service units
         self.mysql_sentry = self.d.sentry.unit['mysql/0']
         self.glance_sentry = self.d.sentry.unit['glance/0']
         self.keystone_sentry = self.d.sentry.unit['keystone/0']
         self.rabbitmq_sentry = self.d.sentry.unit['rabbitmq-server/0']
+        u.log.debug('openstack release val: {}'.format(
+            self._get_openstack_release()))
+        u.log.debug('openstack release str: {}'.format(
+            self._get_openstack_release_string()))
+
+        # Let things settle a bit before moving forward
+        time.sleep(30)
 
         # Authenticate admin with keystone
         self.keystone = u.authenticate_keystone_admin(self.keystone_sentry,
@@ -103,46 +116,103 @@
         # Authenticate admin with glance endpoint
         self.glance = u.authenticate_glance_admin(self.keystone)
 
-        u.log.debug('openstack release: {}'.format(self._get_openstack_release()))
-
-    def test_services(self):
-        '''Verify that the expected services are running on the
-           corresponding service units.'''
-        commands = {
-            self.mysql_sentry: ['status mysql'],
-            self.keystone_sentry: ['status keystone'],
-            self.glance_sentry: ['status glance-api', 'status glance-registry'],
-            self.rabbitmq_sentry: ['sudo service rabbitmq-server status']
+    def test_100_services(self):
+        """Verify that the expected services are running on the
+           corresponding service units."""
+        services = {
+            self.mysql_sentry: ['mysql'],
+            self.keystone_sentry: ['keystone'],
+            self.glance_sentry: ['glance-api', 'glance-registry'],
+            self.rabbitmq_sentry: ['rabbitmq-server']
         }
-        u.log.debug('commands: {}'.format(commands))
-        ret = u.validate_services(commands)
+
+        ret = u.validate_services_by_name(services)
         if ret:
             amulet.raise_status(amulet.FAIL, msg=ret)
 
-    def test_service_catalog(self):
-        '''Verify that the service catalog endpoint data'''
-        endpoint_vol = {'adminURL': u.valid_url,
-                        'region': 'RegionOne',
-                        'publicURL': u.valid_url,
-                        'internalURL': u.valid_url}
-        endpoint_id = {'adminURL': u.valid_url,
-                       'region': 'RegionOne',
-                       'publicURL': u.valid_url,
-                       'internalURL': u.valid_url}
-        if self._get_openstack_release() >= self.trusty_icehouse:
-            endpoint_vol['id'] = u.not_null
-            endpoint_id['id'] = u.not_null
-
-        expected = {'image': [endpoint_id],
-                    'identity': [endpoint_id]}
+    def test_102_service_catalog(self):
+        """Verify that the service catalog endpoint data is valid."""
+        u.log.debug('Checking keystone service catalog...')
+        endpoint_check = {
+            'adminURL': u.valid_url,
+            'id': u.not_null,
+            'region': 'RegionOne',
+            'publicURL': u.valid_url,
+            'internalURL': u.valid_url
+        }
+        expected = {
+            'image': [endpoint_check],
+            'identity': [endpoint_check]
+        }
         actual = self.keystone.service_catalog.get_endpoints()
 
         ret = u.validate_svc_catalog_endpoint_data(expected, actual)
         if ret:
             amulet.raise_status(amulet.FAIL, msg=ret)
 
-    def test_mysql_glance_db_relation(self):
-        '''Verify the mysql:glance shared-db relation data'''
+    def test_104_glance_endpoint(self):
+        """Verify the glance endpoint data."""
+        u.log.debug('Checking glance api endpoint data...')
+        endpoints = self.keystone.endpoints.list()
+        admin_port = internal_port = public_port = '9292'
+        expected = {
+            'id': u.not_null,
+            'region': 'RegionOne',
+            'adminurl': u.valid_url,
+            'internalurl': u.valid_url,
+            'publicurl': u.valid_url,
+            'service_id': u.not_null
+        }
+        ret = u.validate_endpoint_data(endpoints, admin_port, internal_port,
+                                       public_port, expected)
+
+        if ret:
+            amulet.raise_status(amulet.FAIL,
+                                msg='glance endpoint: {}'.format(ret))
+
+    def test_106_keystone_endpoint(self):
+        """Verify the keystone endpoint data."""
+        u.log.debug('Checking keystone api endpoint data...')
+        endpoints = self.keystone.endpoints.list()
+        admin_port = '35357'
+        internal_port = public_port = '5000'
+        expected = {
+            'id': u.not_null,
+            'region': 'RegionOne',
+            'adminurl': u.valid_url,
+            'internalurl': u.valid_url,
+            'publicurl': u.valid_url,
+            'service_id': u.not_null
+        }
+        ret = u.validate_endpoint_data(endpoints, admin_port, internal_port,
+                                       public_port, expected)
+        if ret:
+            amulet.raise_status(amulet.FAIL,
+                                msg='keystone endpoint: {}'.format(ret))
+
+    def test_110_users(self):
+        """Verify expected users."""
+        u.log.debug('Checking keystone users...')
+        expected = [
+            {'name': 'glance',
+             'enabled': True,
+             'tenantId': u.not_null,
+             'id': u.not_null,
+             'email': 'juju@localhost'},
+            {'name': 'admin',
+             'enabled': True,
+             'tenantId': u.not_null,
+             'id': u.not_null,
+             'email': 'juju@localhost'}
+        ]
+        actual = self.keystone.users.list()
+        ret = u.validate_user_data(expected, actual)
+        if ret:
+            amulet.raise_status(amulet.FAIL, msg=ret)
+
+    def test_200_mysql_glance_db_relation(self):
+        """Verify the mysql:glance shared-db relation data"""
+        u.log.debug('Checking mysql to glance shared-db relation data...')
         unit = self.mysql_sentry
         relation = ['shared-db', 'glance:shared-db']
         expected = {
@@ -154,8 +224,9 @@
             message = u.relation_error('mysql shared-db', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_glance_mysql_db_relation(self):
-        '''Verify the glance:mysql shared-db relation data'''
+    def test_201_glance_mysql_db_relation(self):
+        """Verify the glance:mysql shared-db relation data"""
+        u.log.debug('Checking glance to mysql shared-db relation data...')
         unit = self.glance_sentry
         relation = ['shared-db', 'mysql:shared-db']
         expected = {
@@ -169,8 +240,9 @@
             message = u.relation_error('glance shared-db', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_keystone_glance_id_relation(self):
-        '''Verify the keystone:glance identity-service relation data'''
+    def test_202_keystone_glance_id_relation(self):
+        """Verify the keystone:glance identity-service relation data"""
+        u.log.debug('Checking keystone to glance id relation data...')
         unit = self.keystone_sentry
         relation = ['identity-service',
                     'glance:identity-service']
@@ -193,8 +265,9 @@
             message = u.relation_error('keystone identity-service', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_glance_keystone_id_relation(self):
-        '''Verify the glance:keystone identity-service relation data'''
+    def test_203_glance_keystone_id_relation(self):
+        """Verify the glance:keystone identity-service relation data"""
+        u.log.debug('Checking glance to keystone relation data...')
         unit = self.glance_sentry
         relation = ['identity-service',
                     'keystone:identity-service']
@@ -211,8 +284,9 @@
             message = u.relation_error('glance identity-service', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_rabbitmq_glance_amqp_relation(self):
-        '''Verify the rabbitmq-server:glance amqp relation data'''
+    def test_204_rabbitmq_glance_amqp_relation(self):
+        """Verify the rabbitmq-server:glance amqp relation data"""
+        u.log.debug('Checking rmq to glance amqp relation data...')
         unit = self.rabbitmq_sentry
         relation = ['amqp', 'glance:amqp']
         expected = {
@@ -225,8 +299,9 @@
             message = u.relation_error('rabbitmq amqp', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_glance_rabbitmq_amqp_relation(self):
-        '''Verify the glance:rabbitmq-server amqp relation data'''
+    def test_205_glance_rabbitmq_amqp_relation(self):
+        """Verify the glance:rabbitmq-server amqp relation data"""
+        u.log.debug('Checking glance to rmq amqp relation data...')
         unit = self.glance_sentry
         relation = ['amqp', 'rabbitmq-server:amqp']
         expected = {
@@ -239,291 +314,225 @@
             message = u.relation_error('glance amqp', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_image_create_delete(self):
-        '''Create new cirros image in glance, verify, then delete it'''
-
-        # Create a new image
-        image_name = 'cirros-image-1'
-        image_new = u.create_cirros_image(self.glance, image_name)
-
-        # Confirm image is created and has status of 'active' 
-        if not image_new:
-            message = 'glance image create failed'
-            amulet.raise_status(amulet.FAIL, msg=message)
-
-        # Verify new image name
-        images_list = list(self.glance.images.list())
-        if images_list[0].name != image_name:
-            message = 'glance image create failed or unexpected image name {}'.format(images_list[0].name)
-            amulet.raise_status(amulet.FAIL, msg=message)
-
-        # Delete the new image
-        u.log.debug('image count before delete: {}'.format(len(list(self.glance.images.list()))))
-        u.delete_image(self.glance, image_new)
-        u.log.debug('image count after delete: {}'.format(len(list(self.glance.images.list()))))
-
-    def test_glance_api_default_config(self):
-        '''Verify default section configs in glance-api.conf and
-           compare some of the parameters to relation data.'''
-        unit = self.glance_sentry
-        rel_gl_mq = unit.relation('amqp', 'rabbitmq-server:amqp')
-        conf = '/etc/glance/glance-api.conf'
-        expected = {'use_syslog': 'False',
-                    'default_store': 'file',
-                    'filesystem_store_datadir': '/var/lib/glance/images/',
-                    'rabbit_userid': rel_gl_mq['username'],
-                    'log_file': '/var/log/glance/api.log',
-                    'debug': 'False',
-                    'verbose': 'False'}
-        section = 'DEFAULT'
-
-        if self._get_openstack_release() <= self.precise_havana:
-            # Defaults were different before icehouse
-            expected['debug'] = 'True'
-            expected['verbose'] = 'True'
-
-        ret = u.validate_config_data(unit, conf, section, expected)
-        if ret:
-            message = "glance-api default config error: {}".format(ret)
-            amulet.raise_status(amulet.FAIL, msg=message)
-
-    def test_glance_api_auth_config(self):
-        '''Verify authtoken section config in glance-api.conf using
-           glance/keystone relation data.'''
-        unit_gl = self.glance_sentry
-        unit_ks = self.keystone_sentry
-        rel_gl_mq = unit_gl.relation('amqp', 'rabbitmq-server:amqp')
-        rel_ks_gl = unit_ks.relation('identity-service', 'glance:identity-service')
-        conf = '/etc/glance/glance-api.conf'
-        section = 'keystone_authtoken'
-
-        if self._get_openstack_release() > self.precise_havana:
-            # No auth config exists in this file before icehouse
-            expected = {'admin_user': 'glance',
-                    'admin_password': rel_ks_gl['service_password']}
-
-            ret = u.validate_config_data(unit_gl, conf, section, expected)
-            if ret:
-                message = "glance-api auth config error: {}".format(ret)
-                amulet.raise_status(amulet.FAIL, msg=message)
-
-    def test_glance_api_paste_auth_config(self):
-        '''Verify authtoken section config in glance-api-paste.ini using
-           glance/keystone relation data.'''
-        unit_gl = self.glance_sentry
-        unit_ks = self.keystone_sentry
-        rel_gl_mq = unit_gl.relation('amqp', 'rabbitmq-server:amqp')
-        rel_ks_gl = unit_ks.relation('identity-service', 'glance:identity-service')
+    def _get_keystone_authtoken_expected_dict(self, rel_ks_gl):
+        """Return expected authtoken dict for OS release"""
+        expected = {
+            'keystone_authtoken': {
+                'signing_dir': '/var/cache/glance',
+                'admin_tenant_name': 'services',
+                'admin_user': 'glance',
+                'admin_password': rel_ks_gl['service_password'],
+                'auth_uri': u.valid_url
+            }
+        }
+
+        if self._get_openstack_release() >= self.trusty_kilo:
+            # Trusty-Kilo and later
+            expected['keystone_authtoken'].update({
+                'identity_uri': u.valid_url,
+            })
+        else:
+            # Utopic-Juno and earlier
+            expected['keystone_authtoken'].update({
+                'auth_host': rel_ks_gl['auth_host'],
+                'auth_port': rel_ks_gl['auth_port'],
+                'auth_protocol':  rel_ks_gl['auth_protocol']
+            })
+
+        return expected
+
+    def test_300_glance_api_default_config(self):
+        """Verify default section configs in glance-api.conf and
+           compare some of the parameters to relation data."""
+        u.log.debug('Checking glance api config file...')
+        unit = self.glance_sentry
+        unit_ks = self.keystone_sentry
+        rel_mq_gl = self.rabbitmq_sentry.relation('amqp', 'glance:amqp')
+        rel_ks_gl = unit_ks.relation('identity-service',
+                                     'glance:identity-service')
+        rel_my_gl = self.mysql_sentry.relation('shared-db', 'glance:shared-db')
+        db_uri = "mysql://{}:{}@{}/{}".format('glance', rel_my_gl['password'],
+                                              rel_my_gl['db_host'], 'glance')
+        conf = '/etc/glance/glance-api.conf'
+        expected = {
+            'DEFAULT': {
+                'debug': 'False',
+                'verbose': 'False',
+                'use_syslog': 'False',
+                'log_file': '/var/log/glance/api.log',
+                'bind_host': '0.0.0.0',
+                'bind_port': '9282',
+                'registry_host': '0.0.0.0',
+                'registry_port': '9191',
+                'registry_client_protocol': 'http',
+                'delayed_delete': 'False',
+                'scrub_time': '43200',
+                'notification_driver': 'rabbit',
+                'scrubber_datadir': '/var/lib/glance/scrubber',
+                'image_cache_dir': '/var/lib/glance/image-cache/',
+                'db_enforce_mysql_charset': 'False'
+            },
+        }
+
+        expected.update(self._get_keystone_authtoken_expected_dict(rel_ks_gl))
+
+        if self._get_openstack_release() >= self.trusty_kilo:
+            # Kilo or later
+            expected['oslo_messaging_rabbit'] = {
+                'rabbit_userid': 'glance',
+                'rabbit_virtual_host': 'openstack',
+                'rabbit_password': rel_mq_gl['password'],
+                'rabbit_host': rel_mq_gl['hostname']
+            }
+            expected['glance_store'] = {
+                'filesystem_store_datadir': '/var/lib/glance/images/',
+                'stores': 'glance.store.filesystem.'
+                          'Store,glance.store.http.Store',
+                'default_store': 'file'
+            }
+            expected['database'] = {
+                'idle_timeout': '3600',
+                'connection': db_uri
+            }
+        else:
+            # Juno or earlier
+            expected['DEFAULT'].update({
+                'rabbit_userid': 'glance',
+                'rabbit_virtual_host': 'openstack',
+                'rabbit_password': rel_mq_gl['password'],
+                'rabbit_host': rel_mq_gl['hostname'],
+                'filesystem_store_datadir': '/var/lib/glance/images/',
+                'default_store': 'file',
+            })
+            expected['database'] = {
+                'sql_idle_timeout': '3600',
+                'connection': db_uri
+            }
+
+        for section, pairs in expected.iteritems():
+            ret = u.validate_config_data(unit, conf, section, pairs)
+            if ret:
+                message = "glance api config error: {}".format(ret)
+                amulet.raise_status(amulet.FAIL, msg=message)
+
+    def test_302_glance_registry_default_config(self):
+        """Verify configs in glance-registry.conf"""
+        u.log.debug('Checking glance registry config file...')
+        unit = self.glance_sentry
+        unit_ks = self.keystone_sentry
+        rel_ks_gl = unit_ks.relation('identity-service',
+                                     'glance:identity-service')
+        rel_my_gl = self.mysql_sentry.relation('shared-db', 'glance:shared-db')
+        db_uri = "mysql://{}:{}@{}/{}".format('glance', rel_my_gl['password'],
+                                              rel_my_gl['db_host'], 'glance')
+        conf = '/etc/glance/glance-registry.conf'
+
+        expected = {
+            'DEFAULT': {
+                'use_syslog': 'False',
+                'log_file': '/var/log/glance/registry.log',
+                'debug': 'False',
+                'verbose': 'False',
+                'bind_host': '0.0.0.0',
+                'bind_port': '9191'
+            },
+        }
+
+        if self._get_openstack_release() >= self.trusty_kilo:
+            # Kilo or later
+            expected['database'] = {
+                'idle_timeout': '3600',
+                'connection': db_uri
+            }
+        else:
+            # Juno or earlier
+            expected['database'] = {
+                'idle_timeout': '3600',
+                'connection': db_uri
+            }
+
+        expected.update(self._get_keystone_authtoken_expected_dict(rel_ks_gl))
+
+        for section, pairs in expected.iteritems():
+            ret = u.validate_config_data(unit, conf, section, pairs)
+            if ret:
+                message = "glance registry paste config error: {}".format(ret)
+                amulet.raise_status(amulet.FAIL, msg=message)
+
+    def _get_filter_factory_expected_dict(self):
+        """Return expected authtoken filter factory dict for OS release"""
+        if self._get_openstack_release() >= self.trusty_kilo:
+            # Kilo and later
+            val = 'keystonemiddleware.auth_token:filter_factory'
+        else:
+            # Juno and earlier
+            val = 'keystoneclient.middleware.auth_token:filter_factory'
+
+        return {'filter:authtoken': {'paste.filter_factory': val}}
+
+    def test_304_glance_api_paste_auth_config(self):
+        """Verify authtoken section config in glance-api-paste.ini using
+           glance/keystone relation data."""
+        u.log.debug('Checking glance api paste config file...')
+        unit = self.glance_sentry
         conf = '/etc/glance/glance-api-paste.ini'
-        section = 'filter:authtoken'
-
-        if self._get_openstack_release() <= self.precise_havana:
-            # No auth config exists in this file after havana
-            expected = {'admin_user': 'glance',
-                    'admin_password': rel_ks_gl['service_password']}
-
-            ret = u.validate_config_data(unit_gl, conf, section, expected)
+        expected = self._get_filter_factory_expected_dict()
+
+        for section, pairs in expected.iteritems():
+            ret = u.validate_config_data(unit, conf, section, pairs)
             if ret:
-                message = "glance-api-paste auth config error: {}".format(ret)
+                message = "glance api paste config error: {}".format(ret)
                 amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_glance_registry_paste_auth_config(self):
-        '''Verify authtoken section config in glance-registry-paste.ini using
-           glance/keystone relation data.'''
-        unit_gl = self.glance_sentry
-        unit_ks = self.keystone_sentry
-        rel_gl_mq = unit_gl.relation('amqp', 'rabbitmq-server:amqp')
-        rel_ks_gl = unit_ks.relation('identity-service', 'glance:identity-service')
+    def test_306_glance_registry_paste_auth_config(self):
+        """Verify authtoken section config in glance-registry-paste.ini using
+           glance/keystone relation data."""
+        u.log.debug('Checking glance registry paste config file...')
+        unit = self.glance_sentry
         conf = '/etc/glance/glance-registry-paste.ini'
-        section = 'filter:authtoken'
-
-        if self._get_openstack_release() <= self.precise_havana:
-            # No auth config exists in this file after havana
-            expected = {'admin_user': 'glance',
-                    'admin_password': rel_ks_gl['service_password']}
-
-            ret = u.validate_config_data(unit_gl, conf, section, expected)
-            if ret:
-                message = "glance-registry-paste auth config error: {}".format(ret)
-                amulet.raise_status(amulet.FAIL, msg=message)
-
-    def test_glance_registry_default_config(self):
-        '''Verify default section configs in glance-registry.conf'''
-        unit = self.glance_sentry
-        conf = '/etc/glance/glance-registry.conf'
-        expected = {'use_syslog': 'False',
-                    'log_file': '/var/log/glance/registry.log',
-                    'debug': 'False',
-                    'verbose': 'False'}
-        section = 'DEFAULT'
-
-        if self._get_openstack_release() <= self.precise_havana:
-            # Defaults were different before icehouse
-            expected['debug'] = 'True'
-            expected['verbose'] = 'True'
-
-        ret = u.validate_config_data(unit, conf, section, expected)
-        if ret:
-            message = "glance-registry default config error: {}".format(ret)
-            amulet.raise_status(amulet.FAIL, msg=message)
-
-    def test_glance_registry_auth_config(self):
-        '''Verify authtoken section config in glance-registry.conf
-           using glance/keystone relation data.'''
-        unit_gl = self.glance_sentry
-        unit_ks = self.keystone_sentry
-        rel_gl_mq = unit_gl.relation('amqp', 'rabbitmq-server:amqp')
-        rel_ks_gl = unit_ks.relation('identity-service', 'glance:identity-service')
-        conf = '/etc/glance/glance-registry.conf'
-        section = 'keystone_authtoken'
-
-        if self._get_openstack_release() > self.precise_havana:
-            # No auth config exists in this file before icehouse
-            expected = {'admin_user': 'glance',
-                    'admin_password': rel_ks_gl['service_password']}
-
-            ret = u.validate_config_data(unit_gl, conf, section, expected)
-            if ret:
-                message = "glance-registry keystone_authtoken config error: {}".format(ret)
-                amulet.raise_status(amulet.FAIL, msg=message)
-
-    def test_glance_api_database_config(self):
-        '''Verify database config in glance-api.conf and
-           compare with a db uri constructed from relation data.'''
-        unit = self.glance_sentry
-        conf = '/etc/glance/glance-api.conf'
-        relation = self.mysql_sentry.relation('shared-db', 'glance:shared-db')
-        db_uri = "mysql://{}:{}@{}/{}".format('glance', relation['password'],
-                                              relation['db_host'], 'glance')
-        expected = {'connection': db_uri, 'sql_idle_timeout': '3600'}
-        section = 'database'
-
-        if self._get_openstack_release() <= self.precise_havana:
-            # Section and directive for this config changed in icehouse
-            expected = {'sql_connection': db_uri, 'sql_idle_timeout': '3600'}
-            section = 'DEFAULT'
-
-        ret = u.validate_config_data(unit, conf, section, expected) 
-        if ret:
-            message = "glance db config error: {}".format(ret)
-            amulet.raise_status(amulet.FAIL, msg=message)
-
-    def test_glance_registry_database_config(self):
-        '''Verify database config in glance-registry.conf and
-           compare with a db uri constructed from relation data.'''
-        unit = self.glance_sentry
-        conf = '/etc/glance/glance-registry.conf'
-        relation = self.mysql_sentry.relation('shared-db', 'glance:shared-db')
-        db_uri = "mysql://{}:{}@{}/{}".format('glance', relation['password'],
-                                              relation['db_host'], 'glance')
-        expected = {'connection': db_uri, 'sql_idle_timeout': '3600'}
-        section = 'database'
-
-        if self._get_openstack_release() <= self.precise_havana:
-            # Section and directive for this config changed in icehouse
-            expected = {'sql_connection': db_uri, 'sql_idle_timeout': '3600'}
-            section = 'DEFAULT'
-
-        ret = u.validate_config_data(unit, conf, section, expected)
-        if ret:
-            message = "glance db config error: {}".format(ret)
-            amulet.raise_status(amulet.FAIL, msg=message)
-
-    def test_glance_endpoint(self):
-        '''Verify the glance endpoint data.'''
-        endpoints = self.keystone.endpoints.list()
-        admin_port = internal_port = public_port = '9292'
-        expected = {'id': u.not_null,
-                    'region': 'RegionOne',
-                    'adminurl': u.valid_url,
-                    'internalurl': u.valid_url,
-                    'publicurl': u.valid_url,
-                    'service_id': u.not_null}
-        ret = u.validate_endpoint_data(endpoints, admin_port, internal_port,
-                                       public_port, expected)
-
-        if ret:
-            amulet.raise_status(amulet.FAIL,
-                                msg='glance endpoint: {}'.format(ret))
-
-    def test_keystone_endpoint(self):
-        '''Verify the keystone endpoint data.'''
-        endpoints = self.keystone.endpoints.list()
-        admin_port = '35357'
-        internal_port = public_port = '5000'
-        expected = {'id': u.not_null,
-                    'region': 'RegionOne',
-                    'adminurl': u.valid_url,
-                    'internalurl': u.valid_url,
-                    'publicurl': u.valid_url,
-                    'service_id': u.not_null}
-        ret = u.validate_endpoint_data(endpoints, admin_port, internal_port,
-                                       public_port, expected)
-        if ret:
-            amulet.raise_status(amulet.FAIL,
-                                msg='keystone endpoint: {}'.format(ret))
-
-    def _change_config(self):
-        if self._get_openstack_release() > self.precise_havana:
-            self.d.configure('glance', {'debug': 'True'})
-        else:
-            self.d.configure('glance', {'debug': 'False'})
-
-    def _restore_config(self):
-        if self._get_openstack_release() > self.precise_havana:
-            self.d.configure('glance', {'debug': 'False'})
-        else:
-            self.d.configure('glance', {'debug': 'True'})
-
-    def test_z_glance_restart_on_config_change(self):
-        '''Verify that glance is restarted when the config is changed.
-
-           Note(coreycb): The method name with the _z_ is a little odd
-           but it forces the test to run last.  It just makes things
-           easier because restarting services requires re-authorization.
-           '''
-        if self._get_openstack_release() <= self.precise_havana:
-            # /!\ NOTE(beisner): Glance charm before Icehouse doesn't respond
-            # to attempted config changes via juju / juju set.
-            # https://bugs.launchpad.net/charms/+source/glance/+bug/1340307
-            u.log.error('NOTE(beisner): skipping glance restart on config ' +
-                        'change check due to bug 1340307.')
-            return
-
-        # Make config change to trigger a service restart
-        self._change_config()
-
-        if not u.service_restarted(self.glance_sentry, 'glance-api',
-                                   '/etc/glance/glance-api.conf'):
-            self._restore_config()
-            message = "glance service didn't restart after config change"
-            amulet.raise_status(amulet.FAIL, msg=message)
-
-        if not u.service_restarted(self.glance_sentry, 'glance-registry',
-                                   '/etc/glance/glance-registry.conf',
-                                   sleep_time=0):
-            self._restore_config()
-            message = "glance service didn't restart after config change"
-            amulet.raise_status(amulet.FAIL, msg=message)
-
-        # Return to original config
-        self._restore_config()
-
-    def test_users(self):
-        '''Verify expected users.'''
-        user0 = {'name': 'glance',
-                 'enabled': True,
-                 'tenantId': u.not_null,
-                 'id': u.not_null,
-                 'email': 'juju@localhost'}
-        user1 = {'name': 'admin',
-                 'enabled': True,
-                 'tenantId': u.not_null,
-                 'id': u.not_null,
-                 'email': 'juju@localhost'}
-        expected = [user0, user1]
-        actual = self.keystone.users.list()
-
-        ret = u.validate_user_data(expected, actual)
-        if ret:
-            amulet.raise_status(amulet.FAIL, msg=ret)
+        expected = self._get_filter_factory_expected_dict()
+
+        for section, pairs in expected.iteritems():
+            ret = u.validate_config_data(unit, conf, section, pairs)
+            if ret:
+                message = "glance registry paste config error: {}".format(ret)
+                amulet.raise_status(amulet.FAIL, msg=message)
+
+    def test_410_glance_image_create_delete(self):
+        """Create new cirros image in glance, verify, then delete it."""
+        u.log.debug('Creating, checking and deleting glance image...')
+        img_new = u.create_cirros_image(self.glance, "cirros-image-1")
+        img_id = img_new.id
+        u.delete_resource(self.glance.images, img_id, msg="glance image")
+
+    def test_900_glance_restart_on_config_change(self):
+        """Verify that the specified services are restarted when the config
+           is changed."""
+        sentry = self.glance_sentry
+        juju_service = 'glance'
+
+        # Expected default and alternate values
+        set_default = {'use-syslog': 'False'}
+        set_alternate = {'use-syslog': 'True'}
+
+        # Config file affected by juju set config change
+        conf_file = '/etc/glance/glance-api.conf'
+
+        # Services which are expected to restart upon config change
+        services = ['glance-api', 'glance-registry']
+
+        # Make config change, check for service restarts
+        u.log.debug('Making config change on {}...'.format(juju_service))
+        self.d.configure(juju_service, set_alternate)
+
+        sleep_time = 30
+        for s in services:
+            u.log.debug("Checking that service restarted: {}".format(s))
+            if not u.service_restarted(sentry, s,
+                                       conf_file, sleep_time=sleep_time):
+                self.d.configure(juju_service, set_default)
+                msg = "service {} didn't restart after config change".format(s)
+                amulet.raise_status(amulet.FAIL, msg=msg)
+            sleep_time = 0
+
+        self.d.configure(juju_service, set_default)

=== modified file 'tests/charmhelpers/contrib/amulet/utils.py'
--- tests/charmhelpers/contrib/amulet/utils.py	2015-06-19 15:08:48 +0000
+++ tests/charmhelpers/contrib/amulet/utils.py	2015-07-02 12:52:15 +0000
@@ -185,10 +185,23 @@
         for k in expected.keys():
             if not config.has_option(section, k):
                 return "section [{}] is missing option {}".format(section, k)
-            if config.get(section, k) != expected[k]:
-                return "section [{}] {}:{} != expected {}:{}".format(
-                       section, k, config.get(section, k), k, expected[k])
-        return None
+
+            actual = config.get(section, k)
+            v = expected[k]
+            if (isinstance(v, six.string_types) or
+                    isinstance(v, bool) or
+                    isinstance(v, six.integer_types)):
+                # handle explicit values
+                if actual != v:
+                    return "section [{}] {}:{} != expected {}:{}".format(
+                           section, k, actual, k, expected[k])
+            else:
+                # handle not_null, valid_ip boolean comparison methods, etc.
+                if v(actual):
+                    return None
+                else:
+                    return "section [{}] {}:{} != expected {}:{}".format(
+                           section, k, actual, k, expected[k])
 
     def _validate_dict_data(self, expected, actual):
         """Validate dictionary data.
@@ -406,3 +419,123 @@
         """Convert a relative file path to a file URL."""
         _abs_path = os.path.abspath(file_rel_path)
         return urlparse.urlparse(_abs_path, scheme='file').geturl()
+
+    def check_commands_on_units(self, commands, sentry_units):
+        """Check that all commands in a list exit zero on all
+        sentry units in a list.
+
+        :param commands:  list of bash commands
+        :param sentry_units:  list of sentry unit pointers
+        :returns: None if successful; Failure message otherwise
+        """
+        self.log.debug('Checking exit codes for {} commands on {} '
+                       'sentry units...'.format(len(commands),
+                                                len(sentry_units)))
+        for sentry_unit in sentry_units:
+            for cmd in commands:
+                output, code = sentry_unit.run(cmd)
+                if code == 0:
+                    msg = ('{} `{}` returned {} '
+                           '(OK)'.format(sentry_unit.info['unit_name'],
+                                         cmd, code))
+                    self.log.debug(msg)
+                else:
+                    msg = ('{} `{}` returned {} '
+                           '{}'.format(sentry_unit.info['unit_name'],
+                                       cmd, code, output))
+                    return msg
+        return None
+
+    def get_process_id_list(self, sentry_unit, process_name):
+        """Get a list of process ID(s) from a single sentry juju unit
+        for a single process name.
+
+        :param sentry_unit: Pointer to amulet sentry instance (juju unit)
+        :param process_name: Process name
+        :returns: List of process IDs
+        """
+        cmd = 'pidof {}'.format(process_name)
+        output, code = sentry_unit.run(cmd)
+        if code != 0:
+            msg = ('{} `{}` returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, code, output))
+            raise RuntimeError(msg)
+        return str(output).split()
+
+    def get_unit_process_ids(self, unit_processes):
+        """Construct a dict containing unit sentries, process names, and
+        process IDs."""
+        pid_dict = {}
+        for sentry_unit, process_list in unit_processes.iteritems():
+            pid_dict[sentry_unit] = {}
+            for process in process_list:
+                pids = self.get_process_id_list(sentry_unit, process)
+                pid_dict[sentry_unit].update({process: pids})
+        return pid_dict
+
+    def validate_unit_process_ids(self, expected, actual):
+        """Validate process id quantities for services on units."""
+        self.log.debug('Checking units for running processes...')
+        self.log.debug('Expected PIDs: {}'.format(expected))
+        self.log.debug('Actual PIDs: {}'.format(actual))
+
+        if len(actual) != len(expected):
+            msg = ('Unit count mismatch.  expected, actual: {}, '
+                   '{} '.format(len(expected), len(actual)))
+            return msg
+
+        for (e_sentry, e_proc_names) in expected.iteritems():
+            e_sentry_name = e_sentry.info['unit_name']
+            if e_sentry in actual.keys():
+                a_proc_names = actual[e_sentry]
+            else:
+                msg = ('Expected sentry ({}) not found in actual dict data.'
+                       '{}'.format(e_sentry_name, e_sentry))
+                return msg
+
+            if len(e_proc_names.keys()) != len(a_proc_names.keys()):
+                msg = ('Process name count mismatch.  expected, actual: {}, '
+                       '{}'.format(len(expected), len(actual)))
+                return msg
+
+            for (e_proc_name, e_pids_length), (a_proc_name, a_pids) in \
+                    zip(e_proc_names.items(), a_proc_names.items()):
+                if e_proc_name != a_proc_name:
+                    msg = ('Process name mismatch.  expected, actual: {}, '
+                           '{}'.format(e_proc_name, a_proc_name))
+                    return msg
+
+                a_pids_length = len(a_pids)
+                if e_pids_length != a_pids_length:
+                    msg = ('PID count mismatch. {} ({}) expected, actual: {}, '
+                           '{} ({})'.format(e_sentry_name,
+                                            e_proc_name,
+                                            e_pids_length,
+                                            a_pids_length,
+                                            a_pids))
+                    return msg
+                else:
+                    msg = ('PID check OK: {} {} {}: '
+                           '{}'.format(e_sentry_name,
+                                       e_proc_name,
+                                       e_pids_length,
+                                       a_pids))
+                    self.log.debug(msg)
+        return None
+
+    def validate_list_of_identical_dicts(self, list_of_dicts):
+        """Check that all dicts within a list are identical."""
+        hashes = []
+        for _dict in list_of_dicts:
+            hashes.append(hash(frozenset(_dict.items())))
+
+        self.log.debug('Hashes: {}'.format(hashes))
+        if len(set(hashes)) == 1:
+            msg = 'Dicts within list are identical'
+            self.log.debug(msg)
+        else:
+            msg = 'Dicts within list are not identical'
+            return msg
+
+        return None

=== modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py'
--- tests/charmhelpers/contrib/openstack/amulet/deployment.py	2015-06-19 15:08:48 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/deployment.py	2015-07-02 12:52:15 +0000
@@ -79,9 +79,9 @@
         services.append(this_service)
         use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
                       'ceph-osd', 'ceph-radosgw']
-        # Openstack subordinate charms do not expose an origin option as that
-        # is controlled by the principle
-        ignore = ['neutron-openvswitch']
+        # Most OpenStack subordinate charms do not expose an origin option
+        # as that is controlled by the principle.
+        ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch']
 
         if self.openstack:
             for svc in services:
@@ -148,3 +148,35 @@
             return os_origin.split('%s-' % self.series)[1].split('/')[0]
         else:
             return releases[self.series]
+
+    def get_ceph_expected_pools(self, radosgw=False):
+        """Return a list of expected ceph pools based on Ubuntu-OpenStack
+        release and whether ceph radosgw is flagged as present or not."""
+
+        if self._get_openstack_release() >= self.trusty_kilo:
+            # Kilo or later
+            pools = [
+                'rbd',
+                'cinder',
+                'glance'
+            ]
+        else:
+            # Juno or earlier
+            pools = [
+                'data',
+                'metadata',
+                'rbd',
+                'cinder',
+                'glance'
+            ]
+
+        if radosgw:
+            pools.extend([
+                '.rgw.root',
+                '.rgw.control',
+                '.rgw',
+                '.rgw.gc',
+                '.users.uid'
+            ])
+
+        return pools

=== modified file 'tests/charmhelpers/contrib/openstack/amulet/utils.py'
--- tests/charmhelpers/contrib/openstack/amulet/utils.py	2015-06-19 15:08:48 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/utils.py	2015-07-02 12:52:15 +0000
@@ -14,16 +14,19 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import json
 import logging
 import os
 import six
 import time
 import urllib
 
+import cinderclient.v1.client as cinder_client
 import glanceclient.v1.client as glance_client
 import heatclient.v1.client as heat_client
 import keystoneclient.v2_0 as keystone_client
 import novaclient.v1_1.client as nova_client
+import swiftclient
 
 from charmhelpers.contrib.amulet.utils import (
     AmuletUtils
@@ -171,6 +174,15 @@
         self.log.debug('Checking if tenant exists ({})...'.format(tenant))
         return tenant in [t.name for t in keystone.tenants.list()]
 
+    def authenticate_cinder_admin(self, keystone_sentry, username,
+                                  password, tenant):
+        """Authenticates admin user with cinder."""
+        service_ip = \
+            keystone_sentry.relation('shared-db',
+                                     'mysql:shared-db')['private-address']
+        ept = "http://{}:5000/v2.0".format(service_ip.strip().decode('utf-8'))
+        return cinder_client.Client(username, password, tenant, ept)
+
     def authenticate_keystone_admin(self, keystone_sentry, user, password,
                                     tenant):
         """Authenticates admin user with the keystone admin endpoint."""
@@ -212,9 +224,29 @@
         return nova_client.Client(username=user, api_key=password,
                                   project_id=tenant, auth_url=ep)
 
+    def authenticate_swift_user(self, keystone, user, password, tenant):
+        """Authenticates a regular user with swift api."""
+        self.log.debug('Authenticating swift user ({})...'.format(user))
+        ep = keystone.service_catalog.url_for(service_type='identity',
+                                              endpoint_type='publicURL')
+        return swiftclient.Connection(authurl=ep,
+                                      user=user,
+                                      key=password,
+                                      tenant_name=tenant,
+                                      auth_version='2.0')
+
     def create_cirros_image(self, glance, image_name):
-        """Download the latest cirros image and upload it to glance."""
-        self.log.debug('Creating glance image ({})...'.format(image_name))
+        """Download the latest cirros image and upload it to glance,
+        validate and return a resource pointer.
+
+        :param glance: pointer to authenticated glance connection
+        :param image_name: display name for new image
+        :returns: glance image pointer
+        """
+        self.log.debug('Creating glance cirros image '
+                       '({})...'.format(image_name))
+
+        # Download cirros image
         http_proxy = os.getenv('AMULET_HTTP_PROXY')
         self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
         if http_proxy:
@@ -223,33 +255,51 @@
         else:
             opener = urllib.FancyURLopener()
 
-        f = opener.open("http://download.cirros-cloud.net/version/released")
+        f = opener.open('http://download.cirros-cloud.net/version/released')
         version = f.read().strip()
-        cirros_img = "cirros-{}-x86_64-disk.img".format(version)
+        cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
         local_path = os.path.join('tests', cirros_img)
 
         if not os.path.exists(local_path):
-            cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net",
+            cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
                                                   version, cirros_img)
             opener.retrieve(cirros_url, local_path)
         f.close()
 
+        # Create glance image
         with open(local_path) as f:
             image = glance.images.create(name=image_name, is_public=True,
                                          disk_format='qcow2',
                                          container_format='bare', data=f)
-        count = 1
-        status = image.status
-        while status != 'active' and count < 10:
-            time.sleep(3)
-            image = glance.images.get(image.id)
-            status = image.status
-            self.log.debug('image status: {}'.format(status))
-            count += 1
-
-        if status != 'active':
-            self.log.error('image creation timed out')
-            return None
+
+        # Wait for image to reach active status
+        img_id = image.id
+        ret = self.resource_reaches_status(glance.images, img_id,
+                                           expected_stat='active',
+                                           msg='Image status wait')
+        if not ret:
+            msg = 'Glance image failed to reach expected state.'
+            raise RuntimeError(msg)
+
+        # Re-validate new image
+        self.log.debug('Validating image attributes...')
+        val_img_name = glance.images.get(img_id).name
+        val_img_stat = glance.images.get(img_id).status
+        val_img_pub = glance.images.get(img_id).is_public
+        val_img_cfmt = glance.images.get(img_id).container_format
+        val_img_dfmt = glance.images.get(img_id).disk_format
+        msg_attr = ('Image attributes - name:{} public:{} id:{} stat:{} '
+                    'container fmt:{} disk fmt:{}'.format(
+                        val_img_name, val_img_pub, img_id,
+                        val_img_stat, val_img_cfmt, val_img_dfmt))
+
+        if val_img_name == image_name and val_img_stat == 'active' \
+                and val_img_pub is True and val_img_cfmt == 'bare' \
+                and val_img_dfmt == 'qcow2':
+            self.log.debug(msg_attr)
+        else:
+            msg = ('Volume validation failed, {}'.format(msg_attr))
+            raise RuntimeError(msg)
 
         return image
 
@@ -260,22 +310,7 @@
         self.log.warn('/!\\ DEPRECATION WARNING:  use '
                       'delete_resource instead of delete_image.')
         self.log.debug('Deleting glance image ({})...'.format(image))
-        num_before = len(list(glance.images.list()))
-        glance.images.delete(image)
-
-        count = 1
-        num_after = len(list(glance.images.list()))
-        while num_after != (num_before - 1) and count < 10:
-            time.sleep(3)
-            num_after = len(list(glance.images.list()))
-            self.log.debug('number of images: {}'.format(num_after))
-            count += 1
-
-        if num_after != (num_before - 1):
-            self.log.error('image deletion timed out')
-            return False
-
-        return True
+        return self.delete_resource(glance.images, image, msg='glance image')
 
     def create_instance(self, nova, image_name, instance_name, flavor):
         """Create the specified instance."""
@@ -308,22 +343,8 @@
         self.log.warn('/!\\ DEPRECATION WARNING:  use '
                       'delete_resource instead of delete_instance.')
         self.log.debug('Deleting instance ({})...'.format(instance))
-        num_before = len(list(nova.servers.list()))
-        nova.servers.delete(instance)
-
-        count = 1
-        num_after = len(list(nova.servers.list()))
-        while num_after != (num_before - 1) and count < 10:
-            time.sleep(3)
-            num_after = len(list(nova.servers.list()))
-            self.log.debug('number of instances: {}'.format(num_after))
-            count += 1
-
-        if num_after != (num_before - 1):
-            self.log.error('instance deletion timed out')
-            return False
-
-        return True
+        return self.delete_resource(nova.servers, instance,
+                                    msg='nova instance')
 
     def create_or_get_keypair(self, nova, keypair_name="testkey"):
         """Create a new keypair, or return pointer if it already exists."""
@@ -339,6 +360,84 @@
         _keypair = nova.keypairs.create(name=keypair_name)
         return _keypair
 
+    def create_cinder_volume(self, cinder, vol_name="demo-vol", vol_size=1,
+                             img_id=None, src_vol_id=None, snap_id=None):
+        """Create cinder volume, optionally from a glance image, or
+        optionally as a clone of an existing volume, or optionally
+        from a snapshot.  Wait for the new volume status to reach
+        the expected status, validate and return a resource pointer.
+
+        :param vol_name: cinder volume display name
+        :param vol_size: size in gigabytes
+        :param img_id: optional glance image id
+        :param src_vol_id: optional source volume id to clone
+        :param snap_id: optional snapshot id to use
+        :returns: cinder volume pointer
+        """
+        # Handle parameter input
+        if img_id and not src_vol_id and not snap_id:
+            self.log.debug('Creating cinder volume from glance image '
+                           '({})...'.format(img_id))
+            bootable = 'true'
+        elif src_vol_id and not img_id and not snap_id:
+            self.log.debug('Cloning cinder volume...')
+            bootable = cinder.volumes.get(src_vol_id).bootable
+        elif snap_id and not src_vol_id and not img_id:
+            self.log.debug('Creating cinder volume from snapshot...')
+            snap = cinder.volume_snapshots.find(id=snap_id)
+            vol_size = snap.size
+            snap_vol_id = cinder.volume_snapshots.get(snap_id).volume_id
+            bootable = cinder.volumes.get(snap_vol_id).bootable
+        elif not img_id and not src_vol_id and not snap_id:
+            self.log.debug('Creating cinder volume...')
+            bootable = 'false'
+        else:
+            msg = ('Invalid method use - name:{} size:{} img_id:{} '
+                   'src_vol_id:{} snap_id:{}'.format(vol_name, vol_size,
+                                                     img_id, src_vol_id,
+                                                     snap_id))
+            raise RuntimeError(msg)
+
+        # Create new volume
+        try:
+            vol_new = cinder.volumes.create(display_name=vol_name,
+                                            imageRef=img_id,
+                                            size=vol_size,
+                                            source_volid=src_vol_id,
+                                            snapshot_id=snap_id)
+            vol_id = vol_new.id
+        except Exception as e:
+            msg = 'Failed to create volume: {}'.format(e)
+            raise RuntimeError(msg)
+
+        # Wait for volume to reach available status
+        ret = self.resource_reaches_status(cinder.volumes, vol_id,
+                                           expected_stat="available",
+                                           msg="Volume status wait")
+        if not ret:
+            msg = 'Cinder volume failed to reach expected state.'
+            raise RuntimeError(msg)
+
+        # Re-validate new volume
+        self.log.debug('Validating volume attributes...')
+        val_vol_name = cinder.volumes.get(vol_id).display_name
+        val_vol_boot = cinder.volumes.get(vol_id).bootable
+        val_vol_stat = cinder.volumes.get(vol_id).status
+        val_vol_size = cinder.volumes.get(vol_id).size
+        msg_attr = ('Volume attributes - name:{} id:{} stat:{} boot:'
+                    '{} size:{}'.format(val_vol_name, vol_id,
+                                        val_vol_stat, val_vol_boot,
+                                        val_vol_size))
+
+        if val_vol_boot == bootable and val_vol_stat == 'available' \
+                and val_vol_name == vol_name and val_vol_size == vol_size:
+            self.log.debug(msg_attr)
+        else:
+            msg = ('Volume validation failed, {}'.format(msg_attr))
+            raise RuntimeError(msg)
+
+        return vol_new
+
     def delete_resource(self, resource, resource_id,
                         msg="resource", max_wait=120):
         """Delete one openstack resource, such as one instance, keypair,
@@ -350,6 +449,8 @@
         :param max_wait: maximum wait time in seconds
         :returns: True if successful, otherwise False
         """
+        self.log.debug('Deleting OpenStack resource '
+                       '{} ({})'.format(resource_id, msg))
         num_before = len(list(resource.list()))
         resource.delete(resource_id)
 
@@ -411,3 +512,90 @@
             self.log.debug('{} never reached expected status: '
                            '{}'.format(resource_id, expected_stat))
             return False
+
+    def get_ceph_osd_id_cmd(self, index):
+        """Produce a shell command that will return a ceph-osd id."""
+        cmd = ("`initctl list | grep 'ceph-osd ' | awk 'NR=={} {{ print $2 }}'"
+               " | grep -o '[0-9]*'`".format(index + 1))
+        return cmd
+
+    def get_ceph_pools(self, sentry_unit):
+        """Return a dict of ceph pools from a single ceph unit, with
+        pool name as keys, pool id as vals."""
+        pools = {}
+        cmd = 'sudo ceph osd lspools'
+        output, code = sentry_unit.run(cmd)
+        if code != 0:
+            msg = ('{} `{}` returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, code, output))
+            raise RuntimeError(msg)
+
+        # Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance,
+        for pool in str(output).split(','):
+            pool_id_name = pool.split(' ')
+            if len(pool_id_name) == 2:
+                pool_id = pool_id_name[0]
+                pool_name = pool_id_name[1]
+                pools[pool_name] = int(pool_id)
+
+        self.log.debug('Pools on {}: {}'.format(sentry_unit.info['unit_name'],
+                                                pools))
+        return pools
+
+    def get_ceph_df(self, sentry_unit):
+        """Return dict of ceph df json output, including ceph pool state.
+
+        :param sentry_unit: Pointer to amulet sentry instance (juju unit)
+        :returns: Dict of ceph df output
+        """
+        cmd = 'sudo ceph df --format=json'
+        output, code = sentry_unit.run(cmd)
+        if code != 0:
+            msg = ('{} `{}` returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, code, output))
+            raise RuntimeError(msg)
+        return json.loads(output)
+
+    def get_ceph_pool_sample(self, sentry_unit, pool_id=0):
+        """Take a sample of attributes of a ceph pool, returning ceph
+        pool name, object count and disk space used for the specified
+        pool ID number.
+
+        :param sentry_unit: Pointer to amulet sentry instance (juju unit)
+        :param pool_id: Ceph pool ID
+        :returns: List of pool name, object count, kb disk space used
+        """
+        df = self.get_ceph_df(sentry_unit)
+        pool_name = df['pools'][pool_id]['name']
+        obj_count = df['pools'][pool_id]['stats']['objects']
+        kb_used = df['pools'][pool_id]['stats']['kb_used']
+        self.log.debug('Ceph {} pool (ID {}): {} objects, '
+                       '{} kb used'.format(pool_name,
+                                           pool_id,
+                                           obj_count,
+                                           kb_used))
+        return pool_name, obj_count, kb_used
+
+    def validate_ceph_pool_samples(self, samples, sample_type="resource pool"):
+        """Validate ceph pool samples taken over time, such as pool
+        object counts or pool kb used, before adding, after adding, and
+        after deleting items which affect those pool attributes.  The
+        2nd element is expected to be greater than the 1st; 3rd is expected
+        to be less than the 2nd.
+
+        :param samples: List containing 3 data samples
+        :param sample_type: String for logging and usage context
+        :returns: None if successful, Failure message otherwise
+        """
+        original, created, deleted = range(3)
+        if samples[created] <= samples[original] or \
+                samples[deleted] >= samples[created]:
+            msg = ('Ceph {} samples ({}) '
+                   'unexpected.'.format(sample_type, samples))
+            return msg
+        else:
+            self.log.debug('Ceph {} samples (OK): '
+                           '{}'.format(sample_type, samples))
+            return None

=== added file 'tests/tests.yaml'
--- tests/tests.yaml	1970-01-01 00:00:00 +0000
+++ tests/tests.yaml	2015-07-02 12:52:15 +0000
@@ -0,0 +1,18 @@
+bootstrap: true
+reset: true
+virtualenv: true
+makefile:
+    - lint
+    - test
+sources:
+    - ppa:juju/stable
+packages:
+    - amulet
+    - python-amulet
+    - python-cinderclient
+    - python-distro-info
+    - python-glanceclient
+    - python-heatclient
+    - python-keystoneclient
+    - python-novaclient
+    - python-swiftclient