Merge lp:~1chb1n/charms/trusty/neutron-openvswitch/next-amulet-15.11 into lp:~openstack-charmers-archive/charms/trusty/neutron-openvswitch/next

[Ryan Beisner]

Update amulet tests for Liberty. Sync charm helpers. Wait for workload status before testing.

[lp:~1chb1n/charms/trusty/neutron-openvswitch/next-amulet-15.11]

97. By Ryan Beisner on 2015-10-26

update tests

Unmerged revisions

97. By Ryan Beisner on 2015-10-26

update tests

96. By Ryan Beisner on 2015-10-26

update test deployment topology to satisfy workload status; wait for ready state before testing.

95. By Ryan Beisner on 2015-10-26

fix old lint

94. By Ryan Beisner on 2015-10-26

update readme; add liberty amuelt test definitions; move 00-setup to prevent superfluous bootstrap/destroy iteration.

93. By Ryan Beisner on 2015-10-26

sync charmhelpers

=== modified file 'Makefile'
--- Makefile	2015-10-06 16:23:59 +0000
+++ Makefile	2015-10-26 21:25:29 +0000
@@ -13,6 +13,7 @@
 
 functional_test:
 	@echo Starting Amulet tests...
+	@tests/setup/00-setup
 	@juju test -v -p AMULET_HTTP_PROXY,AMULET_OS_VIP --timeout 2700
 
 bin/charm_helpers_sync.py:

=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/deployment.py'
--- hooks/charmhelpers/contrib/openstack/amulet/deployment.py	2015-09-21 22:49:35 +0000
+++ hooks/charmhelpers/contrib/openstack/amulet/deployment.py	2015-10-26 21:25:29 +0000
@@ -14,12 +14,18 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
+import re
+import sys
 import six
 from collections import OrderedDict
 from charmhelpers.contrib.amulet.deployment import (
     AmuletDeployment
 )
 
+DEBUG = logging.DEBUG
+ERROR = logging.ERROR
+
 
 class OpenStackAmuletDeployment(AmuletDeployment):
     """OpenStack amulet deployment.
@@ -28,9 +34,12 @@
        that is specifically for use by OpenStack charms.
        """
 
-    def __init__(self, series=None, openstack=None, source=None, stable=True):
+    def __init__(self, series=None, openstack=None, source=None,
+                 stable=True, log_level=DEBUG):
         """Initialize the deployment environment."""
         super(OpenStackAmuletDeployment, self).__init__(series)
+        self.log = self.get_logger(level=log_level)
+        self.log.info('OpenStackAmuletDeployment:  init')
         self.openstack = openstack
         self.source = source
         self.stable = stable
@@ -38,6 +47,22 @@
         # out.
         self.current_next = "trusty"
 
+    def get_logger(self, name="deployment-logger", level=logging.DEBUG):
+        """Get a logger object that will log to stdout."""
+        log = logging
+        logger = log.getLogger(name)
+        fmt = log.Formatter("%(asctime)s %(funcName)s "
+                            "%(levelname)s: %(message)s")
+
+        handler = log.StreamHandler(stream=sys.stdout)
+        handler.setLevel(level)
+        handler.setFormatter(fmt)
+
+        logger.addHandler(handler)
+        logger.setLevel(level)
+
+        return logger
+
     def _determine_branch_locations(self, other_services):
         """Determine the branch locations for the other services.
 
@@ -45,6 +70,8 @@
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
 
+        self.log.info('OpenStackAmuletDeployment:  determine branch locations')
+
         # Charms outside the lp:~openstack-charmers namespace
         base_charms = ['mysql', 'mongodb', 'nrpe']
 
@@ -82,6 +109,8 @@
 
     def _add_services(self, this_service, other_services):
         """Add services to the deployment and set openstack-origin/source."""
+        self.log.info('OpenStackAmuletDeployment:  adding services')
+
         other_services = self._determine_branch_locations(other_services)
 
         super(OpenStackAmuletDeployment, self)._add_services(this_service,
@@ -111,9 +140,79 @@
 
     def _configure_services(self, configs):
         """Configure all of the services."""
+        self.log.info('OpenStackAmuletDeployment:  configure services')
         for service, config in six.iteritems(configs):
             self.d.configure(service, config)
 
+    def _auto_wait_for_status(self, message=None, exclude_services=None,
+                              include_only=None, timeout=1800):
+        """Wait for all units to have a specific extended status, except
+        for any defined as excluded.  Unless specified via message, any
+        status containing any case of 'ready' will be considered a match.
+
+        Examples of message usage:
+
+          Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
+              message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
+
+          Wait for all units to reach this status (exact match):
+              message = re.compile('^Unit is ready and clustered$')
+
+          Wait for all units to reach any one of these (exact match):
+              message = re.compile('Unit is ready|OK|Ready')
+
+          Wait for at least one unit to reach this status (exact match):
+              message = {'ready'}
+
+        See Amulet's sentry.wait_for_messages() for message usage detail.
+        https://github.com/juju/amulet/blob/master/amulet/sentry.py
+
+        :param message: Expected status match
+        :param exclude_services: List of juju service names to ignore,
+            not to be used in conjuction with include_only.
+        :param include_only: List of juju service names to exclusively check,
+            not to be used in conjuction with exclude_services.
+        :param timeout: Maximum time in seconds to wait for status match
+        :returns: None.  Raises if timeout is hit.
+        """
+        self.log.info('Waiting for extended status on units...')
+
+        all_services = self.d.services.keys()
+
+        if exclude_services and include_only:
+            raise ValueError('exclude_services can not be used '
+                             'with include_only')
+
+        if message:
+            if isinstance(message, re._pattern_type):
+                match = message.pattern
+            else:
+                match = message
+
+            self.log.debug('Custom extended status wait match: '
+                           '{}'.format(match))
+        else:
+            self.log.debug('Default extended status wait match:  contains '
+                           'READY (case-insensitive)')
+            message = re.compile('.*ready.*', re.IGNORECASE)
+
+        if exclude_services:
+            self.log.debug('Excluding services from extended status match: '
+                           '{}'.format(exclude_services))
+        else:
+            exclude_services = []
+
+        if include_only:
+            services = include_only
+        else:
+            services = list(set(all_services) - set(exclude_services))
+
+        self.log.debug('Waiting up to {}s for extended status on services: '
+                       '{}'.format(timeout, services))
+        service_messages = {service: message for service in services}
+        self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
+        self.log.info('OK')
+
     def _get_openstack_release(self):
         """Get openstack release.
 

=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/utils.py'
--- hooks/charmhelpers/contrib/openstack/amulet/utils.py	2015-09-21 22:49:35 +0000
+++ hooks/charmhelpers/contrib/openstack/amulet/utils.py	2015-10-26 21:25:29 +0000
@@ -18,6 +18,7 @@
 import json
 import logging
 import os
+import re
 import six
 import time
 import urllib
@@ -604,7 +605,22 @@
                            '{}'.format(sample_type, samples))
             return None
 
-# rabbitmq/amqp specific helpers:
+    # rabbitmq/amqp specific helpers:
+
+    def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
+        """Wait for rmq units extended status to show cluster readiness,
+        after an optional initial sleep period.  Initial sleep is likely
+        necessary to be effective following a config change, as status
+        message may not instantly update to non-ready."""
+
+        if init_sleep:
+            time.sleep(init_sleep)
+
+        message = re.compile('^Unit is ready and clustered$')
+        deployment._auto_wait_for_status(message=message,
+                                         timeout=timeout,
+                                         include_only=['rabbitmq-server'])
+
     def add_rmq_test_user(self, sentry_units,
                           username="testuser1", password="changeme"):
         """Add a test user via the first rmq juju unit, check connection as
@@ -752,7 +768,7 @@
             self.log.debug('SSL is enabled  @{}:{} '
                            '({})'.format(host, port, unit_name))
             return True
-        elif not port and not conf_ssl:
+        elif not conf_ssl:
             self.log.debug('SSL not enabled @{}:{} '
                            '({})'.format(host, port, unit_name))
             return False
@@ -805,7 +821,10 @@
         if port:
             config['ssl_port'] = port
 
-        deployment.configure('rabbitmq-server', config)
+        deployment.d.configure('rabbitmq-server', config)
+
+        # Wait for unit status
+        self.rmq_wait_for_cluster(deployment)
 
         # Confirm
         tries = 0
@@ -832,7 +851,10 @@
 
         # Disable RMQ SSL
         config = {'ssl': 'off'}
-        deployment.configure('rabbitmq-server', config)
+        deployment.d.configure('rabbitmq-server', config)
+
+        # Wait for unit status
+        self.rmq_wait_for_cluster(deployment)
 
         # Confirm
         tries = 0

=== modified file 'hooks/charmhelpers/contrib/openstack/context.py'
--- hooks/charmhelpers/contrib/openstack/context.py	2015-09-29 20:59:00 +0000
+++ hooks/charmhelpers/contrib/openstack/context.py	2015-10-26 21:25:29 +0000
@@ -952,6 +952,19 @@
                     'config': config}
         return ovs_ctxt
 
+    def midonet_ctxt(self):
+        driver = neutron_plugin_attribute(self.plugin, 'driver',
+                                          self.network_manager)
+        midonet_config = neutron_plugin_attribute(self.plugin, 'config',
+                                                  self.network_manager)
+        mido_ctxt = {'core_plugin': driver,
+                     'neutron_plugin': 'midonet',
+                     'neutron_security_groups': self.neutron_security_groups,
+                     'local_ip': unit_private_ip(),
+                     'config': midonet_config}
+
+        return mido_ctxt
+
     def __call__(self):
         if self.network_manager not in ['quantum', 'neutron']:
             return {}
@@ -973,6 +986,8 @@
             ctxt.update(self.nuage_ctxt())
         elif self.plugin == 'plumgrid':
             ctxt.update(self.pg_ctxt())
+        elif self.plugin == 'midonet':
+            ctxt.update(self.midonet_ctxt())
 
         alchemy_flags = config('neutron-alchemy-flags')
         if alchemy_flags:
@@ -1105,7 +1120,7 @@
 
         ctxt = {
             ... other context ...
-            'subordinate_config': {
+            'subordinate_configuration': {
                 'DEFAULT': {
                     'key1': 'value1',
                 },
@@ -1146,22 +1161,23 @@
                     try:
                         sub_config = json.loads(sub_config)
                     except:
-                        log('Could not parse JSON from subordinate_config '
-                            'setting from %s' % rid, level=ERROR)
+                        log('Could not parse JSON from '
+                            'subordinate_configuration setting from %s'
+                            % rid, level=ERROR)
                         continue
 
                     for service in self.services:
                         if service not in sub_config:
-                            log('Found subordinate_config on %s but it contained'
-                                'nothing for %s service' % (rid, service),
-                                level=INFO)
+                            log('Found subordinate_configuration on %s but it '
+                                'contained nothing for %s service'
+                                % (rid, service), level=INFO)
                             continue
 
                         sub_config = sub_config[service]
                         if self.config_file not in sub_config:
-                            log('Found subordinate_config on %s but it contained'
-                                'nothing for %s' % (rid, self.config_file),
-                                level=INFO)
+                            log('Found subordinate_configuration on %s but it '
+                                'contained nothing for %s'
+                                % (rid, self.config_file), level=INFO)
                             continue
 
                         sub_config = sub_config[self.config_file]

=== modified file 'hooks/charmhelpers/contrib/openstack/neutron.py'
--- hooks/charmhelpers/contrib/openstack/neutron.py	2015-09-28 11:32:27 +0000
+++ hooks/charmhelpers/contrib/openstack/neutron.py	2015-10-26 21:25:29 +0000
@@ -209,6 +209,20 @@
             'server_packages': ['neutron-server',
                                 'neutron-plugin-plumgrid'],
             'server_services': ['neutron-server']
+        },
+        'midonet': {
+            'config': '/etc/neutron/plugins/midonet/midonet.ini',
+            'driver': 'midonet.neutron.plugin.MidonetPluginV2',
+            'contexts': [
+                context.SharedDBContext(user=config('neutron-database-user'),
+                                        database=config('neutron-database'),
+                                        relation_prefix='neutron',
+                                        ssl_dir=NEUTRON_CONF_DIR)],
+            'services': [],
+            'packages': [[headers_package()] + determine_dkms_package()],
+            'server_packages': ['neutron-server',
+                                'python-neutron-plugin-midonet'],
+            'server_services': ['neutron-server']
         }
     }
     if release >= 'icehouse':

=== modified file 'hooks/charmhelpers/contrib/openstack/utils.py'
--- hooks/charmhelpers/contrib/openstack/utils.py	2015-09-28 11:32:27 +0000
+++ hooks/charmhelpers/contrib/openstack/utils.py	2015-10-26 21:25:29 +0000
@@ -121,6 +121,7 @@
     ('2.2.2', 'kilo'),
     ('2.3.0', 'liberty'),
     ('2.4.0', 'liberty'),
+    ('2.5.0', 'liberty'),
 ])
 
 # >= Liberty version->codename mapping
@@ -858,7 +859,9 @@
         if charm_state != 'active' and charm_state != 'unknown':
             state = workload_state_compare(state, charm_state)
             if message:
-                message = "{} {}".format(message, charm_message)
+                charm_message = charm_message.replace("Incomplete relations: ",
+                                                      "")
+                message = "{}, {}".format(message, charm_message)
             else:
                 message = charm_message
 

=== modified file 'hooks/charmhelpers/core/hookenv.py'
--- hooks/charmhelpers/core/hookenv.py	2015-09-28 11:32:27 +0000
+++ hooks/charmhelpers/core/hookenv.py	2015-10-26 21:25:29 +0000
@@ -491,6 +491,19 @@
 
 
 @cached
+def peer_relation_id():
+    '''Get a peer relation id if a peer relation has been joined, else None.'''
+    md = metadata()
+    section = md.get('peers')
+    if section:
+        for key in section:
+            relids = relation_ids(key)
+            if relids:
+                return relids[0]
+    return None
+
+
+@cached
 def relation_to_interface(relation_name):
     """
     Given the name of a relation, return the interface that relation uses.
@@ -820,6 +833,7 @@
 
 def translate_exc(from_exc, to_exc):
     def inner_translate_exc1(f):
+        @wraps(f)
         def inner_translate_exc2(*args, **kwargs):
             try:
                 return f(*args, **kwargs)

=== modified file 'hooks/charmhelpers/core/host.py'
--- hooks/charmhelpers/core/host.py	2015-09-21 22:49:35 +0000
+++ hooks/charmhelpers/core/host.py	2015-10-26 21:25:29 +0000
@@ -566,7 +566,14 @@
         os.chdir(cur)
 
 
-def chownr(path, owner, group, follow_links=True):
+def chownr(path, owner, group, follow_links=True, chowntopdir=False):
+    """
+    Recursively change user and group ownership of files and directories
+    in given path. Doesn't chown path itself by default, only its children.
+
+    :param bool follow_links: Also Chown links if True
+    :param bool chowntopdir: Also chown path itself if True
+    """
     uid = pwd.getpwnam(owner).pw_uid
     gid = grp.getgrnam(group).gr_gid
     if follow_links:
@@ -574,6 +581,10 @@
     else:
         chown = os.lchown
 
+    if chowntopdir:
+        broken_symlink = os.path.lexists(path) and not os.path.exists(path)
+        if not broken_symlink:
+            chown(path, uid, gid)
     for root, dirs, files in os.walk(path):
         for name in dirs + files:
             full = os.path.join(root, name)
@@ -584,3 +595,19 @@
 
 def lchownr(path, owner, group):
     chownr(path, owner, group, follow_links=False)
+
+
+def get_total_ram():
+    '''The total amount of system RAM in bytes.
+
+    This is what is reported by the OS, and may be overcommitted when
+    there are multiple containers hosted on the same machine.
+    '''
+    with open('/proc/meminfo', 'r') as f:
+        for line in f.readlines():
+            if line:
+                key, value, unit = line.split()
+                if key == 'MemTotal:':
+                    assert unit == 'kB', 'Unknown unit'
+                    return int(value) * 1024  # Classic, not KiB.
+        raise NotImplementedError()

=== modified file 'hooks/charmhelpers/core/hugepage.py'
--- hooks/charmhelpers/core/hugepage.py	2015-09-21 22:49:35 +0000
+++ hooks/charmhelpers/core/hugepage.py	2015-10-26 21:25:29 +0000
@@ -46,6 +46,8 @@
     group_info = add_group(group)
     gid = group_info.gr_gid
     add_user_to_group(user, group)
+    if max_map_count < 2 * nr_hugepages:
+        max_map_count = 2 * nr_hugepages
     sysctl_settings = {
         'vm.nr_hugepages': nr_hugepages,
         'vm.max_map_count': max_map_count,

=== modified file 'hooks/charmhelpers/core/templating.py'
--- hooks/charmhelpers/core/templating.py	2015-02-11 18:54:05 +0000
+++ hooks/charmhelpers/core/templating.py	2015-10-26 21:25:29 +0000
@@ -64,5 +64,9 @@
                     level=hookenv.ERROR)
         raise e
     content = template.render(context)
-    host.mkdir(os.path.dirname(target), owner, group, perms=0o755)
+    target_dir = os.path.dirname(target)
+    if not os.path.exists(target_dir):
+        # This is a terrible default directory permission, as the file
+        # or its siblings will often contain secrets.
+        host.mkdir(os.path.dirname(target), owner, group, perms=0o755)
     host.write_file(target, content.encode(encoding), owner, group, perms)

=== modified file 'hooks/charmhelpers/fetch/__init__.py'
--- hooks/charmhelpers/fetch/__init__.py	2015-08-18 21:19:31 +0000
+++ hooks/charmhelpers/fetch/__init__.py	2015-10-26 21:25:29 +0000
@@ -225,12 +225,12 @@
 
 def apt_mark(packages, mark, fatal=False):
     """Flag one or more packages using apt-mark"""
+    log("Marking {} as {}".format(packages, mark))
     cmd = ['apt-mark', mark]
     if isinstance(packages, six.string_types):
         cmd.append(packages)
     else:
         cmd.extend(packages)
-    log("Holding {}".format(packages))
 
     if fatal:
         subprocess.check_call(cmd, universal_newlines=True)

=== modified file 'tests/019-basic-vivid-kilo' (properties changed: -x to +x)
=== added file 'tests/020-basic-trusty-liberty'
--- tests/020-basic-trusty-liberty	1970-01-01 00:00:00 +0000
+++ tests/020-basic-trusty-liberty	2015-10-26 21:25:29 +0000
@@ -0,0 +1,11 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic neutron-openvswitch deployment on trusty-liberty."""
+
+from basic_deployment import NeutronOVSBasicDeployment
+
+if __name__ == '__main__':
+    deployment = NeutronOVSBasicDeployment(series='trusty',
+                                           openstack='cloud:trusty-liberty',
+                                           source='cloud:trusty-updates/liberty')
+    deployment.run_tests()

=== added file 'tests/021-basic-wily-liberty'
--- tests/021-basic-wily-liberty	1970-01-01 00:00:00 +0000
+++ tests/021-basic-wily-liberty	2015-10-26 21:25:29 +0000
@@ -0,0 +1,9 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic neutron-openvswitch deployment on wily-liberty."""
+
+from basic_deployment import NeutronOVSBasicDeployment
+
+if __name__ == '__main__':
+    deployment = NeutronOVSBasicDeployment(series='wily')
+    deployment.run_tests()

=== modified file 'tests/README'
--- tests/README	2015-02-13 12:02:13 +0000
+++ tests/README	2015-10-26 21:25:29 +0000
@@ -1,6 +1,16 @@
 This directory provides Amulet tests that focus on verification of
 neutron-openvswitch deployments.
 
+test_* methods are called in lexical sort order, although each individual test
+should be idempotent, and expected to pass regardless of run order.
+
+Test name convention to ensure desired test order:
+    1xx service and endpoint checks
+    2xx relation checks
+    3xx config checks
+    4xx functional checks
+    9xx restarts and other final check
+
 In order to run tests, you'll need charm-tools installed (in addition to
 juju, of course):
     sudo add-apt-repository ppa:juju/stable

=== modified file 'tests/basic_deployment.py'
--- tests/basic_deployment.py	2015-07-13 16:07:30 +0000
+++ tests/basic_deployment.py	2015-10-26 21:25:29 +0000
@@ -1,8 +1,6 @@
-#!/usr/bin/python
-
 import amulet
 import os
-import time
+import re
 import yaml
 
 from charmhelpers.contrib.openstack.amulet.deployment import (
@@ -11,8 +9,8 @@
 
 from charmhelpers.contrib.openstack.amulet.utils import (
     OpenStackAmuletUtils,
-    DEBUG, # flake8: noqa
-    ERROR
+    DEBUG,
+    # ERROR
 )
 
 # Use DEBUG to turn on debug logging
@@ -36,17 +34,30 @@
         self._add_relations()
         self._configure_services()
         self._deploy()
+        self._wait_for_status()
         self._initialize_tests()
 
+    def _wait_for_status(self):
+        """Wait for unit extended status to indicate ready."""
+        u.log.info('Waiting on extended status checks...')
+        message = re.compile('^Unit is ready$|^Unit is ready and clustered$',
+                             re.IGNORECASE)
+        exclude_services = ['mysql']
+        self._auto_wait_for_status(message=message,
+                                   exclude_services=exclude_services)
+
     def _add_services(self):
         """Add services
 
-           Add the services that we're testing, where neutron-openvswitch is local,
+           Add the services to test, where neutron-openvswitch is local,
            and the rest of the service are from lp branches that are
            compatible with the local charm (e.g. stable or next).
            """
         this_service = {'name': 'neutron-openvswitch'}
         other_services = [{'name': 'nova-compute'},
+                          {'name': 'keystone'},  # satisfy workload stat
+                          {'name': 'mysql'},  # satisfy workload stat
+                          {'name': 'glance'},  # satisfy workload stat
                           {'name': 'rabbitmq-server'},
                           {'name': 'neutron-api'}]
         super(NeutronOVSBasicDeployment, self)._add_services(this_service,
@@ -60,6 +71,16 @@
             'nova-compute:neutron-plugin',
             'neutron-openvswitch:neutron-plugin-api':
             'neutron-api:neutron-plugin-api',
+            # Satisfy workload stat:
+            'neutron-api:identity-service': 'keystone:identity-service',
+            'neutron-api:shared-db': 'mysql:shared-db',
+            'neutron-api:amqp': 'rabbitmq-server:amqp',
+            'nova-compute:amqp': 'rabbitmq-server:amqp',
+            'nova-compute:image-service': 'glance:image-service',
+            'glance:identity-service': 'keystone:identity-service',
+            'glance:shared-db': 'mysql:shared-db',
+            'glance:amqp': 'rabbitmq-server:amqp',
+            'keystone:shared-db': 'mysql:shared-db',
         }
         super(NeutronOVSBasicDeployment, self)._add_relations(relations)
 
@@ -75,19 +96,24 @@
                 openstack_origin_git = {
                     'repositories': [
                         {'name': 'requirements',
-                         'repository': 'git://github.com/openstack/requirements',
+                         'repository':
+                         'git://github.com/openstack/requirements',
                          'branch': branch},
                         {'name': 'neutron-fwaas',
-                         'repository': 'git://github.com/openstack/neutron-fwaas',
+                         'repository':
+                         'git://github.com/openstack/neutron-fwaas',
                          'branch': branch},
                         {'name': 'neutron-lbaas',
-                         'repository': 'git://github.com/openstack/neutron-lbaas',
+                         'repository':
+                         'git://github.com/openstack/neutron-lbaas',
                          'branch': branch},
                         {'name': 'neutron-vpnaas',
-                         'repository': 'git://github.com/openstack/neutron-vpnaas',
+                         'repository':
+                         'git://github.com/openstack/neutron-vpnaas',
                          'branch': branch},
                         {'name': 'neutron',
-                         'repository': 'git://github.com/openstack/neutron',
+                         'repository':
+                         'git://github.com/openstack/neutron',
                          'branch': branch},
                     ],
                     'directory': '/mnt/openstack-git',
@@ -114,7 +140,9 @@
                     'http_proxy': amulet_http_proxy,
                     'https_proxy': amulet_http_proxy,
                 }
-            neutron_ovs_config['openstack-origin-git'] = yaml.dump(openstack_origin_git)
+            neutron_ovs_config['openstack-origin-git'] = \
+                yaml.dump(openstack_origin_git)
+
         configs = {'neutron-openvswitch': neutron_ovs_config}
         super(NeutronOVSBasicDeployment, self)._configure_services(configs)
 
@@ -122,27 +150,34 @@
         """Perform final initialization before tests get run."""
         # Access the sentries for inspecting service units
         self.compute_sentry = self.d.sentry.unit['nova-compute/0']
-        self.rabbitmq_sentry = self.d.sentry.unit['rabbitmq-server/0']
-        self.neutron_api_sentry = self.d.sentry.unit['neutron-api/0']
+        self.rmq_sentry = self.d.sentry.unit['rabbitmq-server/0']
+        self.keystone_sentry = self.d.sentry.unit['keystone/0']
+        self.mysql_sentry = self.d.sentry.unit['mysql/0']
+        self.n_api_sentry = self.d.sentry.unit['neutron-api/0']
+        self.n_ovs_sentry = self.d.sentry.unit['neutron-openvswitch/0']
 
-    def test_services(self):
+    def test_100_services(self):
         """Verify the expected services are running on the corresponding
            service units."""
+        u.log.debug('Checking system services on units...')
 
-        commands = {
-            self.compute_sentry: ['status nova-compute',
-                                  'status neutron-plugin-openvswitch-agent'],
-            self.rabbitmq_sentry: ['service rabbitmq-server status'],
-            self.neutron_api_sentry: ['status neutron-server'],
+        services = {
+            self.compute_sentry: ['nova-compute',
+                                  'neutron-plugin-openvswitch-agent'],
+            self.rmq_sentry: ['rabbitmq-server'],
+            self.n_api_sentry: ['neutron-server'],
         }
 
-        ret = u.validate_services(commands)
+        ret = u.validate_services_by_name(services)
         if ret:
             amulet.raise_status(amulet.FAIL, msg=ret)
 
-    def test_rabbitmq_amqp_relation(self):
+        u.log.debug('OK')
+
+    def test_200_rmq_n_ovs_amqp_relation(self):
         """Verify data in rabbitmq-server/neutron-openvswitch amqp relation"""
-        unit = self.rabbitmq_sentry
+        u.log.debug('Checking rmq:n-ovs amqp relation data...')
+        unit = self.rmq_sentry
         relation = ['amqp', 'neutron-openvswitch:amqp']
         expected = {
             'private-address': u.valid_ip,
@@ -155,8 +190,12 @@
             message = u.relation_error('rabbitmq amqp', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_nova_compute_relation(self):
+        u.log.debug('OK')
+
+    def test_202_nova_compute_relation(self):
         """Verify the nova-compute to neutron-openvswitch relation data"""
+        u.log.debug('Checking nova-compute:n-ovs neutron-plugin '
+                    'relation data...')
         unit = self.compute_sentry
         relation = ['neutron-plugin', 'neutron-openvswitch:neutron-plugin']
         expected = {
@@ -168,9 +207,13 @@
             message = u.relation_error('nova-compute neutron-plugin', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_neutron_api_relation(self):
+        u.log.debug('OK')
+
+    def test_204_neutron_api_relation(self):
         """Verify the neutron-api to neutron-openvswitch relation data"""
-        unit = self.neutron_api_sentry
+        u.log.debug('Checking neuron-api:neutron-ovs neutron-plugin-api '
+                    'relation data...')
+        unit = self.n_api_sentry
         relation = ['neutron-plugin-api',
                     'neutron-openvswitch:neutron-plugin-api']
         expected = {
@@ -182,75 +225,117 @@
             message = u.relation_error('neutron-api neutron-plugin-api', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def process_ret(self, ret=None, message=None):
+        u.log.debug('OK')
+
+    def _process_ret(self, ret=None, message=None):
         if ret:
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def check_ml2_setting_propagation(self, service, charm_key,
-                                      config_file_key, vpair,
-                                      section):
+    def _check_ml2_setting_propagation(self, service, charm_key,
+                                       config_file_key, vpair,
+                                       section):
+
+        # Needs love - test actions not clear in log
         unit = self.compute_sentry
         conf = "/etc/neutron/plugins/ml2/ml2_conf.ini"
         for value in vpair:
             self.d.configure(service, {charm_key: value})
-            time.sleep(60)
+            self._wait_for_status()
+
             ret = u.validate_config_data(unit, conf, section,
                                          {config_file_key: value})
             msg = "Propagation error, expected %s=%s" % (config_file_key,
                                                          value)
-            self.process_ret(ret=ret, message=msg)
-
-    def test_l2pop_propagation(self):
+            self._process_ret(ret=ret, message=msg)
+
+    def test_300_neutron_api_default_config(self):
+        u.log.warn('TEST NOT IMPLEMENTED')
+        pass
+
+    def test_302_l2pop_propagation(self):
         """Verify that neutron-api l2pop setting propagates to neutron-ovs"""
-        self.check_ml2_setting_propagation('neutron-api',
-                                           'l2-population',
-                                           'l2_population',
-                                           ['False', 'True'],
-                                           'agent')
-
-    def test_nettype_propagation(self):
+        u.log.debug('Checking n-api l2pop setting propagation to n-ovs...')
+
+        # Needs love - not idempotent
+        self._check_ml2_setting_propagation('neutron-api',
+                                            'l2-population',
+                                            'l2_population',
+                                            ['False', 'True'],
+                                            'agent')
+        u.log.debug('OK')
+
+    def test_303_nettype_propagation(self):
         """Verify that neutron-api nettype setting propagates to neutron-ovs"""
-        self.check_ml2_setting_propagation('neutron-api',
-                                           'overlay-network-type',
-                                           'tunnel_types',
-                                           ['vxlan', 'gre'],
-                                           'agent')
-
-    def test_secgroup_propagation_local_override(self):
+        u.log.debug('Checking n-api nettype setting propagation to n-ovs...')
+
+        # Needs love - not idempotent
+        self._check_ml2_setting_propagation('neutron-api',
+                                            'overlay-network-type',
+                                            'tunnel_types',
+                                            ['vxlan', 'gre'],
+                                            'agent')
+        u.log.debug('OK')
+
+    def test_304_secgroup_propagation_local_override(self):
         """Verify disable-security-groups overrides what neutron-api says"""
+        u.log.debug('Checking n-api disablee sec group override setting '
+                    'propagation to n-ovs...')
+
+        # Needs love - not idempotent
         unit = self.compute_sentry
         conf = "/etc/neutron/plugins/ml2/ml2_conf.ini"
         self.d.configure('neutron-api', {'neutron-security-groups': 'True'})
         self.d.configure('neutron-openvswitch',
                          {'disable-security-groups': 'True'})
-        time.sleep(30)
+
+        self._wait_for_status()
+
         ret = u.validate_config_data(unit, conf, 'securitygroup',
                                      {'enable_security_group': 'False'})
         msg = "Propagation error, expected %s=%s" % ('enable_security_group',
                                                      'False')
-        self.process_ret(ret=ret, message=msg)
+        self._process_ret(ret=ret, message=msg)
         self.d.configure('neutron-openvswitch',
                          {'disable-security-groups': 'False'})
         self.d.configure('neutron-api', {'neutron-security-groups': 'True'})
-        time.sleep(30)
+
+        self._wait_for_status()
         ret = u.validate_config_data(unit, conf, 'securitygroup',
                                      {'enable_security_group': 'True'})
-
-    def test_z_restart_on_config_change(self):
-        """Verify that the specified services are restarted when the config
-           is changed.
-
-           Note(coreycb): The method name with the _z_ is a little odd
-           but it forces the test to run last.  It just makes things
-           easier because restarting services requires re-authorization.
-           """
-        conf = '/etc/neutron/neutron.conf'
-        self.d.configure('neutron-openvswitch', {'use-syslog': 'True'})
-        if not u.service_restarted(self.compute_sentry,
-                                   'neutron-openvswitch-agent', conf,
-                                   pgrep_full=True, sleep_time=60):
-            self.d.configure('neutron-openvswitch', {'use-syslog': 'False'})
-            msg = ('service neutron-openvswitch-agent did not restart after '
-                   'config change')
-            amulet.raise_status(amulet.FAIL, msg=msg)
-        self.d.configure('neutron-openvswitch', {'use-syslog': 'False'})
+        u.log.debug('OK')
+
+    def test_900_restart_on_config_change(self):
+        """Verify that the specified services are restarted when the
+        config is changed."""
+
+        sentry = self.n_ovs_sentry
+        juju_service = 'neutron-openvswitch'
+
+        # Expected default and alternate values
+        set_default = {'debug': 'False'}
+        set_alternate = {'debug': 'True'}
+
+        # Services which are expected to restart upon config change,
+        # and corresponding config files affected by the change
+        conf_file = '/etc/neutron/neutron.conf'
+        services = {
+            'neutron-openvswitch-agent': conf_file
+        }
+
+        # Make config change, check for svc restart, conf file mod time change
+        u.log.debug('Making config change on {}...'.format(juju_service))
+        mtime = u.get_sentry_time(sentry)
+        self.d.configure(juju_service, set_alternate)
+
+        sleep_time = 60
+        for s, conf_file in services.iteritems():
+            u.log.debug("Checking that service restarted: {}".format(s))
+            if not u.validate_service_config_changed(sentry, mtime, s,
+                                                     conf_file,
+                                                     sleep_time=sleep_time):
+                self.d.configure(juju_service, set_default)
+                msg = "service {} didn't restart after config change".format(s)
+                amulet.raise_status(amulet.FAIL, msg=msg)
+
+        self.d.configure(juju_service, set_default)
+        u.log.debug('OK')

=== modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py'
--- tests/charmhelpers/contrib/openstack/amulet/deployment.py	2015-09-21 22:49:35 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/deployment.py	2015-10-26 21:25:29 +0000
@@ -14,12 +14,18 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
+import re
+import sys
 import six
 from collections import OrderedDict
 from charmhelpers.contrib.amulet.deployment import (
     AmuletDeployment
 )
 
+DEBUG = logging.DEBUG
+ERROR = logging.ERROR
+
 
 class OpenStackAmuletDeployment(AmuletDeployment):
     """OpenStack amulet deployment.
@@ -28,9 +34,12 @@
        that is specifically for use by OpenStack charms.
        """
 
-    def __init__(self, series=None, openstack=None, source=None, stable=True):
+    def __init__(self, series=None, openstack=None, source=None,
+                 stable=True, log_level=DEBUG):
         """Initialize the deployment environment."""
         super(OpenStackAmuletDeployment, self).__init__(series)
+        self.log = self.get_logger(level=log_level)
+        self.log.info('OpenStackAmuletDeployment:  init')
         self.openstack = openstack
         self.source = source
         self.stable = stable
@@ -38,6 +47,22 @@
         # out.
         self.current_next = "trusty"
 
+    def get_logger(self, name="deployment-logger", level=logging.DEBUG):
+        """Get a logger object that will log to stdout."""
+        log = logging
+        logger = log.getLogger(name)
+        fmt = log.Formatter("%(asctime)s %(funcName)s "
+                            "%(levelname)s: %(message)s")
+
+        handler = log.StreamHandler(stream=sys.stdout)
+        handler.setLevel(level)
+        handler.setFormatter(fmt)
+
+        logger.addHandler(handler)
+        logger.setLevel(level)
+
+        return logger
+
     def _determine_branch_locations(self, other_services):
         """Determine the branch locations for the other services.
 
@@ -45,6 +70,8 @@
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
 
+        self.log.info('OpenStackAmuletDeployment:  determine branch locations')
+
         # Charms outside the lp:~openstack-charmers namespace
         base_charms = ['mysql', 'mongodb', 'nrpe']
 
@@ -82,6 +109,8 @@
 
     def _add_services(self, this_service, other_services):
         """Add services to the deployment and set openstack-origin/source."""
+        self.log.info('OpenStackAmuletDeployment:  adding services')
+
         other_services = self._determine_branch_locations(other_services)
 
         super(OpenStackAmuletDeployment, self)._add_services(this_service,
@@ -111,9 +140,79 @@
 
     def _configure_services(self, configs):
         """Configure all of the services."""
+        self.log.info('OpenStackAmuletDeployment:  configure services')
         for service, config in six.iteritems(configs):
             self.d.configure(service, config)
 
+    def _auto_wait_for_status(self, message=None, exclude_services=None,
+                              include_only=None, timeout=1800):
+        """Wait for all units to have a specific extended status, except
+        for any defined as excluded.  Unless specified via message, any
+        status containing any case of 'ready' will be considered a match.
+
+        Examples of message usage:
+
+          Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
+              message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
+
+          Wait for all units to reach this status (exact match):
+              message = re.compile('^Unit is ready and clustered$')
+
+          Wait for all units to reach any one of these (exact match):
+              message = re.compile('Unit is ready|OK|Ready')
+
+          Wait for at least one unit to reach this status (exact match):
+              message = {'ready'}
+
+        See Amulet's sentry.wait_for_messages() for message usage detail.
+        https://github.com/juju/amulet/blob/master/amulet/sentry.py
+
+        :param message: Expected status match
+        :param exclude_services: List of juju service names to ignore,
+            not to be used in conjuction with include_only.
+        :param include_only: List of juju service names to exclusively check,
+            not to be used in conjuction with exclude_services.
+        :param timeout: Maximum time in seconds to wait for status match
+        :returns: None.  Raises if timeout is hit.
+        """
+        self.log.info('Waiting for extended status on units...')
+
+        all_services = self.d.services.keys()
+
+        if exclude_services and include_only:
+            raise ValueError('exclude_services can not be used '
+                             'with include_only')
+
+        if message:
+            if isinstance(message, re._pattern_type):
+                match = message.pattern
+            else:
+                match = message
+
+            self.log.debug('Custom extended status wait match: '
+                           '{}'.format(match))
+        else:
+            self.log.debug('Default extended status wait match:  contains '
+                           'READY (case-insensitive)')
+            message = re.compile('.*ready.*', re.IGNORECASE)
+
+        if exclude_services:
+            self.log.debug('Excluding services from extended status match: '
+                           '{}'.format(exclude_services))
+        else:
+            exclude_services = []
+
+        if include_only:
+            services = include_only
+        else:
+            services = list(set(all_services) - set(exclude_services))
+
+        self.log.debug('Waiting up to {}s for extended status on services: '
+                       '{}'.format(timeout, services))
+        service_messages = {service: message for service in services}
+        self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
+        self.log.info('OK')
+
     def _get_openstack_release(self):
         """Get openstack release.
 

=== modified file 'tests/charmhelpers/contrib/openstack/amulet/utils.py'
--- tests/charmhelpers/contrib/openstack/amulet/utils.py	2015-09-29 20:59:00 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/utils.py	2015-10-26 21:25:29 +0000
@@ -18,6 +18,7 @@
 import json
 import logging
 import os
+import re
 import six
 import time
 import urllib
@@ -604,7 +605,22 @@
                            '{}'.format(sample_type, samples))
             return None
 
-# rabbitmq/amqp specific helpers:
+    # rabbitmq/amqp specific helpers:
+
+    def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
+        """Wait for rmq units extended status to show cluster readiness,
+        after an optional initial sleep period.  Initial sleep is likely
+        necessary to be effective following a config change, as status
+        message may not instantly update to non-ready."""
+
+        if init_sleep:
+            time.sleep(init_sleep)
+
+        message = re.compile('^Unit is ready and clustered$')
+        deployment._auto_wait_for_status(message=message,
+                                         timeout=timeout,
+                                         include_only=['rabbitmq-server'])
+
     def add_rmq_test_user(self, sentry_units,
                           username="testuser1", password="changeme"):
         """Add a test user via the first rmq juju unit, check connection as
@@ -805,7 +821,10 @@
         if port:
             config['ssl_port'] = port
 
-        deployment.configure('rabbitmq-server', config)
+        deployment.d.configure('rabbitmq-server', config)
+
+        # Wait for unit status
+        self.rmq_wait_for_cluster(deployment)
 
         # Confirm
         tries = 0
@@ -832,7 +851,10 @@
 
         # Disable RMQ SSL
         config = {'ssl': 'off'}
-        deployment.configure('rabbitmq-server', config)
+        deployment.d.configure('rabbitmq-server', config)
+
+        # Wait for unit status
+        self.rmq_wait_for_cluster(deployment)
 
         # Confirm
         tries = 0

=== added directory 'tests/setup'
=== renamed file 'tests/00-setup' => 'tests/setup/00-setup'