Merge lp:~plumgrid-team/charms/trusty/plumgrid-director/trunk into lp:charms/trusty/plumgrid-director

[Bilal Baqar]

Trusty - Liberty/Mitaka support added

Mitaka/Liberty changes
- Created new relation with neutron-api-plumgrid
- getting pg creds in config
- nginx conf changes for middleware

[Review Queue (review-queue)]

This item has failed automated testing! Results available here http://juju-ci.vapour.ws:8080/job/charm-bundle-test-lxc/2161/

review: Needs Fixing (automated testing)

[Review Queue (review-queue)]

This item has failed automated testing! Results available here http://juju-ci.vapour.ws:8080/job/charm-bundle-test-aws/2141/

review: Needs Fixing (automated testing)

[Charles Butler (lazypower)]

Greetings Bilal,

This branch doesn't appear to apply cleanly. Can you take a look and resolve the merge conflicts?

review: Needs Fixing

[Bilal Baqar (bbaqar)]

Hey Charles

Thanks for taking the time out to review the merge proposal. I ll deal with the conflicts.

[Review Queue (review-queue)]

This item has failed automated testing! Results available here http://juju-ci.vapour.ws:8080/job/charm-bundle-test-aws/4275/

review: Needs Fixing (automated testing)

[Review Queue (review-queue)]

This item has failed automated testing! Results available here http://juju-ci.vapour.ws:8080/job/charm-bundle-test-lxc/4262/

review: Needs Fixing (automated testing)

[Bilal Baqar (bbaqar)]

Looking at the results. Will provide fix shortly.

=== modified file 'Makefile'
--- Makefile	2016-03-03 20:56:40 +0000
+++ Makefile	2016-05-18 10:01:02 +0000
@@ -4,7 +4,7 @@
 virtualenv:
 	virtualenv .venv
 	.venv/bin/pip install flake8 nose coverage mock pyyaml netifaces \
-        netaddr jinja2
+        netaddr jinja2 pyflakes pep8 six pbr funcsigs psutil
 
 lint: virtualenv
 	.venv/bin/flake8 --exclude hooks/charmhelpers hooks unit_tests tests --ignore E402
 
=== added directory 'bin'
=== added file 'bin/charm_helpers_sync.py'
--- bin/charm_helpers_sync.py	1970-01-01 00:00:00 +0000
+++ bin/charm_helpers_sync.py	2016-05-18 10:01:02 +0000
@@ -0,0 +1,253 @@
+#!/usr/bin/python
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+# Authors:
+#   Adam Gandelman <[e-mail]>
+
+import logging
+import optparse
+import os
+import subprocess
+import shutil
+import sys
+import tempfile
+import yaml
+from fnmatch import fnmatch
+
+import six
+
+CHARM_HELPERS_BRANCH = 'lp:charm-helpers'
+
+
+def parse_config(conf_file):
+    if not os.path.isfile(conf_file):
+        logging.error('Invalid config file: %s.' % conf_file)
+        return False
+    return yaml.load(open(conf_file).read())
+
+
+def clone_helpers(work_dir, branch):
+    dest = os.path.join(work_dir, 'charm-helpers')
+    logging.info('Checking out %s to %s.' % (branch, dest))
+    cmd = ['bzr', 'checkout', '--lightweight', branch, dest]
+    subprocess.check_call(cmd)
+    return dest
+
+
+def _module_path(module):
+    return os.path.join(*module.split('.'))
+
+
+def _src_path(src, module):
+    return os.path.join(src, 'charmhelpers', _module_path(module))
+
+
+def _dest_path(dest, module):
+    return os.path.join(dest, _module_path(module))
+
+
+def _is_pyfile(path):
+    return os.path.isfile(path + '.py')
+
+
+def ensure_init(path):
+    '''
+    ensure directories leading up to path are importable, omitting
+    parent directory, eg path='/hooks/helpers/foo'/:
+        hooks/
+        hooks/helpers/__init__.py
+        hooks/helpers/foo/__init__.py
+    '''
+    for d, dirs, files in os.walk(os.path.join(*path.split('/')[:2])):
+        _i = os.path.join(d, '__init__.py')
+        if not os.path.exists(_i):
+            logging.info('Adding missing __init__.py: %s' % _i)
+            open(_i, 'wb').close()
+
+
+def sync_pyfile(src, dest):
+    src = src + '.py'
+    src_dir = os.path.dirname(src)
+    logging.info('Syncing pyfile: %s -> %s.' % (src, dest))
+    if not os.path.exists(dest):
+        os.makedirs(dest)
+    shutil.copy(src, dest)
+    if os.path.isfile(os.path.join(src_dir, '__init__.py')):
+        shutil.copy(os.path.join(src_dir, '__init__.py'),
+                    dest)
+    ensure_init(dest)
+
+
+def get_filter(opts=None):
+    opts = opts or []
+    if 'inc=*' in opts:
+        # do not filter any files, include everything
+        return None
+
+    def _filter(dir, ls):
+        incs = [opt.split('=').pop() for opt in opts if 'inc=' in opt]
+        _filter = []
+        for f in ls:
+            _f = os.path.join(dir, f)
+
+            if not os.path.isdir(_f) and not _f.endswith('.py') and incs:
+                if True not in [fnmatch(_f, inc) for inc in incs]:
+                    logging.debug('Not syncing %s, does not match include '
+                                  'filters (%s)' % (_f, incs))
+                    _filter.append(f)
+                else:
+                    logging.debug('Including file, which matches include '
+                                  'filters (%s): %s' % (incs, _f))
+            elif (os.path.isfile(_f) and not _f.endswith('.py')):
+                logging.debug('Not syncing file: %s' % f)
+                _filter.append(f)
+            elif (os.path.isdir(_f) and not
+                  os.path.isfile(os.path.join(_f, '__init__.py'))):
+                logging.debug('Not syncing directory: %s' % f)
+                _filter.append(f)
+        return _filter
+    return _filter
+
+
+def sync_directory(src, dest, opts=None):
+    if os.path.exists(dest):
+        logging.debug('Removing existing directory: %s' % dest)
+        shutil.rmtree(dest)
+    logging.info('Syncing directory: %s -> %s.' % (src, dest))
+
+    shutil.copytree(src, dest, ignore=get_filter(opts))
+    ensure_init(dest)
+
+
+def sync(src, dest, module, opts=None):
+
+    # Sync charmhelpers/__init__.py for bootstrap code.
+    sync_pyfile(_src_path(src, '__init__'), dest)
+
+    # Sync other __init__.py files in the path leading to module.
+    m = []
+    steps = module.split('.')[:-1]
+    while steps:
+        m.append(steps.pop(0))
+        init = '.'.join(m + ['__init__'])
+        sync_pyfile(_src_path(src, init),
+                    os.path.dirname(_dest_path(dest, init)))
+
+    # Sync the module, or maybe a .py file.
+    if os.path.isdir(_src_path(src, module)):
+        sync_directory(_src_path(src, module), _dest_path(dest, module), opts)
+    elif _is_pyfile(_src_path(src, module)):
+        sync_pyfile(_src_path(src, module),
+                    os.path.dirname(_dest_path(dest, module)))
+    else:
+        logging.warn('Could not sync: %s. Neither a pyfile or directory, '
+                     'does it even exist?' % module)
+
+
+def parse_sync_options(options):
+    if not options:
+        return []
+    return options.split(',')
+
+
+def extract_options(inc, global_options=None):
+    global_options = global_options or []
+    if global_options and isinstance(global_options, six.string_types):
+        global_options = [global_options]
+    if '|' not in inc:
+        return (inc, global_options)
+    inc, opts = inc.split('|')
+    return (inc, parse_sync_options(opts) + global_options)
+
+
+def sync_helpers(include, src, dest, options=None):
+    if not os.path.isdir(dest):
+        os.makedirs(dest)
+
+    global_options = parse_sync_options(options)
+
+    for inc in include:
+        if isinstance(inc, str):
+            inc, opts = extract_options(inc, global_options)
+            sync(src, dest, inc, opts)
+        elif isinstance(inc, dict):
+            # could also do nested dicts here.
+            for k, v in six.iteritems(inc):
+                if isinstance(v, list):
+                    for m in v:
+                        inc, opts = extract_options(m, global_options)
+                        sync(src, dest, '%s.%s' % (k, inc), opts)
+
+if __name__ == '__main__':
+    parser = optparse.OptionParser()
+    parser.add_option('-c', '--config', action='store', dest='config',
+                      default=None, help='helper config file')
+    parser.add_option('-D', '--debug', action='store_true', dest='debug',
+                      default=False, help='debug')
+    parser.add_option('-b', '--branch', action='store', dest='branch',
+                      help='charm-helpers bzr branch (overrides config)')
+    parser.add_option('-d', '--destination', action='store', dest='dest_dir',
+                      help='sync destination dir (overrides config)')
+    (opts, args) = parser.parse_args()
+
+    if opts.debug:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
+
+    if opts.config:
+        logging.info('Loading charm helper config from %s.' % opts.config)
+        config = parse_config(opts.config)
+        if not config:
+            logging.error('Could not parse config from %s.' % opts.config)
+            sys.exit(1)
+    else:
+        config = {}
+
+    if 'branch' not in config:
+        config['branch'] = CHARM_HELPERS_BRANCH
+    if opts.branch:
+        config['branch'] = opts.branch
+    if opts.dest_dir:
+        config['destination'] = opts.dest_dir
+
+    if 'destination' not in config:
+        logging.error('No destination dir. specified as option or config.')
+        sys.exit(1)
+
+    if 'include' not in config:
+        if not args:
+            logging.error('No modules to sync specified as option or config.')
+            sys.exit(1)
+        config['include'] = []
+        [config['include'].append(a) for a in args]
+
+    sync_options = None
+    if 'options' in config:
+        sync_options = config['options']
+    tmpd = tempfile.mkdtemp()
+    try:
+        checkout = clone_helpers(tmpd, config['branch'])
+        sync_helpers(config['include'], checkout, config['destination'],
+                     options=sync_options)
+    except Exception as e:
+        logging.error("Could not sync: %s" % e)
+        raise e
+    finally:
+        logging.debug('Cleaning up %s' % tmpd)
+        shutil.rmtree(tmpd)
 
=== modified file 'charm-helpers-sync.yaml'
--- charm-helpers-sync.yaml	2015-07-29 18:07:31 +0000
+++ charm-helpers-sync.yaml	2016-05-18 10:01:02 +0000
@@ -3,5 +3,10 @@
 include:
     - core
     - fetch
-    - contrib
+    - contrib.amulet
+    - contrib.hahelpers
+    - contrib.network
+    - contrib.openstack
+    - contrib.python
+    - contrib.storage
     - payload
 
=== modified file 'config.yaml'
--- config.yaml	2016-03-24 12:33:25 +0000
+++ config.yaml	2016-05-18 10:01:02 +0000
@@ -3,6 +3,14 @@
     default: 192.168.100.250
     type: string
     description: IP address of the Director's Management interface. Same IP can be used to access PG Console.
+  plumgrid-username:
+    default: plumgrid
+    type: string
+    description: Username to access PLUMgrid Director
+  plumgrid-password:
+    default: plumgrid
+    type: string
+    description: Password to access PLUMgrid Director
   lcm-ssh-key:
     default: 'null'
     type: string
 
=== modified file 'hooks/charmhelpers/contrib/amulet/deployment.py'
--- hooks/charmhelpers/contrib/amulet/deployment.py	2015-07-29 18:07:31 +0000
+++ hooks/charmhelpers/contrib/amulet/deployment.py	2016-05-18 10:01:02 +0000
@@ -51,7 +51,8 @@
         if 'units' not in this_service:
             this_service['units'] = 1
 
-        self.d.add(this_service['name'], units=this_service['units'])
+        self.d.add(this_service['name'], units=this_service['units'],
+                   constraints=this_service.get('constraints'))
 
         for svc in other_services:
             if 'location' in svc:
@@ -64,7 +65,8 @@
             if 'units' not in svc:
                 svc['units'] = 1
 
-            self.d.add(svc['name'], charm=branch_location, units=svc['units'])
+            self.d.add(svc['name'], charm=branch_location, units=svc['units'],
+                       constraints=svc.get('constraints'))
 
     def _add_relations(self, relations):
         """Add all of the relations for the services."""
 
=== modified file 'hooks/charmhelpers/contrib/amulet/utils.py'
--- hooks/charmhelpers/contrib/amulet/utils.py	2015-07-29 18:07:31 +0000
+++ hooks/charmhelpers/contrib/amulet/utils.py	2016-05-18 10:01:02 +0000
@@ -14,17 +14,25 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
-import amulet
-import ConfigParser
-import distro_info
 import io
+import json
 import logging
 import os
 import re
-import six
+import socket
+import subprocess
 import sys
 import time
-import urlparse
+import uuid
+
+import amulet
+import distro_info
+import six
+from six.moves import configparser
+if six.PY3:
+    from urllib import parse as urlparse
+else:
+    import urlparse
 
 
 class AmuletUtils(object):
@@ -108,7 +116,7 @@
         # /!\ DEPRECATION WARNING (beisner):
         # New and existing tests should be rewritten to use
         # validate_services_by_name() as it is aware of init systems.
-        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+        self.log.warn('DEPRECATION WARNING:  use '
                       'validate_services_by_name instead of validate_services '
                       'due to init system differences.')
 
@@ -142,19 +150,23 @@
 
             for service_name in services_list:
                 if (self.ubuntu_releases.index(release) >= systemd_switch or
-                        service_name == "rabbitmq-server"):
-                    # init is systemd
+                        service_name in ['rabbitmq-server', 'apache2']):
+                    # init is systemd (or regular sysv)
                     cmd = 'sudo service {} status'.format(service_name)
+                    output, code = sentry_unit.run(cmd)
+                    service_running = code == 0
                 elif self.ubuntu_releases.index(release) < systemd_switch:
                     # init is upstart
                     cmd = 'sudo status {}'.format(service_name)
+                    output, code = sentry_unit.run(cmd)
+                    service_running = code == 0 and "start/running" in output
 
-                output, code = sentry_unit.run(cmd)
                 self.log.debug('{} `{}` returned '
                                '{}'.format(sentry_unit.info['unit_name'],
                                            cmd, code))
-                if code != 0:
-                    return "command `{}` returned {}".format(cmd, str(code))
+                if not service_running:
+                    return u"command `{}` returned {} {}".format(
+                        cmd, output, str(code))
         return None
 
     def _get_config(self, unit, filename):
@@ -164,7 +176,7 @@
         # NOTE(beisner):  by default, ConfigParser does not handle options
         # with no value, such as the flags used in the mysql my.cnf file.
         # https://bugs.python.org/issue7005
-        config = ConfigParser.ConfigParser(allow_no_value=True)
+        config = configparser.ConfigParser(allow_no_value=True)
         config.readfp(io.StringIO(file_contents))
         return config
 
@@ -259,33 +271,52 @@
         """Get last modification time of directory."""
         return sentry_unit.directory_stat(directory)['mtime']
 
-    def _get_proc_start_time(self, sentry_unit, service, pgrep_full=False):
-        """Get process' start time.
-
-           Determine start time of the process based on the last modification
-           time of the /proc/pid directory. If pgrep_full is True, the process
-           name is matched against the full command line.
-           """
-        if pgrep_full:
-            cmd = 'pgrep -o -f {}'.format(service)
-        else:
-            cmd = 'pgrep -o {}'.format(service)
-        cmd = cmd + '  | grep  -v pgrep || exit 0'
-        cmd_out = sentry_unit.run(cmd)
-        self.log.debug('CMDout: ' + str(cmd_out))
-        if cmd_out[0]:
-            self.log.debug('Pid for %s %s' % (service, str(cmd_out[0])))
-            proc_dir = '/proc/{}'.format(cmd_out[0].strip())
-            return self._get_dir_mtime(sentry_unit, proc_dir)
+    def _get_proc_start_time(self, sentry_unit, service, pgrep_full=None):
+        """Get start time of a process based on the last modification time
+           of the /proc/pid directory.
+
+        :sentry_unit:  The sentry unit to check for the service on
+        :service:  service name to look for in process table
+        :pgrep_full:  [Deprecated] Use full command line search mode with pgrep
+        :returns:  epoch time of service process start
+        :param commands:  list of bash commands
+        :param sentry_units:  list of sentry unit pointers
+        :returns:  None if successful; Failure message otherwise
+        """
+        if pgrep_full is not None:
+            # /!\ DEPRECATION WARNING (beisner):
+            # No longer implemented, as pidof is now used instead of pgrep.
+            # https://bugs.launchpad.net/charm-helpers/+bug/1474030
+            self.log.warn('DEPRECATION WARNING:  pgrep_full bool is no '
+                          'longer implemented re: lp 1474030.')
+
+        pid_list = self.get_process_id_list(sentry_unit, service)
+        pid = pid_list[0]
+        proc_dir = '/proc/{}'.format(pid)
+        self.log.debug('Pid for {} on {}: {}'.format(
+            service, sentry_unit.info['unit_name'], pid))
+
+        return self._get_dir_mtime(sentry_unit, proc_dir)
 
     def service_restarted(self, sentry_unit, service, filename,
-                          pgrep_full=False, sleep_time=20):
+                          pgrep_full=None, sleep_time=20):
         """Check if service was restarted.
 
            Compare a service's start time vs a file's last modification time
            (such as a config file for that service) to determine if the service
            has been restarted.
            """
+        # /!\ DEPRECATION WARNING (beisner):
+        # This method is prone to races in that no before-time is known.
+        # Use validate_service_config_changed instead.
+
+        # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
+        # used instead of pgrep.  pgrep_full is still passed through to ensure
+        # deprecation WARNS.  lp1474030
+        self.log.warn('DEPRECATION WARNING:  use '
+                      'validate_service_config_changed instead of '
+                      'service_restarted due to known races.')
+
         time.sleep(sleep_time)
         if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >=
                 self._get_file_mtime(sentry_unit, filename)):
@@ -294,78 +325,122 @@
             return False
 
     def service_restarted_since(self, sentry_unit, mtime, service,
-                                pgrep_full=False, sleep_time=20,
-                                retry_count=2):
+                                pgrep_full=None, sleep_time=20,
+                                retry_count=30, retry_sleep_time=10):
         """Check if service was been started after a given time.
 
         Args:
           sentry_unit (sentry): The sentry unit to check for the service on
           mtime (float): The epoch time to check against
           service (string): service name to look for in process table
-          pgrep_full (boolean): Use full command line search mode with pgrep
-          sleep_time (int): Seconds to sleep before looking for process
-          retry_count (int): If service is not found, how many times to retry
+          pgrep_full: [Deprecated] Use full command line search mode with pgrep
+          sleep_time (int): Initial sleep time (s) before looking for file
+          retry_sleep_time (int): Time (s) to sleep between retries
+          retry_count (int): If file is not found, how many times to retry
 
         Returns:
           bool: True if service found and its start time it newer than mtime,
                 False if service is older than mtime or if service was
                 not found.
         """
-        self.log.debug('Checking %s restarted since %s' % (service, mtime))
+        # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
+        # used instead of pgrep.  pgrep_full is still passed through to ensure
+        # deprecation WARNS.  lp1474030
+
+        unit_name = sentry_unit.info['unit_name']
+        self.log.debug('Checking that %s service restarted since %s on '
+                       '%s' % (service, mtime, unit_name))
         time.sleep(sleep_time)
-        proc_start_time = self._get_proc_start_time(sentry_unit, service,
-                                                    pgrep_full)
-        while retry_count > 0 and not proc_start_time:
-            self.log.debug('No pid file found for service %s, will retry %i '
-                           'more times' % (service, retry_count))
-            time.sleep(30)
-            proc_start_time = self._get_proc_start_time(sentry_unit, service,
-                                                        pgrep_full)
-            retry_count = retry_count - 1
+        proc_start_time = None
+        tries = 0
+        while tries <= retry_count and not proc_start_time:
+            try:
+                proc_start_time = self._get_proc_start_time(sentry_unit,
+                                                            service,
+                                                            pgrep_full)
+                self.log.debug('Attempt {} to get {} proc start time on {} '
+                               'OK'.format(tries, service, unit_name))
+            except IOError as e:
+                # NOTE(beisner) - race avoidance, proc may not exist yet.
+                # https://bugs.launchpad.net/charm-helpers/+bug/1474030
+                self.log.debug('Attempt {} to get {} proc start time on {} '
+                               'failed\n{}'.format(tries, service,
+                                                   unit_name, e))
+                time.sleep(retry_sleep_time)
+                tries += 1
 
         if not proc_start_time:
             self.log.warn('No proc start time found, assuming service did '
                           'not start')
             return False
         if proc_start_time >= mtime:
-            self.log.debug('proc start time is newer than provided mtime'
-                           '(%s >= %s)' % (proc_start_time, mtime))
+            self.log.debug('Proc start time is newer than provided mtime'
+                           '(%s >= %s) on %s (OK)' % (proc_start_time,
+                                                      mtime, unit_name))
             return True
         else:
-            self.log.warn('proc start time (%s) is older than provided mtime '
-                          '(%s), service did not restart' % (proc_start_time,
-                                                             mtime))
+            self.log.warn('Proc start time (%s) is older than provided mtime '
+                          '(%s) on %s, service did not '
+                          'restart' % (proc_start_time, mtime, unit_name))
             return False
 
     def config_updated_since(self, sentry_unit, filename, mtime,
-                             sleep_time=20):
+                             sleep_time=20, retry_count=30,
+                             retry_sleep_time=10):
         """Check if file was modified after a given time.
 
         Args:
           sentry_unit (sentry): The sentry unit to check the file mtime on
           filename (string): The file to check mtime of
           mtime (float): The epoch time to check against
-          sleep_time (int): Seconds to sleep before looking for process
+          sleep_time (int): Initial sleep time (s) before looking for file
+          retry_sleep_time (int): Time (s) to sleep between retries
+          retry_count (int): If file is not found, how many times to retry
 
         Returns:
           bool: True if file was modified more recently than mtime, False if
-                file was modified before mtime,
+                file was modified before mtime, or if file not found.
         """
-        self.log.debug('Checking %s updated since %s' % (filename, mtime))
+        unit_name = sentry_unit.info['unit_name']
+        self.log.debug('Checking that %s updated since %s on '
+                       '%s' % (filename, mtime, unit_name))
         time.sleep(sleep_time)
-        file_mtime = self._get_file_mtime(sentry_unit, filename)
+        file_mtime = None
+        tries = 0
+        while tries <= retry_count and not file_mtime:
+            try:
+                file_mtime = self._get_file_mtime(sentry_unit, filename)
+                self.log.debug('Attempt {} to get {} file mtime on {} '
+                               'OK'.format(tries, filename, unit_name))
+            except IOError as e:
+                # NOTE(beisner) - race avoidance, file may not exist yet.
+                # https://bugs.launchpad.net/charm-helpers/+bug/1474030
+                self.log.debug('Attempt {} to get {} file mtime on {} '
+                               'failed\n{}'.format(tries, filename,
+                                                   unit_name, e))
+                time.sleep(retry_sleep_time)
+                tries += 1
+
+        if not file_mtime:
+            self.log.warn('Could not determine file mtime, assuming '
+                          'file does not exist')
+            return False
+
         if file_mtime >= mtime:
             self.log.debug('File mtime is newer than provided mtime '
-                           '(%s >= %s)' % (file_mtime, mtime))
+                           '(%s >= %s) on %s (OK)' % (file_mtime,
+                                                      mtime, unit_name))
             return True
         else:
-            self.log.warn('File mtime %s is older than provided mtime %s'
-                          % (file_mtime, mtime))
+            self.log.warn('File mtime is older than provided mtime'
+                          '(%s < on %s) on %s' % (file_mtime,
+                                                  mtime, unit_name))
             return False
 
     def validate_service_config_changed(self, sentry_unit, mtime, service,
-                                        filename, pgrep_full=False,
-                                        sleep_time=20, retry_count=2):
+                                        filename, pgrep_full=None,
+                                        sleep_time=20, retry_count=30,
+                                        retry_sleep_time=10):
         """Check service and file were updated after mtime
 
         Args:
@@ -373,9 +448,10 @@
           mtime (float): The epoch time to check against
           service (string): service name to look for in process table
           filename (string): The file to check mtime of
-          pgrep_full (boolean): Use full command line search mode with pgrep
-          sleep_time (int): Seconds to sleep before looking for process
+          pgrep_full: [Deprecated] Use full command line search mode with pgrep
+          sleep_time (int): Initial sleep in seconds to pass to test helpers
           retry_count (int): If service is not found, how many times to retry
+          retry_sleep_time (int): Time in seconds to wait between retries
 
         Typical Usage:
             u = OpenStackAmuletUtils(ERROR)
@@ -392,15 +468,27 @@
                 mtime, False if service is older than mtime or if service was
                 not found or if filename was modified before mtime.
         """
-        self.log.debug('Checking %s restarted since %s' % (service, mtime))
-        time.sleep(sleep_time)
-        service_restart = self.service_restarted_since(sentry_unit, mtime,
-                                                       service,
-                                                       pgrep_full=pgrep_full,
-                                                       sleep_time=0,
-                                                       retry_count=retry_count)
-        config_update = self.config_updated_since(sentry_unit, filename, mtime,
-                                                  sleep_time=0)
+
+        # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
+        # used instead of pgrep.  pgrep_full is still passed through to ensure
+        # deprecation WARNS.  lp1474030
+
+        service_restart = self.service_restarted_since(
+            sentry_unit, mtime,
+            service,
+            pgrep_full=pgrep_full,
+            sleep_time=sleep_time,
+            retry_count=retry_count,
+            retry_sleep_time=retry_sleep_time)
+
+        config_update = self.config_updated_since(
+            sentry_unit,
+            filename,
+            mtime,
+            sleep_time=sleep_time,
+            retry_count=retry_count,
+            retry_sleep_time=retry_sleep_time)
+
         return service_restart and config_update
 
     def get_sentry_time(self, sentry_unit):
@@ -418,7 +506,6 @@
         """Return a list of all Ubuntu releases in order of release."""
         _d = distro_info.UbuntuDistroInfo()
         _release_list = _d.all
-        self.log.debug('Ubuntu release list: {}'.format(_release_list))
         return _release_list
 
     def file_to_url(self, file_rel_path):
@@ -450,15 +537,20 @@
                                         cmd, code, output))
         return None
 
-    def get_process_id_list(self, sentry_unit, process_name):
+    def get_process_id_list(self, sentry_unit, process_name,
+                            expect_success=True):
         """Get a list of process ID(s) from a single sentry juju unit
         for a single process name.
 
-        :param sentry_unit: Pointer to amulet sentry instance (juju unit)
+        :param sentry_unit: Amulet sentry instance (juju unit)
         :param process_name: Process name
+        :param expect_success: If False, expect the PID to be missing,
+            raise if it is present.
         :returns: List of process IDs
         """
-        cmd = 'pidof {}'.format(process_name)
+        cmd = 'pidof -x {}'.format(process_name)
+        if not expect_success:
+            cmd += " || exit 0 && exit 1"
         output, code = sentry_unit.run(cmd)
         if code != 0:
             msg = ('{} `{}` returned {} '
@@ -467,14 +559,23 @@
             amulet.raise_status(amulet.FAIL, msg=msg)
         return str(output).split()
 
-    def get_unit_process_ids(self, unit_processes):
+    def get_unit_process_ids(self, unit_processes, expect_success=True):
         """Construct a dict containing unit sentries, process names, and
-        process IDs."""
+        process IDs.
+
+        :param unit_processes: A dictionary of Amulet sentry instance
+            to list of process names.
+        :param expect_success: if False expect the processes to not be
+            running, raise if they are.
+        :returns: Dictionary of Amulet sentry instance to dictionary
+            of process names to PIDs.
+        """
         pid_dict = {}
-        for sentry_unit, process_list in unit_processes.iteritems():
+        for sentry_unit, process_list in six.iteritems(unit_processes):
             pid_dict[sentry_unit] = {}
             for process in process_list:
-                pids = self.get_process_id_list(sentry_unit, process)
+                pids = self.get_process_id_list(
+                    sentry_unit, process, expect_success=expect_success)
                 pid_dict[sentry_unit].update({process: pids})
         return pid_dict
 
@@ -488,7 +589,7 @@
             return ('Unit count mismatch.  expected, actual: {}, '
                     '{} '.format(len(expected), len(actual)))
 
-        for (e_sentry, e_proc_names) in expected.iteritems():
+        for (e_sentry, e_proc_names) in six.iteritems(expected):
             e_sentry_name = e_sentry.info['unit_name']
             if e_sentry in actual.keys():
                 a_proc_names = actual[e_sentry]
@@ -500,22 +601,40 @@
                 return ('Process name count mismatch.  expected, actual: {}, '
                         '{}'.format(len(expected), len(actual)))
 
-            for (e_proc_name, e_pids_length), (a_proc_name, a_pids) in \
+            for (e_proc_name, e_pids), (a_proc_name, a_pids) in \
                     zip(e_proc_names.items(), a_proc_names.items()):
                 if e_proc_name != a_proc_name:
                     return ('Process name mismatch.  expected, actual: {}, '
                             '{}'.format(e_proc_name, a_proc_name))
 
                 a_pids_length = len(a_pids)
-                if e_pids_length != a_pids_length:
-                    return ('PID count mismatch. {} ({}) expected, actual: '
+                fail_msg = ('PID count mismatch. {} ({}) expected, actual: '
                             '{}, {} ({})'.format(e_sentry_name, e_proc_name,
-                                                 e_pids_length, a_pids_length,
+                                                 e_pids, a_pids_length,
                                                  a_pids))
+
+                # If expected is a list, ensure at least one PID quantity match
+                if isinstance(e_pids, list) and \
+                        a_pids_length not in e_pids:
+                    return fail_msg
+                # If expected is not bool and not list,
+                # ensure PID quantities match
+                elif not isinstance(e_pids, bool) and \
+                        not isinstance(e_pids, list) and \
+                        a_pids_length != e_pids:
+                    return fail_msg
+                # If expected is bool True, ensure 1 or more PIDs exist
+                elif isinstance(e_pids, bool) and \
+                        e_pids is True and a_pids_length < 1:
+                    return fail_msg
+                # If expected is bool False, ensure 0 PIDs exist
+                elif isinstance(e_pids, bool) and \
+                        e_pids is False and a_pids_length != 0:
+                    return fail_msg
                 else:
                     self.log.debug('PID check OK: {} {} {}: '
                                    '{}'.format(e_sentry_name, e_proc_name,
-                                               e_pids_length, a_pids))
+                                               e_pids, a_pids))
         return None
 
     def validate_list_of_identical_dicts(self, list_of_dicts):
@@ -531,3 +650,180 @@
             return 'Dicts within list are not identical'
 
         return None
+
+    def validate_sectionless_conf(self, file_contents, expected):
+        """A crude conf parser.  Useful to inspect configuration files which
+        do not have section headers (as would be necessary in order to use
+        the configparser).  Such as openstack-dashboard or rabbitmq confs."""
+        for line in file_contents.split('\n'):
+            if '=' in line:
+                args = line.split('=')
+                if len(args) <= 1:
+                    continue
+                key = args[0].strip()
+                value = args[1].strip()
+                if key in expected.keys():
+                    if expected[key] != value:
+                        msg = ('Config mismatch.  Expected, actual:  {}, '
+                               '{}'.format(expected[key], value))
+                        amulet.raise_status(amulet.FAIL, msg=msg)
+
+    def get_unit_hostnames(self, units):
+        """Return a dict of juju unit names to hostnames."""
+        host_names = {}
+        for unit in units:
+            host_names[unit.info['unit_name']] = \
+                str(unit.file_contents('/etc/hostname').strip())
+        self.log.debug('Unit host names: {}'.format(host_names))
+        return host_names
+
+    def run_cmd_unit(self, sentry_unit, cmd):
+        """Run a command on a unit, return the output and exit code."""
+        output, code = sentry_unit.run(cmd)
+        if code == 0:
+            self.log.debug('{} `{}` command returned {} '
+                           '(OK)'.format(sentry_unit.info['unit_name'],
+                                         cmd, code))
+        else:
+            msg = ('{} `{}` command returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, code, output))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+        return str(output), code
+
+    def file_exists_on_unit(self, sentry_unit, file_name):
+        """Check if a file exists on a unit."""
+        try:
+            sentry_unit.file_stat(file_name)
+            return True
+        except IOError:
+            return False
+        except Exception as e:
+            msg = 'Error checking file {}: {}'.format(file_name, e)
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+    def file_contents_safe(self, sentry_unit, file_name,
+                           max_wait=60, fatal=False):
+        """Get file contents from a sentry unit.  Wrap amulet file_contents
+        with retry logic to address races where a file checks as existing,
+        but no longer exists by the time file_contents is called.
+        Return None if file not found. Optionally raise if fatal is True."""
+        unit_name = sentry_unit.info['unit_name']
+        file_contents = False
+        tries = 0
+        while not file_contents and tries < (max_wait / 4):
+            try:
+                file_contents = sentry_unit.file_contents(file_name)
+            except IOError:
+                self.log.debug('Attempt {} to open file {} from {} '
+                               'failed'.format(tries, file_name,
+                                               unit_name))
+                time.sleep(4)
+                tries += 1
+
+        if file_contents:
+            return file_contents
+        elif not fatal:
+            return None
+        elif fatal:
+            msg = 'Failed to get file contents from unit.'
+            amulet.raise_status(amulet.FAIL, msg)
+
+    def port_knock_tcp(self, host="localhost", port=22, timeout=15):
+        """Open a TCP socket to check for a listening sevice on a host.
+
+        :param host: host name or IP address, default to localhost
+        :param port: TCP port number, default to 22
+        :param timeout: Connect timeout, default to 15 seconds
+        :returns: True if successful, False if connect failed
+        """
+
+        # Resolve host name if possible
+        try:
+            connect_host = socket.gethostbyname(host)
+            host_human = "{} ({})".format(connect_host, host)
+        except socket.error as e:
+            self.log.warn('Unable to resolve address: '
+                          '{} ({}) Trying anyway!'.format(host, e))
+            connect_host = host
+            host_human = connect_host
+
+        # Attempt socket connection
+        try:
+            knock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            knock.settimeout(timeout)
+            knock.connect((connect_host, port))
+            knock.close()
+            self.log.debug('Socket connect OK for host '
+                           '{} on port {}.'.format(host_human, port))
+            return True
+        except socket.error as e:
+            self.log.debug('Socket connect FAIL for'
+                           ' {} port {} ({})'.format(host_human, port, e))
+            return False
+
+    def port_knock_units(self, sentry_units, port=22,
+                         timeout=15, expect_success=True):
+        """Open a TCP socket to check for a listening sevice on each
+        listed juju unit.
+
+        :param sentry_units: list of sentry unit pointers
+        :param port: TCP port number, default to 22
+        :param timeout: Connect timeout, default to 15 seconds
+        :expect_success: True by default, set False to invert logic
+        :returns: None if successful, Failure message otherwise
+        """
+        for unit in sentry_units:
+            host = unit.info['public-address']
+            connected = self.port_knock_tcp(host, port, timeout)
+            if not connected and expect_success:
+                return 'Socket connect failed.'
+            elif connected and not expect_success:
+                return 'Socket connected unexpectedly.'
+
+    def get_uuid_epoch_stamp(self):
+        """Returns a stamp string based on uuid4 and epoch time.  Useful in
+        generating test messages which need to be unique-ish."""
+        return '[{}-{}]'.format(uuid.uuid4(), time.time())
+
+# amulet juju action helpers:
+    def run_action(self, unit_sentry, action,
+                   _check_output=subprocess.check_output,
+                   params=None):
+        """Run the named action on a given unit sentry.
+
+        params a dict of parameters to use
+        _check_output parameter is used for dependency injection.
+
+        @return action_id.
+        """
+        unit_id = unit_sentry.info["unit_name"]
+        command = ["juju", "action", "do", "--format=json", unit_id, action]
+        if params is not None:
+            for key, value in params.iteritems():
+                command.append("{}={}".format(key, value))
+        self.log.info("Running command: %s\n" % " ".join(command))
+        output = _check_output(command, universal_newlines=True)
+        data = json.loads(output)
+        action_id = data[u'Action queued with id']
+        return action_id
+
+    def wait_on_action(self, action_id, _check_output=subprocess.check_output):
+        """Wait for a given action, returning if it completed or not.
+
+        _check_output parameter is used for dependency injection.
+        """
+        command = ["juju", "action", "fetch", "--format=json", "--wait=0",
+                   action_id]
+        output = _check_output(command, universal_newlines=True)
+        data = json.loads(output)
+        return data.get(u"status") == "completed"
+
+    def status_get(self, unit):
+        """Return the current service status of this unit."""
+        raw_status, return_code = unit.run(
+            "status-get --format=json --include-data")
+        if return_code != 0:
+            return ("unknown", "")
+        status = json.loads(raw_status)
+        return (status["status"], status["message"])
 
=== removed directory 'hooks/charmhelpers/contrib/ansible'
=== removed file 'hooks/charmhelpers/contrib/ansible/__init__.py'
--- hooks/charmhelpers/contrib/ansible/__init__.py	2015-07-29 18:07:31 +0000
+++ hooks/charmhelpers/contrib/ansible/__init__.py	1970-01-01 00:00:00 +0000
@@ -1,254 +0,0 @@
-# Copyright 2014-2015 Canonical Limited.
-#
-# This file is part of charm-helpers.
-#
-# charm-helpers is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# charm-helpers is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
-
-# Copyright 2013 Canonical Ltd.
-#
-# Authors:
-#  Charm Helpers Developers <[e-mail]>
-"""Charm Helpers ansible - declare the state of your machines.
-
-This helper enables you to declare your machine state, rather than
-program it procedurally (and have to test each change to your procedures).
-Your install hook can be as simple as::
-
-    {{{
-    import charmhelpers.contrib.ansible
-
-
-    def install():
-        charmhelpers.contrib.ansible.install_ansible_support()
-        charmhelpers.contrib.ansible.apply_playbook('playbooks/install.yaml')
-    }}}
-
-and won't need to change (nor will its tests) when you change the machine
-state.
-
-All of your juju config and relation-data are available as template
-variables within your playbooks and templates. An install playbook looks
-something like::
-
-    {{{
-    ---
-    - hosts: localhost
-      user: root
-
-      tasks:
-        - name: Add private repositories.
-          template:
-            src: ../templates/private-repositories.list.jinja2
-            dest: /etc/apt/sources.list.d/private.list
-
-        - name: Update the cache.
-          apt: update_cache=yes
-
-        - name: Install dependencies.
-          apt: pkg={{ item }}
-          with_items:
-            - python-mimeparse
-            - python-webob
-            - sunburnt
-
-        - name: Setup groups.
-          group: name={{ item.name }} gid={{ item.gid }}
-          with_items:
-            - { name: 'deploy_user', gid: 1800 }
-            - { name: 'service_user', gid: 1500 }
-
-      ...
-    }}}
-
-Read more online about `playbooks`_ and standard ansible `modules`_.
-
-.. _playbooks: http://www.ansibleworks.com/docs/playbooks.html
-.. _modules: http://www.ansibleworks.com/docs/modules.html
-
-A further feature os the ansible hooks is to provide a light weight "action"
-scripting tool. This is a decorator that you apply to a function, and that
-function can now receive cli args, and can pass extra args to the playbook.
-
-e.g.
-
-
[e-mail]()
-def some_action(amount, force="False"):
-    "Usage: some-action AMOUNT [force=True]"  # <-- shown on error
-    # process the arguments
-    # do some calls
-    # return extra-vars to be passed to ansible-playbook
-    return {
-        'amount': int(amount),
-        'type': force,
-    }
-
-You can now create a symlink to hooks.py that can be invoked like a hook, but
-with cli params:
-
-# link actions/some-action to hooks/hooks.py
-
-actions/some-action amount=10 force=true
-
-"""
-import os
-import stat
-import subprocess
-import functools
-
-import charmhelpers.contrib.templating.contexts
-import charmhelpers.core.host
-import charmhelpers.core.hookenv
-import charmhelpers.fetch
-
-
-charm_dir = os.environ.get('CHARM_DIR', '')
-ansible_hosts_path = '/etc/ansible/hosts'
-# Ansible will automatically include any vars in the following
-# file in its inventory when run locally.
-ansible_vars_path = '/etc/ansible/host_vars/localhost'
-
-
-def install_ansible_support(from_ppa=True, ppa_location='ppa:rquillo/ansible'):
-    """Installs the ansible package.
-
-    By default it is installed from the `PPA`_ linked from
-    the ansible `website`_ or from a ppa specified by a charm config..
-
-    .. _PPA: https://launchpad.net/~rquillo/+archive/ansible
-    .. _website: http://docs.ansible.com/intro_installation.html#latest-releases-via-apt-ubuntu
-
-    If from_ppa is empty, you must ensure that the package is available
-    from a configured repository.
-    """
-    if from_ppa:
-        charmhelpers.fetch.add_source(ppa_location)
-        charmhelpers.fetch.apt_update(fatal=True)
-    charmhelpers.fetch.apt_install('ansible')
-    with open(ansible_hosts_path, 'w+') as hosts_file:
-        hosts_file.write('localhost ansible_connection=local')
-
-
-def apply_playbook(playbook, tags=None, extra_vars=None):
-    tags = tags or []
-    tags = ",".join(tags)
-    charmhelpers.contrib.templating.contexts.juju_state_to_yaml(
-        ansible_vars_path, namespace_separator='__',
-        allow_hyphens_in_keys=False, mode=(stat.S_IRUSR | stat.S_IWUSR))
-
-    # we want ansible's log output to be unbuffered
-    env = os.environ.copy()
-    env['PYTHONUNBUFFERED'] = "1"
-    call = [
-        'ansible-playbook',
-        '-c',
-        'local',
-        playbook,
-    ]
-    if tags:
-        call.extend(['--tags', '{}'.format(tags)])
-    if extra_vars:
-        extra = ["%s=%s" % (k, v) for k, v in extra_vars.items()]
-        call.extend(['--extra-vars', " ".join(extra)])
-    subprocess.check_call(call, env=env)
-
-
-class AnsibleHooks(charmhelpers.core.hookenv.Hooks):
-    """Run a playbook with the hook-name as the tag.
-
-    This helper builds on the standard hookenv.Hooks helper,
-    but additionally runs the playbook with the hook-name specified
-    using --tags (ie. running all the tasks tagged with the hook-name).
-
-    Example::
-
-        hooks = AnsibleHooks(playbook_path='playbooks/my_machine_state.yaml')
-
-        # All the tasks within my_machine_state.yaml tagged with 'install'
-        # will be run automatically after do_custom_work()
-        @hooks.hook()
-        def install():
-            do_custom_work()
-
-        # For most of your hooks, you won't need to do anything other
-        # than run the tagged tasks for the hook:
-        @hooks.hook('config-changed', 'start', 'stop')
-        def just_use_playbook():
-            pass
-
-        # As a convenience, you can avoid the above noop function by specifying
-        # the hooks which are handled by ansible-only and they'll be registered
-        # for you:
-        # hooks = AnsibleHooks(
-        #     'playbooks/my_machine_state.yaml',
-        #     default_hooks=['config-changed', 'start', 'stop'])
-
-        if __name__ == "__main__":
-            # execute a hook based on the name the program is called by
-            hooks.execute(sys.argv)
-
-    """
-
-    def __init__(self, playbook_path, default_hooks=None):
-        """Register any hooks handled by ansible."""
-        super(AnsibleHooks, self).__init__()
-
-        self._actions = {}
-        self.playbook_path = playbook_path
-
-        default_hooks = default_hooks or []
-
-        def noop(*args, **kwargs):
-            pass
-
-        for hook in default_hooks:
-            self.register(hook, noop)
-
-    def register_action(self, name, function):
-        """Register a hook"""
-        self._actions[name] = function
-
-    def execute(self, args):
-        """Execute the hook followed by the playbook using the hook as tag."""
-        hook_name = os.path.basename(args[0])
-        extra_vars = None
-        if hook_name in self._actions:
-            extra_vars = self._actions[hook_name](args[1:])
-        else:
-            super(AnsibleHooks, self).execute(args)
-
-        charmhelpers.contrib.ansible.apply_playbook(
-            self.playbook_path, tags=[hook_name], extra_vars=extra_vars)
-
-    def action(self, *action_names):
-        """Decorator, registering them as actions"""
-        def action_wrapper(decorated):
-
-            @functools.wraps(decorated)
-            def wrapper(argv):
-                kwargs = dict(arg.split('=') for arg in argv)
-                try:
-                    return decorated(**kwargs)
-                except TypeError as e:
-                    if decorated.__doc__:
-                        e.args += (decorated.__doc__,)
-                    raise
-
-            self.register_action(decorated.__name__, wrapper)
-            if '_' in decorated.__name__:
-                self.register_action(
-                    decorated.__name__.replace('_', '-'), wrapper)
-
-            return wrapper
-
-        return action_wrapper
 
=== removed directory 'hooks/charmhelpers/contrib/benchmark'
=== removed file 'hooks/charmhelpers/contrib/benchmark/__init__.py'
--- hooks/charmhelpers/contrib/benchmark/__init__.py	2015-07-29 18:07:31 +0000
+++ hooks/charmhelpers/contrib/benchmark/__init__.py	1970-01-01 00:00:00 +0000
@@ -1,126 +0,0 @@
-# Copyright 2014-2015 Canonical Limited.
-#
-# This file is part of charm-helpers.
-#
-# charm-helpers is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# charm-helpers is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
-
-import subprocess
-import time
-import os
-from distutils.spawn import find_executable
-
-from charmhelpers.core.hookenv import (
-    in_relation_hook,
-    relation_ids,
-    relation_set,
-    relation_get,
-)
-
-
-def action_set(key, val):
-    if find_executable('action-set'):
-        action_cmd = ['action-set']
-
-        if isinstance(val, dict):
-            for k, v in iter(val.items()):
-                action_set('%s.%s' % (key, k), v)
-            return True
-
-        action_cmd.append('%s=%s' % (key, val))
-        subprocess.check_call(action_cmd)
-        return True
-    return False
-
-
-class Benchmark():
-    """
-    Helper class for the `benchmark` interface.
-
-    :param list actions: Define the actions that are also benchmarks
-
-    From inside the benchmark-relation-changed hook, you would
-    Benchmark(['memory', 'cpu', 'disk', 'smoke', 'custom'])
-
-    Examples:
-
-        siege = Benchmark(['siege'])
-        siege.start()
-        [... run siege ...]
-        # The higher the score, the better the benchmark
-        siege.set_composite_score(16.70, 'trans/sec', 'desc')
-        siege.finish()
-
-
-    """
-
-    BENCHMARK_CONF = '/etc/benchmark.conf'  # Replaced in testing
-
-    required_keys = [
-        'hostname',
-        'port',
-        'graphite_port',
-        'graphite_endpoint',
-        'api_port'
-    ]
-
-    def __init__(self, benchmarks=None):
-        if in_relation_hook():
-            if benchmarks is not None:
-                for rid in sorted(relation_ids('benchmark')):
-                    relation_set(relation_id=rid, relation_settings={
-                        'benchmarks': ",".join(benchmarks)
-                    })
-
-            # Check the relation data
-            config = {}
-            for key in self.required_keys:
-                val = relation_get(key)
-                if val is not None:
-                    config[key] = val
-                else:
-                    # We don't have all of the required keys
-                    config = {}
-                    break
-
-            if len(config):
-                with open(self.BENCHMARK_CONF, 'w') as f:
-                    for key, val in iter(config.items()):
-                        f.write("%s=%s\n" % (key, val))
-
-    @staticmethod
-    def start():
-        action_set('meta.start', time.strftime('%Y-%m-%dT%H:%M:%SZ'))
-
-        """
-        If the collectd charm is also installed, tell it to send a snapshot
-        of the current profile data.
-        """
-        COLLECT_PROFILE_DATA = '/usr/local/bin/collect-profile-data'
-        if os.path.exists(COLLECT_PROFILE_DATA):
-            subprocess.check_output([COLLECT_PROFILE_DATA])
-
-    @staticmethod
-    def finish():
-        action_set('meta.stop', time.strftime('%Y-%m-%dT%H:%M:%SZ'))
-
-    @staticmethod
-    def set_composite_score(value, units, direction='asc'):
-        """
-        Set the composite score for a benchmark run. This is a single number
-        representative of the benchmark results. This could be the most
-        important metric, or an amalgamation of metric scores.
-        """
-        return action_set(
-            "meta.composite",
-            {'value': value, 'units': units, 'direction': direction}
-        )
 
=== removed directory 'hooks/charmhelpers/contrib/charmhelpers'
=== removed file 'hooks/charmhelpers/contrib/charmhelpers/__init__.py'
--- hooks/charmhelpers/contrib/charmhelpers/__init__.py	2015-07-29 18:07:31 +0000
+++ hooks/charmhelpers/contrib/charmhelpers/__init__.py	1970-01-01 00:00:00 +0000
@@ -1,208 +0,0 @@
-# Copyright 2014-2015 Canonical Limited.
-#
-# This file is part of charm-helpers.
-#
-# charm-helpers is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# charm-helpers is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
-
-# Copyright 2012 Canonical Ltd.  This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-import warnings
-warnings.warn("contrib.charmhelpers is deprecated", DeprecationWarning)  # noqa
-
-import operator
-import tempfile
-import time
-import yaml
-import subprocess
-
-import six
-if six.PY3:
-    from urllib.request import urlopen
-    from urllib.error import (HTTPError, URLError)
-else:
-    from urllib2 import (urlopen, HTTPError, URLError)
-
-"""Helper functions for writing Juju charms in Python."""
-
-__metaclass__ = type
-__all__ = [
-    # 'get_config',             # core.hookenv.config()
-    # 'log',                    # core.hookenv.log()
-    # 'log_entry',              # core.hookenv.log()
-    # 'log_exit',               # core.hookenv.log()
-    # 'relation_get',           # core.hookenv.relation_get()
-    # 'relation_set',           # core.hookenv.relation_set()
-    # 'relation_ids',           # core.hookenv.relation_ids()
-    # 'relation_list',          # core.hookenv.relation_units()
-    # 'config_get',             # core.hookenv.config()
-    # 'unit_get',               # core.hookenv.unit_get()
-    # 'open_port',              # core.hookenv.open_port()
-    # 'close_port',             # core.hookenv.close_port()
-    # 'service_control',        # core.host.service()
-    'unit_info',              # client-side, NOT IMPLEMENTED
-    'wait_for_machine',       # client-side, NOT IMPLEMENTED
-    'wait_for_page_contents',  # client-side, NOT IMPLEMENTED
-    'wait_for_relation',      # client-side, NOT IMPLEMENTED
-    'wait_for_unit',          # client-side, NOT IMPLEMENTED
-]
-
-
-SLEEP_AMOUNT = 0.1
-
-
-# We create a juju_status Command here because it makes testing much,
-# much easier.
-def juju_status():
-    subprocess.check_call(['juju', 'status'])
-
-# re-implemented as charmhelpers.fetch.configure_sources()
-# def configure_source(update=False):
-#    source = config_get('source')
-#    if ((source.startswith('ppa:') or
-#         source.startswith('cloud:') or
-#         source.startswith('http:'))):
-#        run('add-apt-repository', source)
-#    if source.startswith("http:"):
-#        run('apt-key', 'import', config_get('key'))
-#    if update:
-#        run('apt-get', 'update')
-
-
-# DEPRECATED: client-side only
-def make_charm_config_file(charm_config):
-    charm_config_file = tempfile.NamedTemporaryFile(mode='w+')
-    charm_config_file.write(yaml.dump(charm_config))
-    charm_config_file.flush()
-    # The NamedTemporaryFile instance is returned instead of just the name
-    # because we want to take advantage of garbage collection-triggered
-    # deletion of the temp file when it goes out of scope in the caller.
-    return charm_config_file
-
-
-# DEPRECATED: client-side only
-def unit_info(service_name, item_name, data=None, unit=None):
-    if data is None:
-        data = yaml.safe_load(juju_status())
-    service = data['services'].get(service_name)
-    if service is None:
-        # XXX 2012-02-08 gmb:
-        #     This allows us to cope with the race condition that we
-        #     have between deploying a service and having it come up in
-        #     `juju status`. We could probably do with cleaning it up so
-        #     that it fails a bit more noisily after a while.
-        return ''
-    units = service['units']
-    if unit is not None:
-        item = units[unit][item_name]
-    else:
-        # It might seem odd to sort the units here, but we do it to
-        # ensure that when no unit is specified, the first unit for the
-        # service (or at least the one with the lowest number) is the
-        # one whose data gets returned.
-        sorted_unit_names = sorted(units.keys())
-        item = units[sorted_unit_names[0]][item_name]
-    return item
-
-
-# DEPRECATED: client-side only
-def get_machine_data():
-    return yaml.safe_load(juju_status())['machines']
-
-
-# DEPRECATED: client-side only
-def wait_for_machine(num_machines=1, timeout=300):
-    """Wait `timeout` seconds for `num_machines` machines to come up.
-
-    This wait_for... function can be called by other wait_for functions
-    whose timeouts might be too short in situations where only a bare
-    Juju setup has been bootstrapped.
-
-    :return: A tuple of (num_machines, time_taken). This is used for
-             testing.
-    """
-    # You may think this is a hack, and you'd be right. The easiest way
-    # to tell what environment we're working in (LXC vs EC2) is to check
-    # the dns-name of the first machine. If it's localhost we're in LXC
-    # and we can just return here.
-    if get_machine_data()[0]['dns-name'] == 'localhost':
-        return 1, 0
-    start_time = time.time()
-    while True:
-        # Drop the first machine, since it's the Zookeeper and that's
-        # not a machine that we need to wait for. This will only work
-        # for EC2 environments, which is why we return early above if
-        # we're in LXC.
-        machine_data = get_machine_data()
-        non_zookeeper_machines = [
-            machine_data[key] for key in list(machine_data.keys())[1:]]
-        if len(non_zookeeper_machines) >= num_machines:
-            all_machines_running = True
-            for machine in non_zookeeper_machines:
-                if machine.get('instance-state') != 'running':
-                    all_machines_running = False
-                    break
-            if all_machines_running:
-                break
-        if time.time() - start_time >= timeout:
-            raise RuntimeError('timeout waiting for service to start')
-        time.sleep(SLEEP_AMOUNT)
-    return num_machines, time.time() - start_time
-
-
-# DEPRECATED: client-side only
-def wait_for_unit(service_name, timeout=480):
-    """Wait `timeout` seconds for a given service name to come up."""
-    wait_for_machine(num_machines=1)
-    start_time = time.time()
-    while True:
-        state = unit_info(service_name, 'agent-state')
-        if 'error' in state or state == 'started':
-            break
-        if time.time() - start_time >= timeout:
-            raise RuntimeError('timeout waiting for service to start')
-        time.sleep(SLEEP_AMOUNT)
-    if state != 'started':
-        raise RuntimeError('unit did not start, agent-state: ' + state)
-
-
-# DEPRECATED: client-side only
-def wait_for_relation(service_name, relation_name, timeout=120):
-    """Wait `timeout` seconds for a given relation to come up."""
-    start_time = time.time()
-    while True:
-        relation = unit_info(service_name, 'relations').get(relation_name)
-        if relation is not None and relation['state'] == 'up':
-            break
-        if time.time() - start_time >= timeout:
-            raise RuntimeError('timeout waiting for relation to be up')
-        time.sleep(SLEEP_AMOUNT)
-
-
-# DEPRECATED: client-side only
-def wait_for_page_contents(url, contents, timeout=120, validate=None):
-    if validate is None:
-        validate = operator.contains
-    start_time = time.time()
-    while True:
-        try:
-            stream = urlopen(url)
-        except (HTTPError, URLError):
-            pass
-        else:
-            page = stream.read()
-            if validate(page, contents):
-                return page
-        if time.time() - start_time >= timeout:
-            raise RuntimeError('timeout waiting for contents of ' + url)
-        time.sleep(SLEEP_AMOUNT)
 
=== removed directory 'hooks/charmhelpers/contrib/charmsupport'
=== removed file 'hooks/charmhelpers/contrib/charmsupport/__init__.py'
--- hooks/charmhelpers/contrib/charmsupport/__init__.py	2015-07-29 18:07:31 +0000
+++ hooks/charmhelpers/contrib/charmsupport/__init__.py	1970-01-01 00:00:00 +0000
@@ -1,15 +0,0 @@
-# Copyright 2014-2015 Canonical Limited.
-#
-# This file is part of charm-helpers.
-#
-# charm-helpers is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# charm-helpers is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
=== removed file 'hooks/charmhelpers/contrib/charmsupport/nrpe.py'
--- hooks/charmhelpers/contrib/charmsupport/nrpe.py	2015-07-29 18:07:31 +0000
+++ hooks/charmhelpers/contrib/charmsupport/nrpe.py	1970-01-01 00:00:00 +0000
@@ -1,360 +0,0 @@
-# Copyright 2014-2015 Canonical Limited.
-#
-# This file is part of charm-helpers.
-#
-# charm-helpers is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# charm-helpers is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
-
-"""Compatibility with the nrpe-external-master charm"""
-# Copyright 2012 Canonical Ltd.
-#
-# Authors:
-#  Matthew Wedgwood <[e-mail]>
-
-import subprocess
-import pwd
-import grp
-import os
-import glob
-import shutil
-import re
-import shlex
-import yaml
-
-from charmhelpers.core.hookenv import (
-    config,
-    local_unit,
-    log,
-    relation_ids,
-    relation_set,
-    relations_of_type,
-)
-
-from charmhelpers.core.host import service
-
-# This module adds compatibility with the nrpe-external-master and plain nrpe
-# subordinate charms. To use it in your charm:
-#
-# 1. Update metadata.yaml
-#
-#   provides:
-#     (...)
-#     nrpe-external-master:
-#       interface: nrpe-external-master
-#       scope: container
-#
-#   and/or
-#
-#   provides:
-#     (...)
-#     local-monitors:
-#       interface: local-monitors
-#       scope: container
-
-#
-# 2. Add the following to config.yaml
-#
-#    nagios_context:
-#      default: "juju"
-#      type: string
-#      description: |
-#        Used by the nrpe subordinate charms.
-#        A string that will be prepended to instance name to set the host name
-#        in nagios. So for instance the hostname would be something like:
-#            juju-myservice-0
-#        If you're running multiple environments with the same services in them
-#        this allows you to differentiate between them.
-#    nagios_servicegroups:
-#      default: ""
-#      type: string
-#      description: |
-#        A comma-separated list of nagios servicegroups.
-#        If left empty, the nagios_context will be used as the servicegroup
-#
-# 3. Add custom checks (Nagios plugins) to files/nrpe-external-master
-#
-# 4. Update your hooks.py with something like this:
-#
-#    from charmsupport.nrpe import NRPE
-#    (...)
-#    def update_nrpe_config():
-#        nrpe_compat = NRPE()
-#        nrpe_compat.add_check(
-#            shortname = "myservice",
-#            description = "Check MyService",
-#            check_cmd = "check_http -w 2 -c 10 http://localhost"
-#            )
-#        nrpe_compat.add_check(
-#            "myservice_other",
-#            "Check for widget failures",
-#            check_cmd = "/srv/myapp/scripts/widget_check"
-#            )
-#        nrpe_compat.write()
-#
-#    def config_changed():
-#        (...)
-#        update_nrpe_config()
-#
-#    def nrpe_external_master_relation_changed():
-#        update_nrpe_config()
-#
-#    def local_monitors_relation_changed():
-#        update_nrpe_config()
-#
-# 5. ln -s hooks.py nrpe-external-master-relation-changed
-#    ln -s hooks.py local-monitors-relation-changed
-
-
-class CheckException(Exception):
-    pass
-
-
-class Check(object):
-    shortname_re = '[A-Za-z0-9-_]+$'
-    service_template = ("""
-#---------------------------------------------------
-# This file is Juju managed
-#---------------------------------------------------
-define service {{
-    use                             active-service
-    host_name                       {nagios_hostname}
-    service_description             {nagios_hostname}[{shortname}] """
-                        """{description}
-    check_command                   check_nrpe!{command}
-    servicegroups                   {nagios_servicegroup}
-}}
-""")
-
-    def __init__(self, shortname, description, check_cmd):
-        super(Check, self).__init__()
-        # XXX: could be better to calculate this from the service name
-        if not re.match(self.shortname_re, shortname):
-            raise CheckException("shortname must match {}".format(
-                Check.shortname_re))
-        self.shortname = shortname
-        self.command = "check_{}".format(shortname)
-        # Note: a set of invalid characters is defined by the
-        # Nagios server config
-        # The default is: illegal_object_name_chars=`~!$%^&*"|'<>?,()=
-        self.description = description
-        self.check_cmd = self._locate_cmd(check_cmd)
-
-    def _locate_cmd(self, check_cmd):
-        search_path = (
-            '/usr/lib/nagios/plugins',
-            '/usr/local/lib/nagios/plugins',
-        )
-        parts = shlex.split(check_cmd)
-        for path in search_path:
-            if os.path.exists(os.path.join(path, parts[0])):
-                command = os.path.join(path, parts[0])
-                if len(parts) > 1:
-                    command += " " + " ".join(parts[1:])
-                return command
-        log('Check command not found: {}'.format(parts[0]))
-        return ''
-
-    def write(self, nagios_context, hostname, nagios_servicegroups):
-        nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format(
-            self.command)
-        with open(nrpe_check_file, 'w') as nrpe_check_config:
-            nrpe_check_config.write("# check {}\n".format(self.shortname))
-            nrpe_check_config.write("command[{}]={}\n".format(
-                self.command, self.check_cmd))
-
-        if not os.path.exists(NRPE.nagios_exportdir):
-            log('Not writing service config as {} is not accessible'.format(
-                NRPE.nagios_exportdir))
-        else:
-            self.write_service_config(nagios_context, hostname,
-                                      nagios_servicegroups)
-
-    def write_service_config(self, nagios_context, hostname,
-                             nagios_servicegroups):
-        for f in os.listdir(NRPE.nagios_exportdir):
-            if re.search('.*{}.cfg'.format(self.command), f):
-                os.remove(os.path.join(NRPE.nagios_exportdir, f))
-
-        templ_vars = {
-            'nagios_hostname': hostname,
-            'nagios_servicegroup': nagios_servicegroups,
-            'description': self.description,
-            'shortname': self.shortname,
-            'command': self.command,
-        }
-        nrpe_service_text = Check.service_template.format(**templ_vars)
-        nrpe_service_file = '{}/service__{}_{}.cfg'.format(
-            NRPE.nagios_exportdir, hostname, self.command)
-        with open(nrpe_service_file, 'w') as nrpe_service_config:
-            nrpe_service_config.write(str(nrpe_service_text))
-
-    def run(self):
-        subprocess.call(self.check_cmd)
-
-
-class NRPE(object):
-    nagios_logdir = '/var/log/nagios'
-    nagios_exportdir = '/var/lib/nagios/export'
-    nrpe_confdir = '/etc/nagios/nrpe.d'
-
-    def __init__(self, hostname=None):
-        super(NRPE, self).__init__()
-        self.config = config()
-        self.nagios_context = self.config['nagios_context']
-        if 'nagios_servicegroups' in self.config and self.config['nagios_servicegroups']:
-            self.nagios_servicegroups = self.config['nagios_servicegroups']
-        else:
-            self.nagios_servicegroups = self.nagios_context
-        self.unit_name = local_unit().replace('/', '-')
-        if hostname:
-            self.hostname = hostname
-        else:
-            self.hostname = "{}-{}".format(self.nagios_context, self.unit_name)
-        self.checks = []
-
-    def add_check(self, *args, **kwargs):
-        self.checks.append(Check(*args, **kwargs))
-
-    def write(self):
-        try:
-            nagios_uid = pwd.getpwnam('nagios').pw_uid
-            nagios_gid = grp.getgrnam('nagios').gr_gid
-        except:
-            log("Nagios user not set up, nrpe checks not updated")
-            return
-
-        if not os.path.exists(NRPE.nagios_logdir):
-            os.mkdir(NRPE.nagios_logdir)
-            os.chown(NRPE.nagios_logdir, nagios_uid, nagios_gid)
-
-        nrpe_monitors = {}
-        monitors = {"monitors": {"remote": {"nrpe": nrpe_monitors}}}
-        for nrpecheck in self.checks:
-            nrpecheck.write(self.nagios_context, self.hostname,
-                            self.nagios_servicegroups)
-            nrpe_monitors[nrpecheck.shortname] = {
-                "command": nrpecheck.command,
-            }
-
-        service('restart', 'nagios-nrpe-server')
-
-        monitor_ids = relation_ids("local-monitors") + \
-            relation_ids("nrpe-external-master")
-        for rid in monitor_ids:
-            relation_set(relation_id=rid, monitors=yaml.dump(monitors))
-
-
-def get_nagios_hostcontext(relation_name='nrpe-external-master'):
-    """
-    Query relation with nrpe subordinate, return the nagios_host_context
-
-    :param str relation_name: Name of relation nrpe sub joined to
-    """
-    for rel in relations_of_type(relation_name):
-        if 'nagios_hostname' in rel:
-            return rel['nagios_host_context']
-
-
-def get_nagios_hostname(relation_name='nrpe-external-master'):
-    """
-    Query relation with nrpe subordinate, return the nagios_hostname
-
-    :param str relation_name: Name of relation nrpe sub joined to
-    """
-    for rel in relations_of_type(relation_name):
-        if 'nagios_hostname' in rel:
-            return rel['nagios_hostname']
-
-
-def get_nagios_unit_name(relation_name='nrpe-external-master'):
-    """
-    Return the nagios unit name prepended with host_context if needed
-
-    :param str relation_name: Name of relation nrpe sub joined to
-    """
-    host_context = get_nagios_hostcontext(relation_name)
-    if host_context:
-        unit = "%s:%s" % (host_context, local_unit())
-    else:
-        unit = local_unit()
-    return unit
-
-
-def add_init_service_checks(nrpe, services, unit_name):
-    """
-    Add checks for each service in list
-
-    :param NRPE nrpe: NRPE object to add check to
-    :param list services: List of services to check
-    :param str unit_name: Unit name to use in check description
-    """
-    for svc in services:
-        upstart_init = '/etc/init/%s.conf' % svc
-        sysv_init = '/etc/init.d/%s' % svc
-        if os.path.exists(upstart_init):
-            nrpe.add_check(
-                shortname=svc,
-                description='process check {%s}' % unit_name,
-                check_cmd='check_upstart_job %s' % svc
-            )
-        elif os.path.exists(sysv_init):
-            cronpath = '/etc/cron.d/nagios-service-check-%s' % svc
-            cron_file = ('*/5 * * * * root '
-                         '/usr/local/lib/nagios/plugins/check_exit_status.pl '
-                         '-s /etc/init.d/%s status > '
-                         '/var/lib/nagios/service-check-%s.txt\n' % (svc,
-                                                                     svc)
-                         )
-            f = open(cronpath, 'w')
-            f.write(cron_file)
-            f.close()
-            nrpe.add_check(
-                shortname=svc,
-                description='process check {%s}' % unit_name,
-                check_cmd='check_status_file.py -f '
-                          '/var/lib/nagios/service-check-%s.txt' % svc,
-            )
-
-
-def copy_nrpe_checks():
-    """
-    Copy the nrpe checks into place
-
-    """
-    NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins'
-    nrpe_files_dir = os.path.join(os.getenv('CHARM_DIR'), 'hooks',
-                                  'charmhelpers', 'contrib', 'openstack',
-                                  'files')
-
-    if not os.path.exists(NAGIOS_PLUGINS):
-        os.makedirs(NAGIOS_PLUGINS)
-    for fname in glob.glob(os.path.join(nrpe_files_dir, "check_*")):
-        if os.path.isfile(fname):
-            shutil.copy2(fname,
-                         os.path.join(NAGIOS_PLUGINS, os.path.basename(fname)))
-
-
-def add_haproxy_checks(nrpe, unit_name):
-    """
-    Add checks for each service in list
-
-    :param NRPE nrpe: NRPE object to add check to
-    :param str unit_name: Unit name to use in check description
-    """
-    nrpe.add_check(
-        shortname='haproxy_servers',
-        description='Check HAProxy {%s}' % unit_name,
-        check_cmd='check_haproxy.sh')
-    nrpe.add_check(
-        shortname='haproxy_queue',
-        description='Check HAProxy queue depth {%s}' % unit_name,
-        check_cmd='check_haproxy_queue_depth.sh')
 
=== removed file 'hooks/charmhelpers/contrib/charmsupport/volumes.py'
--- hooks/charmhelpers/contrib/charmsupport/volumes.py	2015-07-29 18:07:31 +0000
+++ hooks/charmhelpers/contrib/charmsupport/volumes.py	1970-01-01 00:00:00 +0000
@@ -1,175 +0,0 @@
-# Copyright 2014-2015 Canonical Limited.
-#
-# This file is part of charm-helpers.
-#
-# charm-helpers is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# charm-helpers is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
-
-'''
-Functions for managing volumes in juju units. One volume is supported per unit.
-Subordinates may have their own storage, provided it is on its own partition.
-
-Configuration stanzas::
-
-  volume-ephemeral:
-    type: boolean
-    default: true
-    description: >
-      If false, a volume is mounted as sepecified in "volume-map"
-      If true, ephemeral storage will be used, meaning that log data
-         will only exist as long as the machine. YOU HAVE BEEN WARNED.
-  volume-map:
-    type: string
-    default: {}
-    description: >
-      YAML map of units to device names, e.g:
-        "{ rsyslog/0: /dev/vdb, rsyslog/1: /dev/vdb }"
-      Service units will raise a configure-error if volume-ephemeral
-      is 'true' and no volume-map value is set. Use 'juju set' to set a
-      value and 'juju resolved' to complete configuration.
-
-Usage::
-
-    from charmsupport.volumes import configure_volume, VolumeConfigurationError
-    from charmsupport.hookenv import log, ERROR
-    def post_mount_hook():
-        stop_service('myservice')
-    def post_mount_hook():
-        start_service('myservice')
-
-    if __name__ == '__main__':
-        try:
-            configure_volume(before_change=pre_mount_hook,
-                             after_change=post_mount_hook)
-        except VolumeConfigurationError:
-            log('Storage could not be configured', ERROR)
-
-'''
-
-# XXX: Known limitations
-# - fstab is neither consulted nor updated
-
-import os
-from charmhelpers.core import hookenv
-from charmhelpers.core import host
-import yaml
-
-
-MOUNT_BASE = '/srv/juju/volumes'
-
-
-class VolumeConfigurationError(Exception):
-    '''Volume configuration data is missing or invalid'''
-    pass
-
-
-def get_config():
-    '''Gather and sanity-check volume configuration data'''
-    volume_config = {}
-    config = hookenv.config()
-
-    errors = False
-
-    if config.get('volume-ephemeral') in (True, 'True', 'true', 'Yes', 'yes'):
-        volume_config['ephemeral'] = True
-    else:
-        volume_config['ephemeral'] = False
-
-    try:
-        volume_map = yaml.safe_load(config.get('volume-map', '{}'))
-    except yaml.YAMLError as e:
-        hookenv.log("Error parsing YAML volume-map: {}".format(e),
-                    hookenv.ERROR)
-        errors = True
-    if volume_map is None:
-        # probably an empty string
-        volume_map = {}
-    elif not isinstance(volume_map, dict):
-        hookenv.log("Volume-map should be a dictionary, not {}".format(
-            type(volume_map)))
-        errors = True
-
-    volume_config['device'] = volume_map.get(os.environ['JUJU_UNIT_NAME'])
-    if volume_config['device'] and volume_config['ephemeral']:
-        # asked for ephemeral storage but also defined a volume ID
-        hookenv.log('A volume is defined for this unit, but ephemeral '
-                    'storage was requested', hookenv.ERROR)
-        errors = True
-    elif not volume_config['device'] and not volume_config['ephemeral']:
-        # asked for permanent storage but did not define volume ID
-        hookenv.log('Ephemeral storage was requested, but there is no volume '
-                    'defined for this unit.', hookenv.ERROR)
-        errors = True
-
-    unit_mount_name = hookenv.local_unit().replace('/', '-')
-    volume_config['mountpoint'] = os.path.join(MOUNT_BASE, unit_mount_name)
-
-    if errors:
-        return None
-    return volume_config
-
-
-def mount_volume(config):
-    if os.path.exists(config['mountpoint']):
-        if not os.path.isdir(config['mountpoint']):
-            hookenv.log('Not a directory: {}'.format(config['mountpoint']))
-            raise VolumeConfigurationError()
-    else:
-        host.mkdir(config['mountpoint'])
-    if os.path.ismount(config['mountpoint']):
-        unmount_volume(config)
-    if not host.mount(config['device'], config['mountpoint'], persist=True):
-        raise VolumeConfigurationError()
-
-
-def unmount_volume(config):
-    if os.path.ismount(config['mountpoint']):
-        if not host.umount(config['mountpoint'], persist=True):
-            raise VolumeConfigurationError()
-
-
-def managed_mounts():
-    '''List of all mounted managed volumes'''
-    return filter(lambda mount: mount[0].startswith(MOUNT_BASE), host.mounts())
-
-
-def configure_volume(before_change=lambda: None, after_change=lambda: None):
-    '''Set up storage (or don't) according to the charm's volume configuration.
-       Returns the mount point or "ephemeral". before_change and after_change
-       are optional functions to be called if the volume configuration changes.
-    '''
-
-    config = get_config()
-    if not config:
-        hookenv.log('Failed to read volume configuration', hookenv.CRITICAL)
-        raise VolumeConfigurationError()
-
-    if config['ephemeral']:
-        if os.path.ismount(config['mountpoint']):
-            before_change()
-            unmount_volume(config)
-            after_change()
-        return 'ephemeral'
-    else:
-        # persistent storage
-        if os.path.ismount(config['mountpoint']):
-            mounts = dict(managed_mounts())
-            if mounts.get(config['mountpoint']) != config['device']:
-                before_change()
-                unmount_volume(config)
-                mount_volume(config)
-                after_change()
-        else:
-            before_change()
-            mount_volume(config)
-            after_change()
-        return config['mountpoint']
 
=== removed directory 'hooks/charmhelpers/contrib/database'
=== removed file 'hooks/charmhelpers/contrib/database/__init__.py'
=== removed file 'hooks/charmhelpers/contrib/database/mysql.py'
--- hooks/charmhelpers/contrib/database/mysql.py	2015-07-29 18:07:31 +0000
+++ hooks/charmhelpers/contrib/database/mysql.py	1970-01-01 00:00:00 +0000
@@ -1,412 +0,0 @@
-"""Helper for working with a MySQL database"""
-import json
-import re
-import sys
-import platform
-import os
-import glob
-
-# from string import upper
-
-from charmhelpers.core.host import (
-    mkdir,
-    pwgen,
-    write_file
-)
-from charmhelpers.core.hookenv import (
-    config as config_get,
-    relation_get,
-    related_units,
-    unit_get,
-    log,
-    DEBUG,
-    INFO,
-    WARNING,
-)
-from charmhelpers.fetch import (
-    apt_install,
-    apt_update,
-    filter_installed_packages,
-)
-from charmhelpers.contrib.peerstorage import (
-    peer_store,
-    peer_retrieve,
-)
-from charmhelpers.contrib.network.ip import get_host_ip
-
-try:
-    import MySQLdb
-except ImportError:
-    apt_update(fatal=True)
-    apt_install(filter_installed_packages(['python-mysqldb']), fatal=True)
-    import MySQLdb
-
-
-class MySQLHelper(object):
-
-    def __init__(self, rpasswdf_template, upasswdf_template, host='localhost',
-                 migrate_passwd_to_peer_relation=True,
-                 delete_ondisk_passwd_file=True):
-        self.host = host
-        # Password file path templates
-        self.root_passwd_file_template = rpasswdf_template
-        self.user_passwd_file_template = upasswdf_template
-
-        self.migrate_passwd_to_peer_relation = migrate_passwd_to_peer_relation
-        # If we migrate we have the option to delete local copy of root passwd
-        self.delete_ondisk_passwd_file = delete_ondisk_passwd_file
-
-    def connect(self, user='root', password=None):
-        log("Opening db connection for %s@%s" % (user, self.host), level=DEBUG)
-        self.connection = MySQLdb.connect(user=user, host=self.host,
-                                          passwd=password)
-
-    def database_exists(self, db_name):
-        cursor = self.connection.cursor()
-        try:
-            cursor.execute("SHOW DATABASES")
-            databases = [i[0] for i in cursor.fetchall()]
-        finally:
-            cursor.close()
-
-        return db_name in databases
-
-    def create_database(self, db_name):
-        cursor = self.connection.cursor()
-        try:
-            cursor.execute("CREATE DATABASE {} CHARACTER SET UTF8"
-                           .format(db_name))
-        finally:
-            cursor.close()
-
-    def grant_exists(self, db_name, db_user, remote_ip):
-        cursor = self.connection.cursor()
-        priv_string = "GRANT ALL PRIVILEGES ON `{}`.* " \
-                      "TO '{}'@'{}'".format(db_name, db_user, remote_ip)
-        try:
-            cursor.execute("SHOW GRANTS for '{}'@'{}'".format(db_user,
-                                                              remote_ip))
-            grants = [i[0] for i in cursor.fetchall()]
-        except MySQLdb.OperationalError:
-            return False
-        finally:
-            cursor.close()
-
-        # TODO: review for different grants
-        return priv_string in grants
-
-    def create_grant(self, db_name, db_user, remote_ip, password):
-        cursor = self.connection.cursor()
-        try:
-            # TODO: review for different grants
-            cursor.execute("GRANT ALL PRIVILEGES ON {}.* TO '{}'@'{}' "
-                           "IDENTIFIED BY '{}'".format(db_name,
-                                                       db_user,
-                                                       remote_ip,
-                                                       password))
-        finally:
-            cursor.close()
-
-    def create_admin_grant(self, db_user, remote_ip, password):
-        cursor = self.connection.cursor()
-        try:
-            cursor.execute("GRANT ALL PRIVILEGES ON *.* TO '{}'@'{}' "
-                           "IDENTIFIED BY '{}'".format(db_user,
-                                                       remote_ip,
-                                                       password))
-        finally:
-            cursor.close()
-
-    def cleanup_grant(self, db_user, remote_ip):
-        cursor = self.connection.cursor()
-        try:
-            cursor.execute("DROP FROM mysql.user WHERE user='{}' "
-                           "AND HOST='{}'".format(db_user,
-                                                  remote_ip))
-        finally:
-            cursor.close()
-
-    def execute(self, sql):
-        """Execute arbitary SQL against the database."""
-        cursor = self.connection.cursor()
-        try:
-            cursor.execute(sql)
-        finally:
-            cursor.close()
-
-    def migrate_passwords_to_peer_relation(self, excludes=None):
-        """Migrate any passwords storage on disk to cluster peer relation."""
-        dirname = os.path.dirname(self.root_passwd_file_template)
-        path = os.path.join(dirname, '*.passwd')
-        for f in glob.glob(path):
-            if excludes and f in excludes:
-                log("Excluding %s from peer migration" % (f), level=DEBUG)
-                continue
-
-            key = os.path.basename(f)
-            with open(f, 'r') as passwd:
-                _value = passwd.read().strip()
-
-            try:
-                peer_store(key, _value)
-
-                if self.delete_ondisk_passwd_file:
-                    os.unlink(f)
-            except ValueError:
-                # NOTE cluster relation not yet ready - skip for now
-                pass
-
-    def get_mysql_password_on_disk(self, username=None, password=None):
-        """Retrieve, generate or store a mysql password for the provided
-        username on disk."""
-        if username:
-            template = self.user_passwd_file_template
-            passwd_file = template.format(username)
-        else:
-            passwd_file = self.root_passwd_file_template
-
-        _password = None
-        if os.path.exists(passwd_file):
-            log("Using existing password file '%s'" % passwd_file, level=DEBUG)
-            with open(passwd_file, 'r') as passwd:
-                _password = passwd.read().strip()
-        else:
-            log("Generating new password file '%s'" % passwd_file, level=DEBUG)
-            if not os.path.isdir(os.path.dirname(passwd_file)):
-                # NOTE: need to ensure this is not mysql root dir (which needs
-                # to be mysql readable)
-                mkdir(os.path.dirname(passwd_file), owner='root', group='root',
-                      perms=0o770)
-                # Force permissions - for some reason the chmod in makedirs
-                # fails
-                os.chmod(os.path.dirname(passwd_file), 0o770)
-
-            _password = password or pwgen(length=32)
-            write_file(passwd_file, _password, owner='root', group='root',
-                       perms=0o660)
-
-        return _password
-
-    def passwd_keys(self, username):
-        """Generator to return keys used to store passwords in peer store.
-
-        NOTE: we support both legacy and new format to support mysql
-        charm prior to refactor. This is necessary to avoid LP 1451890.
-        """
-        keys = []
-        if username == 'mysql':
-            log("Bad username '%s'" % (username), level=WARNING)
-
-        if username:
-            # IMPORTANT: *newer* format must be returned first
-            keys.append('mysql-%s.passwd' % (username))
-            keys.append('%s.passwd' % (username))
-        else:
-            keys.append('mysql.passwd')
-
-        for key in keys:
-            yield key
-
-    def get_mysql_password(self, username=None, password=None):
-        """Retrieve, generate or store a mysql password for the provided
-        username using peer relation cluster."""
-        excludes = []
-
-        # First check peer relation.
-        try:
-            for key in self.passwd_keys(username):
-                _password = peer_retrieve(key)
-                if _password:
-                    break
-
-            # If root password available don't update peer relation from local
-            if _password and not username:
-                excludes.append(self.root_passwd_file_template)
-
-        except ValueError:
-            # cluster relation is not yet started; use on-disk
-            _password = None
-
-        # If none available, generate new one
-        if not _password:
-            _password = self.get_mysql_password_on_disk(username, password)
-
-        # Put on wire if required
-        if self.migrate_passwd_to_peer_relation:
-            self.migrate_passwords_to_peer_relation(excludes=excludes)
-
-        return _password
-
-    def get_mysql_root_password(self, password=None):
-        """Retrieve or generate mysql root password for service units."""
-        return self.get_mysql_password(username=None, password=password)
-
-    def normalize_address(self, hostname):
-        """Ensure that address returned is an IP address (i.e. not fqdn)"""
-        if config_get('prefer-ipv6'):
-            # TODO: add support for ipv6 dns
-            return hostname
-
-        if hostname != unit_get('private-address'):
-            return get_host_ip(hostname, fallback=hostname)
-
-        # Otherwise assume localhost
-        return '127.0.0.1'
-
-    def get_allowed_units(self, database, username, relation_id=None):
-        """Get list of units with access grants for database with username.
-
-        This is typically used to provide shared-db relations with a list of
-        which units have been granted access to the given database.
-        """
-        self.connect(password=self.get_mysql_root_password())
-        allowed_units = set()
-        for unit in related_units(relation_id):
-            settings = relation_get(rid=relation_id, unit=unit)
-            # First check for setting with prefix, then without
-            for attr in ["%s_hostname" % (database), 'hostname']:
-                hosts = settings.get(attr, None)
-                if hosts:
-                    break
-
-            if hosts:
-                # hostname can be json-encoded list of hostnames
-                try:
-                    hosts = json.loads(hosts)
-                except ValueError:
-                    hosts = [hosts]
-            else:
-                hosts = [settings['private-address']]
-
-            if hosts:
-                for host in hosts:
-                    host = self.normalize_address(host)
-                    if self.grant_exists(database, username, host):
-                        log("Grant exists for host '%s' on db '%s'" %
-                            (host, database), level=DEBUG)
-                        if unit not in allowed_units:
-                            allowed_units.add(unit)
-                    else:
-                        log("Grant does NOT exist for host '%s' on db '%s'" %
-                            (host, database), level=DEBUG)
-            else:
-                log("No hosts found for grant check", level=INFO)
-
-        return allowed_units
-
-    def configure_db(self, hostname, database, username, admin=False):
-        """Configure access to database for username from hostname."""
-        self.connect(password=self.get_mysql_root_password())
-        if not self.database_exists(database):
-            self.create_database(database)
-
-        remote_ip = self.normalize_address(hostname)
-        password = self.get_mysql_password(username)
-        if not self.grant_exists(database, username, remote_ip):
-            if not admin:
-                self.create_grant(database, username, remote_ip, password)
-            else:
-                self.create_admin_grant(username, remote_ip, password)
-
-        return password
-
-
-class PerconaClusterHelper(object):
-
-    # Going for the biggest page size to avoid wasted bytes.
-    # InnoDB page size is 16MB
-
-    DEFAULT_PAGE_SIZE = 16 * 1024 * 1024
-    DEFAULT_INNODB_BUFFER_FACTOR = 0.50
-
-    def human_to_bytes(self, human):
-        """Convert human readable configuration options to bytes."""
-        num_re = re.compile('^[0-9]+$')
-        if num_re.match(human):
-            return human
-
-        factors = {
-            'K': 1024,
-            'M': 1048576,
-            'G': 1073741824,
-            'T': 1099511627776
-        }
-        modifier = human[-1]
-        if modifier in factors:
-            return int(human[:-1]) * factors[modifier]
-
-        if modifier == '%':
-            total_ram = self.human_to_bytes(self.get_mem_total())
-            if self.is_32bit_system() and total_ram > self.sys_mem_limit():
-                total_ram = self.sys_mem_limit()
-            factor = int(human[:-1]) * 0.01
-            pctram = total_ram * factor
-            return int(pctram - (pctram % self.DEFAULT_PAGE_SIZE))
-
-        raise ValueError("Can only convert K,M,G, or T")
-
-    def is_32bit_system(self):
-        """Determine whether system is 32 or 64 bit."""
-        try:
-            return sys.maxsize < 2 ** 32
-        except OverflowError:
-            return False
-
-    def sys_mem_limit(self):
-        """Determine the default memory limit for the current service unit."""
-        if platform.machine() in ['armv7l']:
-            _mem_limit = self.human_to_bytes('2700M')  # experimentally determined
-        else:
-            # Limit for x86 based 32bit systems
-            _mem_limit = self.human_to_bytes('4G')
-
-        return _mem_limit
-
-    def get_mem_total(self):
-        """Calculate the total memory in the current service unit."""
-        with open('/proc/meminfo') as meminfo_file:
-            for line in meminfo_file:
-                key, mem = line.split(':', 2)
-                if key == 'MemTotal':
-                    mtot, modifier = mem.strip().split(' ')
-                    return '%s%s' % (mtot, modifier[0].upper())
-
-    def parse_config(self):
-        """Parse charm configuration and calculate values for config files."""
-        config = config_get()
-        mysql_config = {}
-        if 'max-connections' in config:
-            mysql_config['max_connections'] = config['max-connections']
-
-        if 'wait-timeout' in config:
-            mysql_config['wait_timeout'] = config['wait-timeout']
-
-        if 'innodb-flush-log-at-trx-commit' in config:
-            mysql_config['innodb_flush_log_at_trx_commit'] = config['innodb-flush-log-at-trx-commit']
-
-        # Set a sane default key_buffer size
-        mysql_config['key_buffer'] = self.human_to_bytes('32M')
-        total_memory = self.human_to_bytes(self.get_mem_total())
-
-        dataset_bytes = config.get('dataset-size', None)
-        innodb_buffer_pool_size = config.get('innodb-buffer-pool-size', None)
-
-        if innodb_buffer_pool_size:
-            innodb_buffer_pool_size = self.human_to_bytes(
-                innodb_buffer_pool_size)
-        elif dataset_bytes:
-            log("Option 'dataset-size' has been deprecated, please use"
-                "innodb_buffer_pool_size option instead", level="WARN")
-            innodb_buffer_pool_size = self.human_to_bytes(
-                dataset_bytes)
-        else:
-            innodb_buffer_pool_size = int(
-                total_memory * self.DEFAULT_INNODB_BUFFER_FACTOR)
-
-        if innodb_buffer_pool_size > total_memory:
-            log("innodb_buffer_pool_size; {} is greater than system available memory:{}".format(
-                innodb_buffer_pool_size,
-                total_memory), level='WARN')
-
-        mysql_config['innodb_buffer_pool_size'] = innodb_buffer_pool_size
-        return mysql_config
 
=== modified file 'hooks/charmhelpers/contrib/network/ip.py'
--- hooks/charmhelpers/contrib/network/ip.py	2015-05-19 21:31:00 +0000
+++ hooks/charmhelpers/contrib/network/ip.py	2016-05-18 10:01:02 +0000
@@ -23,7 +23,7 @@
 from functools import partial
 
 from charmhelpers.core.hookenv import unit_get
-from charmhelpers.fetch import apt_install
+from charmhelpers.fetch import apt_install, apt_update
 from charmhelpers.core.hookenv import (
     log,
     WARNING,
@@ -32,13 +32,15 @@
 try:
     import netifaces
 except ImportError:
-    apt_install('python-netifaces')
+    apt_update(fatal=True)
+    apt_install('python-netifaces', fatal=True)
     import netifaces
 
 try:
     import netaddr
 except ImportError:
-    apt_install('python-netaddr')
+    apt_update(fatal=True)
+    apt_install('python-netaddr', fatal=True)
     import netaddr
 
 
@@ -51,7 +53,7 @@
 
 
 def no_ip_found_error_out(network):
-    errmsg = ("No IP address found in network: %s" % network)
+    errmsg = ("No IP address found in network(s): %s" % network)
     raise ValueError(errmsg)
 
 
@@ -59,7 +61,7 @@
     """Get an IPv4 or IPv6 address within the network from the host.
 
     :param network (str): CIDR presentation format. For example,
-        '192.168.1.0/24'.
+        '192.168.1.0/24'. Supports multiple networks as a space-delimited list.
     :param fallback (str): If no address is found, return fallback.
     :param fatal (boolean): If no address is found, fallback is not
         set and fatal is True then exit(1).
@@ -73,24 +75,26 @@
         else:
             return None
 
-    _validate_cidr(network)
-    network = netaddr.IPNetwork(network)
-    for iface in netifaces.interfaces():
-        addresses = netifaces.ifaddresses(iface)
-        if network.version == 4 and netifaces.AF_INET in addresses:
-            addr = addresses[netifaces.AF_INET][0]['addr']
-            netmask = addresses[netifaces.AF_INET][0]['netmask']
-            cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask))
-            if cidr in network:
-                return str(cidr.ip)
+    networks = network.split() or [network]
+    for network in networks:
+        _validate_cidr(network)
+        network = netaddr.IPNetwork(network)
+        for iface in netifaces.interfaces():
+            addresses = netifaces.ifaddresses(iface)
+            if network.version == 4 and netifaces.AF_INET in addresses:
+                addr = addresses[netifaces.AF_INET][0]['addr']
+                netmask = addresses[netifaces.AF_INET][0]['netmask']
+                cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask))
+                if cidr in network:
+                    return str(cidr.ip)
 
-        if network.version == 6 and netifaces.AF_INET6 in addresses:
-            for addr in addresses[netifaces.AF_INET6]:
-                if not addr['addr'].startswith('fe80'):
-                    cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'],
-                                                        addr['netmask']))
-                    if cidr in network:
-                        return str(cidr.ip)
+            if network.version == 6 and netifaces.AF_INET6 in addresses:
+                for addr in addresses[netifaces.AF_INET6]:
+                    if not addr['addr'].startswith('fe80'):
+                        cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'],
+                                                            addr['netmask']))
+                        if cidr in network:
+                            return str(cidr.ip)
 
     if fallback is not None:
         return fallback
@@ -187,6 +191,15 @@
 get_netmask_for_address = partial(_get_for_address, key='netmask')
 
 
+def resolve_network_cidr(ip_address):
+    '''
+    Resolves the full address cidr of an ip_address based on
+    configured network interfaces
+    '''
+    netmask = get_netmask_for_address(ip_address)
+    return str(netaddr.IPNetwork("%s/%s" % (ip_address, netmask)).cidr)
+
+
 def format_ipv6_addr(address):
     """If address is IPv6, wrap it in '[]' otherwise return None.
 
@@ -435,8 +448,12 @@
 
         rev = dns.reversename.from_address(address)
         result = ns_query(rev)
+
         if not result:
-            return None
+            try:
+                result = socket.gethostbyaddr(address)[0]
+            except:
+                return None
     else:
         result = address
 
@@ -448,3 +465,18 @@
             return result
     else:
         return result.split('.')[0]
+
+
+def port_has_listener(address, port):
+    """
+    Returns True if the address:port is open and being listened to,
+    else False.
+
+    @param address: an IP address or hostname
+    @param port: integer port
+
+    Note calls 'zc' via a subprocess shell
+    """
+    cmd = ['nc', '-z', address, str(port)]
+    result = subprocess.call(cmd)
+    return not(bool(result))
 
=== modified file 'hooks/charmhelpers/contrib/network/ovs/__init__.py'
--- hooks/charmhelpers/contrib/network/ovs/__init__.py	2015-05-19 21:31:00 +0000
+++ hooks/charmhelpers/contrib/network/ovs/__init__.py	2016-05-18 10:01:02 +0000
@@ -25,10 +25,14 @@
 )
 
 
-def add_bridge(name):
+def add_bridge(name, datapath_type=None):
     ''' Add the named bridge to openvswitch '''
     log('Creating bridge {}'.format(name))
-    subprocess.check_call(["ovs-vsctl", "--", "--may-exist", "add-br", name])
+    cmd = ["ovs-vsctl", "--", "--may-exist", "add-br", name]
+    if datapath_type is not None:
+        cmd += ['--', 'set', 'bridge', name,
+                'datapath_type={}'.format(datapath_type)]
+    subprocess.check_call(cmd)
 
 
 def del_bridge(name):
 
=== modified file 'hooks/charmhelpers/contrib/network/ufw.py'
--- hooks/charmhelpers/contrib/network/ufw.py	2015-07-29 18:07:31 +0000
+++ hooks/charmhelpers/contrib/network/ufw.py	2016-05-18 10:01:02 +0000
@@ -40,7 +40,9 @@
 import re
 import os
 import subprocess
+
 from charmhelpers.core import hookenv
+from charmhelpers.core.kernel import modprobe, is_module_loaded
 
 __author__ = "Felipe Reyes <[e-mail]>"
 
@@ -82,14 +84,11 @@
     # do we have IPv6 in the machine?
     if os.path.isdir('/proc/sys/net/ipv6'):
         # is ip6tables kernel module loaded?
-        lsmod = subprocess.check_output(['lsmod'], universal_newlines=True)
-        matches = re.findall('^ip6_tables[ ]+', lsmod, re.M)
-        if len(matches) == 0:
+        if not is_module_loaded('ip6_tables'):
             # ip6tables support isn't complete, let's try to load it
             try:
-                subprocess.check_output(['modprobe', 'ip6_tables'],
-                                        universal_newlines=True)
-                # great, we could load the module
+                modprobe('ip6_tables')
+                # great, we can load the module
                 return True
             except subprocess.CalledProcessError as ex:
                 hookenv.log("Couldn't load ip6_tables module: %s" % ex.output,
 
=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/deployment.py'
--- hooks/charmhelpers/contrib/openstack/amulet/deployment.py	2015-07-29 18:07:31 +0000
+++ hooks/charmhelpers/contrib/openstack/amulet/deployment.py	2016-05-18 10:01:02 +0000
@@ -14,12 +14,18 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
+import re
+import sys
 import six
 from collections import OrderedDict
 from charmhelpers.contrib.amulet.deployment import (
     AmuletDeployment
 )
 
+DEBUG = logging.DEBUG
+ERROR = logging.ERROR
+
 
 class OpenStackAmuletDeployment(AmuletDeployment):
     """OpenStack amulet deployment.
@@ -28,9 +34,12 @@
        that is specifically for use by OpenStack charms.
        """
 
-    def __init__(self, series=None, openstack=None, source=None, stable=True):
+    def __init__(self, series=None, openstack=None, source=None,
+                 stable=True, log_level=DEBUG):
         """Initialize the deployment environment."""
         super(OpenStackAmuletDeployment, self).__init__(series)
+        self.log = self.get_logger(level=log_level)
+        self.log.info('OpenStackAmuletDeployment:  init')
         self.openstack = openstack
         self.source = source
         self.stable = stable
@@ -38,26 +47,55 @@
         # out.
         self.current_next = "trusty"
 
+    def get_logger(self, name="deployment-logger", level=logging.DEBUG):
+        """Get a logger object that will log to stdout."""
+        log = logging
+        logger = log.getLogger(name)
+        fmt = log.Formatter("%(asctime)s %(funcName)s "
+                            "%(levelname)s: %(message)s")
+
+        handler = log.StreamHandler(stream=sys.stdout)
+        handler.setLevel(level)
+        handler.setFormatter(fmt)
+
+        logger.addHandler(handler)
+        logger.setLevel(level)
+
+        return logger
+
     def _determine_branch_locations(self, other_services):
         """Determine the branch locations for the other services.
 
            Determine if the local branch being tested is derived from its
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
-        base_charms = ['mysql', 'mongodb']
+
+        self.log.info('OpenStackAmuletDeployment:  determine branch locations')
+
+        # Charms outside the lp:~openstack-charmers namespace
+        base_charms = ['mysql', 'mongodb', 'nrpe']
+
+        # Force these charms to current series even when using an older series.
+        # ie. Use trusty/nrpe even when series is precise, as the P charm
+        # does not possess the necessary external master config and hooks.
+        force_series_current = ['nrpe']
 
         if self.series in ['precise', 'trusty']:
             base_series = self.series
         else:
             base_series = self.current_next
 
-        if self.stable:
-            for svc in other_services:
+        for svc in other_services:
+            if svc['name'] in force_series_current:
+                base_series = self.current_next
+            # If a location has been explicitly set, use it
+            if svc.get('location'):
+                continue
+            if self.stable:
                 temp = 'lp:charms/{}/{}'
                 svc['location'] = temp.format(base_series,
                                               svc['name'])
-        else:
-            for svc in other_services:
+            else:
                 if svc['name'] in base_charms:
                     temp = 'lp:charms/{}/{}'
                     svc['location'] = temp.format(base_series,
@@ -66,10 +104,13 @@
                     temp = 'lp:~openstack-charmers/charms/{}/{}/next'
                     svc['location'] = temp.format(self.current_next,
                                                   svc['name'])
+
         return other_services
 
     def _add_services(self, this_service, other_services):
         """Add services to the deployment and set openstack-origin/source."""
+        self.log.info('OpenStackAmuletDeployment:  adding services')
+
         other_services = self._determine_branch_locations(other_services)
 
         super(OpenStackAmuletDeployment, self)._add_services(this_service,
@@ -77,29 +118,105 @@
 
         services = other_services
         services.append(this_service)
+
+        # Charms which should use the source config option
         use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
-                      'ceph-osd', 'ceph-radosgw']
-        # Most OpenStack subordinate charms do not expose an origin option
-        # as that is controlled by the principle.
-        ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch']
+                      'ceph-osd', 'ceph-radosgw', 'ceph-mon']
+
+        # Charms which can not use openstack-origin, ie. many subordinates
+        no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe',
+                     'openvswitch-odl', 'neutron-api-odl', 'odl-controller',
+                     'cinder-backup', 'nexentaedge-data',
+                     'nexentaedge-iscsi-gw', 'nexentaedge-swift-gw',
+                     'cinder-nexentaedge', 'nexentaedge-mgmt']
 
         if self.openstack:
             for svc in services:
-                if svc['name'] not in use_source + ignore:
+                if svc['name'] not in use_source + no_origin:
                     config = {'openstack-origin': self.openstack}
                     self.d.configure(svc['name'], config)
 
         if self.source:
             for svc in services:
-                if svc['name'] in use_source and svc['name'] not in ignore:
+                if svc['name'] in use_source and svc['name'] not in no_origin:
                     config = {'source': self.source}
                     self.d.configure(svc['name'], config)
 
     def _configure_services(self, configs):
         """Configure all of the services."""
+        self.log.info('OpenStackAmuletDeployment:  configure services')
         for service, config in six.iteritems(configs):
             self.d.configure(service, config)
 
+    def _auto_wait_for_status(self, message=None, exclude_services=None,
+                              include_only=None, timeout=1800):
+        """Wait for all units to have a specific extended status, except
+        for any defined as excluded.  Unless specified via message, any
+        status containing any case of 'ready' will be considered a match.
+
+        Examples of message usage:
+
+          Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
+              message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
+
+          Wait for all units to reach this status (exact match):
+              message = re.compile('^Unit is ready and clustered$')
+
+          Wait for all units to reach any one of these (exact match):
+              message = re.compile('Unit is ready|OK|Ready')
+
+          Wait for at least one unit to reach this status (exact match):
+              message = {'ready'}
+
+        See Amulet's sentry.wait_for_messages() for message usage detail.
+        https://github.com/juju/amulet/blob/master/amulet/sentry.py
+
+        :param message: Expected status match
+        :param exclude_services: List of juju service names to ignore,
+            not to be used in conjuction with include_only.
+        :param include_only: List of juju service names to exclusively check,
+            not to be used in conjuction with exclude_services.
+        :param timeout: Maximum time in seconds to wait for status match
+        :returns: None.  Raises if timeout is hit.
+        """
+        self.log.info('Waiting for extended status on units...')
+
+        all_services = self.d.services.keys()
+
+        if exclude_services and include_only:
+            raise ValueError('exclude_services can not be used '
+                             'with include_only')
+
+        if message:
+            if isinstance(message, re._pattern_type):
+                match = message.pattern
+            else:
+                match = message
+
+            self.log.debug('Custom extended status wait match: '
+                           '{}'.format(match))
+        else:
+            self.log.debug('Default extended status wait match:  contains '
+                           'READY (case-insensitive)')
+            message = re.compile('.*ready.*', re.IGNORECASE)
+
+        if exclude_services:
+            self.log.debug('Excluding services from extended status match: '
+                           '{}'.format(exclude_services))
+        else:
+            exclude_services = []
+
+        if include_only:
+            services = include_only
+        else:
+            services = list(set(all_services) - set(exclude_services))
+
+        self.log.debug('Waiting up to {}s for extended status on services: '
+                       '{}'.format(timeout, services))
+        service_messages = {service: message for service in services}
+        self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
+        self.log.info('OK')
+
     def _get_openstack_release(self):
         """Get openstack release.
 
@@ -111,7 +228,8 @@
          self.precise_havana, self.precise_icehouse,
          self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
          self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
-         self.wily_liberty) = range(12)
+         self.wily_liberty, self.trusty_mitaka,
+         self.xenial_mitaka) = range(14)
 
         releases = {
             ('precise', None): self.precise_essex,
@@ -123,9 +241,11 @@
             ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
             ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
             ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
+            ('trusty', 'cloud:trusty-mitaka'): self.trusty_mitaka,
             ('utopic', None): self.utopic_juno,
             ('vivid', None): self.vivid_kilo,
-            ('wily', None): self.wily_liberty}
+            ('wily', None): self.wily_liberty,
+            ('xenial', None): self.xenial_mitaka}
         return releases[(self.series, self.openstack)]
 
     def _get_openstack_release_string(self):
@@ -142,6 +262,7 @@
             ('utopic', 'juno'),
             ('vivid', 'kilo'),
             ('wily', 'liberty'),
+            ('xenial', 'mitaka'),
         ])
         if self.openstack:
             os_origin = self.openstack.split(':')[1]
 
=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/utils.py'
--- hooks/charmhelpers/contrib/openstack/amulet/utils.py	2015-07-29 18:07:31 +0000
+++ hooks/charmhelpers/contrib/openstack/amulet/utils.py	2016-05-18 10:01:02 +0000
@@ -18,6 +18,7 @@
 import json
 import logging
 import os
+import re
 import six
 import time
 import urllib
@@ -26,7 +27,12 @@
 import glanceclient.v1.client as glance_client
 import heatclient.v1.client as heat_client
 import keystoneclient.v2_0 as keystone_client
-import novaclient.v1_1.client as nova_client
+from keystoneclient.auth.identity import v3 as keystone_id_v3
+from keystoneclient import session as keystone_session
+from keystoneclient.v3 import client as keystone_client_v3
+
+import novaclient.client as nova_client
+import pika
 import swiftclient
 
 from charmhelpers.contrib.amulet.utils import (
@@ -36,6 +42,8 @@
 DEBUG = logging.DEBUG
 ERROR = logging.ERROR
 
+NOVA_CLIENT_VERSION = "2"
+
 
 class OpenStackAmuletUtils(AmuletUtils):
     """OpenStack amulet utilities.
@@ -137,7 +145,7 @@
                 return "role {} does not exist".format(e['name'])
         return ret
 
-    def validate_user_data(self, expected, actual):
+    def validate_user_data(self, expected, actual, api_version=None):
         """Validate user data.
 
            Validate a list of actual user data vs a list of expected user
@@ -148,10 +156,15 @@
         for e in expected:
             found = False
             for act in actual:
-                a = {'enabled': act.enabled, 'name': act.name,
-                     'email': act.email, 'tenantId': act.tenantId,
-                     'id': act.id}
-                if e['name'] == a['name']:
+                if e['name'] == act.name:
+                    a = {'enabled': act.enabled, 'name': act.name,
+                         'email': act.email, 'id': act.id}
+                    if api_version == 3:
+                        a['default_project_id'] = getattr(act,
+                                                          'default_project_id',
+                                                          'none')
+                    else:
+                        a['tenantId'] = act.tenantId
                     found = True
                     ret = self._validate_dict_data(e, a)
                     if ret:
@@ -186,15 +199,30 @@
         return cinder_client.Client(username, password, tenant, ept)
 
     def authenticate_keystone_admin(self, keystone_sentry, user, password,
-                                    tenant):
+                                    tenant=None, api_version=None,
+                                    keystone_ip=None):
         """Authenticates admin user with the keystone admin endpoint."""
         self.log.debug('Authenticating keystone admin...')
         unit = keystone_sentry
-        service_ip = unit.relation('shared-db',
-                                   'mysql:shared-db')['private-address']
-        ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8'))
-        return keystone_client.Client(username=user, password=password,
-                                      tenant_name=tenant, auth_url=ep)
+        if not keystone_ip:
+            keystone_ip = unit.relation('shared-db',
+                                        'mysql:shared-db')['private-address']
+        base_ep = "http://{}:35357".format(keystone_ip.strip().decode('utf-8'))
+        if not api_version or api_version == 2:
+            ep = base_ep + "/v2.0"
+            return keystone_client.Client(username=user, password=password,
+                                          tenant_name=tenant, auth_url=ep)
+        else:
+            ep = base_ep + "/v3"
+            auth = keystone_id_v3.Password(
+                user_domain_name='admin_domain',
+                username=user,
+                password=password,
+                domain_name='admin_domain',
+                auth_url=ep,
+            )
+            sess = keystone_session.Session(auth=auth)
+            return keystone_client_v3.Client(session=sess)
 
     def authenticate_keystone_user(self, keystone, user, password, tenant):
         """Authenticates a regular user with the keystone public endpoint."""
@@ -223,7 +251,8 @@
         self.log.debug('Authenticating nova user ({})...'.format(user))
         ep = keystone.service_catalog.url_for(service_type='identity',
                                               endpoint_type='publicURL')
-        return nova_client.Client(username=user, api_key=password,
+        return nova_client.Client(NOVA_CLIENT_VERSION,
+                                  username=user, api_key=password,
                                   project_id=tenant, auth_url=ep)
 
     def authenticate_swift_user(self, keystone, user, password, tenant):
@@ -602,3 +631,382 @@
             self.log.debug('Ceph {} samples (OK): '
                            '{}'.format(sample_type, samples))
             return None
+
+    # rabbitmq/amqp specific helpers:
+
+    def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
+        """Wait for rmq units extended status to show cluster readiness,
+        after an optional initial sleep period.  Initial sleep is likely
+        necessary to be effective following a config change, as status
+        message may not instantly update to non-ready."""
+
+        if init_sleep:
+            time.sleep(init_sleep)
+
+        message = re.compile('^Unit is ready and clustered$')
+        deployment._auto_wait_for_status(message=message,
+                                         timeout=timeout,
+                                         include_only=['rabbitmq-server'])
+
+    def add_rmq_test_user(self, sentry_units,
+                          username="testuser1", password="changeme"):
+        """Add a test user via the first rmq juju unit, check connection as
+        the new user against all sentry units.
+
+        :param sentry_units: list of sentry unit pointers
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :returns: None if successful.  Raise on error.
+        """
+        self.log.debug('Adding rmq user ({})...'.format(username))
+
+        # Check that user does not already exist
+        cmd_user_list = 'rabbitmqctl list_users'
+        output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list)
+        if username in output:
+            self.log.warning('User ({}) already exists, returning '
+                             'gracefully.'.format(username))
+            return
+
+        perms = '".*" ".*" ".*"'
+        cmds = ['rabbitmqctl add_user {} {}'.format(username, password),
+                'rabbitmqctl set_permissions {} {}'.format(username, perms)]
+
+        # Add user via first unit
+        for cmd in cmds:
+            output, _ = self.run_cmd_unit(sentry_units[0], cmd)
+
+        # Check connection against the other sentry_units
+        self.log.debug('Checking user connect against units...')
+        for sentry_unit in sentry_units:
+            connection = self.connect_amqp_by_unit(sentry_unit, ssl=False,
+                                                   username=username,
+                                                   password=password)
+            connection.close()
+
+    def delete_rmq_test_user(self, sentry_units, username="testuser1"):
+        """Delete a rabbitmq user via the first rmq juju unit.
+
+        :param sentry_units: list of sentry unit pointers
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :returns: None if successful or no such user.
+        """
+        self.log.debug('Deleting rmq user ({})...'.format(username))
+
+        # Check that the user exists
+        cmd_user_list = 'rabbitmqctl list_users'
+        output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list)
+
+        if username not in output:
+            self.log.warning('User ({}) does not exist, returning '
+                             'gracefully.'.format(username))
+            return
+
+        # Delete the user
+        cmd_user_del = 'rabbitmqctl delete_user {}'.format(username)
+        output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_del)
+
+    def get_rmq_cluster_status(self, sentry_unit):
+        """Execute rabbitmq cluster status command on a unit and return
+        the full output.
+
+        :param unit: sentry unit
+        :returns: String containing console output of cluster status command
+        """
+        cmd = 'rabbitmqctl cluster_status'
+        output, _ = self.run_cmd_unit(sentry_unit, cmd)
+        self.log.debug('{} cluster_status:\n{}'.format(
+            sentry_unit.info['unit_name'], output))
+        return str(output)
+
+    def get_rmq_cluster_running_nodes(self, sentry_unit):
+        """Parse rabbitmqctl cluster_status output string, return list of
+        running rabbitmq cluster nodes.
+
+        :param unit: sentry unit
+        :returns: List containing node names of running nodes
+        """
+        # NOTE(beisner): rabbitmqctl cluster_status output is not
+        # json-parsable, do string chop foo, then json.loads that.
+        str_stat = self.get_rmq_cluster_status(sentry_unit)
+        if 'running_nodes' in str_stat:
+            pos_start = str_stat.find("{running_nodes,") + 15
+            pos_end = str_stat.find("]},", pos_start) + 1
+            str_run_nodes = str_stat[pos_start:pos_end].replace("'", '"')
+            run_nodes = json.loads(str_run_nodes)
+            return run_nodes
+        else:
+            return []
+
+    def validate_rmq_cluster_running_nodes(self, sentry_units):
+        """Check that all rmq unit hostnames are represented in the
+        cluster_status output of all units.
+
+        :param host_names: dict of juju unit names to host names
+        :param units: list of sentry unit pointers (all rmq units)
+        :returns: None if successful, otherwise return error message
+        """
+        host_names = self.get_unit_hostnames(sentry_units)
+        errors = []
+
+        # Query every unit for cluster_status running nodes
+        for query_unit in sentry_units:
+            query_unit_name = query_unit.info['unit_name']
+            running_nodes = self.get_rmq_cluster_running_nodes(query_unit)
+
+            # Confirm that every unit is represented in the queried unit's
+            # cluster_status running nodes output.
+            for validate_unit in sentry_units:
+                val_host_name = host_names[validate_unit.info['unit_name']]
+                val_node_name = 'rabbit@{}'.format(val_host_name)
+
+                if val_node_name not in running_nodes:
+                    errors.append('Cluster member check failed on {}: {} not '
+                                  'in {}\n'.format(query_unit_name,
+                                                   val_node_name,
+                                                   running_nodes))
+        if errors:
+            return ''.join(errors)
+
+    def rmq_ssl_is_enabled_on_unit(self, sentry_unit, port=None):
+        """Check a single juju rmq unit for ssl and port in the config file."""
+        host = sentry_unit.info['public-address']
+        unit_name = sentry_unit.info['unit_name']
+
+        conf_file = '/etc/rabbitmq/rabbitmq.config'
+        conf_contents = str(self.file_contents_safe(sentry_unit,
+                                                    conf_file, max_wait=16))
+        # Checks
+        conf_ssl = 'ssl' in conf_contents
+        conf_port = str(port) in conf_contents
+
+        # Port explicitly checked in config
+        if port and conf_port and conf_ssl:
+            self.log.debug('SSL is enabled  @{}:{} '
+                           '({})'.format(host, port, unit_name))
+            return True
+        elif port and not conf_port and conf_ssl:
+            self.log.debug('SSL is enabled @{} but not on port {} '
+                           '({})'.format(host, port, unit_name))
+            return False
+        # Port not checked (useful when checking that ssl is disabled)
+        elif not port and conf_ssl:
+            self.log.debug('SSL is enabled  @{}:{} '
+                           '({})'.format(host, port, unit_name))
+            return True
+        elif not conf_ssl:
+            self.log.debug('SSL not enabled @{}:{} '
+                           '({})'.format(host, port, unit_name))
+            return False
+        else:
+            msg = ('Unknown condition when checking SSL status @{}:{} '
+                   '({})'.format(host, port, unit_name))
+            amulet.raise_status(amulet.FAIL, msg)
+
+    def validate_rmq_ssl_enabled_units(self, sentry_units, port=None):
+        """Check that ssl is enabled on rmq juju sentry units.
+
+        :param sentry_units: list of all rmq sentry units
+        :param port: optional ssl port override to validate
+        :returns: None if successful, otherwise return error message
+        """
+        for sentry_unit in sentry_units:
+            if not self.rmq_ssl_is_enabled_on_unit(sentry_unit, port=port):
+                return ('Unexpected condition:  ssl is disabled on unit '
+                        '({})'.format(sentry_unit.info['unit_name']))
+        return None
+
+    def validate_rmq_ssl_disabled_units(self, sentry_units):
+        """Check that ssl is enabled on listed rmq juju sentry units.
+
+        :param sentry_units: list of all rmq sentry units
+        :returns: True if successful.  Raise on error.
+        """
+        for sentry_unit in sentry_units:
+            if self.rmq_ssl_is_enabled_on_unit(sentry_unit):
+                return ('Unexpected condition:  ssl is enabled on unit '
+                        '({})'.format(sentry_unit.info['unit_name']))
+        return None
+
+    def configure_rmq_ssl_on(self, sentry_units, deployment,
+                             port=None, max_wait=60):
+        """Turn ssl charm config option on, with optional non-default
+        ssl port specification.  Confirm that it is enabled on every
+        unit.
+
+        :param sentry_units: list of sentry units
+        :param deployment: amulet deployment object pointer
+        :param port: amqp port, use defaults if None
+        :param max_wait: maximum time to wait in seconds to confirm
+        :returns: None if successful.  Raise on error.
+        """
+        self.log.debug('Setting ssl charm config option:  on')
+
+        # Enable RMQ SSL
+        config = {'ssl': 'on'}
+        if port:
+            config['ssl_port'] = port
+
+        deployment.d.configure('rabbitmq-server', config)
+
+        # Wait for unit status
+        self.rmq_wait_for_cluster(deployment)
+
+        # Confirm
+        tries = 0
+        ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port)
+        while ret and tries < (max_wait / 4):
+            time.sleep(4)
+            self.log.debug('Attempt {}: {}'.format(tries, ret))
+            ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port)
+            tries += 1
+
+        if ret:
+            amulet.raise_status(amulet.FAIL, ret)
+
+    def configure_rmq_ssl_off(self, sentry_units, deployment, max_wait=60):
+        """Turn ssl charm config option off, confirm that it is disabled
+        on every unit.
+
+        :param sentry_units: list of sentry units
+        :param deployment: amulet deployment object pointer
+        :param max_wait: maximum time to wait in seconds to confirm
+        :returns: None if successful.  Raise on error.
+        """
+        self.log.debug('Setting ssl charm config option:  off')
+
+        # Disable RMQ SSL
+        config = {'ssl': 'off'}
+        deployment.d.configure('rabbitmq-server', config)
+
+        # Wait for unit status
+        self.rmq_wait_for_cluster(deployment)
+
+        # Confirm
+        tries = 0
+        ret = self.validate_rmq_ssl_disabled_units(sentry_units)
+        while ret and tries < (max_wait / 4):
+            time.sleep(4)
+            self.log.debug('Attempt {}: {}'.format(tries, ret))
+            ret = self.validate_rmq_ssl_disabled_units(sentry_units)
+            tries += 1
+
+        if ret:
+            amulet.raise_status(amulet.FAIL, ret)
+
+    def connect_amqp_by_unit(self, sentry_unit, ssl=False,
+                             port=None, fatal=True,
+                             username="testuser1", password="changeme"):
+        """Establish and return a pika amqp connection to the rabbitmq service
+        running on a rmq juju unit.
+
+        :param sentry_unit: sentry unit pointer
+        :param ssl: boolean, default to False
+        :param port: amqp port, use defaults if None
+        :param fatal: boolean, default to True (raises on connect error)
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :returns: pika amqp connection pointer or None if failed and non-fatal
+        """
+        host = sentry_unit.info['public-address']
+        unit_name = sentry_unit.info['unit_name']
+
+        # Default port logic if port is not specified
+        if ssl and not port:
+            port = 5671
+        elif not ssl and not port:
+            port = 5672
+
+        self.log.debug('Connecting to amqp on {}:{} ({}) as '
+                       '{}...'.format(host, port, unit_name, username))
+
+        try:
+            credentials = pika.PlainCredentials(username, password)
+            parameters = pika.ConnectionParameters(host=host, port=port,
+                                                   credentials=credentials,
+                                                   ssl=ssl,
+                                                   connection_attempts=3,
+                                                   retry_delay=5,
+                                                   socket_timeout=1)
+            connection = pika.BlockingConnection(parameters)
+            assert connection.server_properties['product'] == 'RabbitMQ'
+            self.log.debug('Connect OK')
+            return connection
+        except Exception as e:
+            msg = ('amqp connection failed to {}:{} as '
+                   '{} ({})'.format(host, port, username, str(e)))
+            if fatal:
+                amulet.raise_status(amulet.FAIL, msg)
+            else:
+                self.log.warn(msg)
+                return None
+
+    def publish_amqp_message_by_unit(self, sentry_unit, message,
+                                     queue="test", ssl=False,
+                                     username="testuser1",
+                                     password="changeme",
+                                     port=None):
+        """Publish an amqp message to a rmq juju unit.
+
+        :param sentry_unit: sentry unit pointer
+        :param message: amqp message string
+        :param queue: message queue, default to test
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :param ssl: boolean, default to False
+        :param port: amqp port, use defaults if None
+        :returns: None.  Raises exception if publish failed.
+        """
+        self.log.debug('Publishing message to {} queue:\n{}'.format(queue,
+                                                                    message))
+        connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl,
+                                               port=port,
+                                               username=username,
+                                               password=password)
+
+        # NOTE(beisner): extra debug here re: pika hang potential:
+        #   https://github.com/pika/pika/issues/297
+        #   https://groups.google.com/forum/#!topic/rabbitmq-users/Ja0iyfF0Szw
+        self.log.debug('Defining channel...')
+        channel = connection.channel()
+        self.log.debug('Declaring queue...')
+        channel.queue_declare(queue=queue, auto_delete=False, durable=True)
+        self.log.debug('Publishing message...')
+        channel.basic_publish(exchange='', routing_key=queue, body=message)
+        self.log.debug('Closing channel...')
+        channel.close()
+        self.log.debug('Closing connection...')
+        connection.close()
+
+    def get_amqp_message_by_unit(self, sentry_unit, queue="test",
+                                 username="testuser1",
+                                 password="changeme",
+                                 ssl=False, port=None):
+        """Get an amqp message from a rmq juju unit.
+
+        :param sentry_unit: sentry unit pointer
+        :param queue: message queue, default to test
+        :param username: amqp user name, default to testuser1
+        :param password: amqp user password
+        :param ssl: boolean, default to False
+        :param port: amqp port, use defaults if None
+        :returns: amqp message body as string.  Raise if get fails.
+        """
+        connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl,
+                                               port=port,
+                                               username=username,
+                                               password=password)
+        channel = connection.channel()
+        method_frame, _, body = channel.basic_get(queue)
+
+        if method_frame:
+            self.log.debug('Retreived message from {} queue:\n{}'.format(queue,
+                                                                         body))
+            channel.basic_ack(method_frame.delivery_tag)
+            channel.close()
+            connection.close()
+            return body
+        else:
+            msg = 'No message retrieved.'
+            amulet.raise_status(amulet.FAIL, msg)
 
=== modified file 'hooks/charmhelpers/contrib/openstack/context.py'
--- hooks/charmhelpers/contrib/openstack/context.py	2015-07-29 18:07:31 +0000
+++ hooks/charmhelpers/contrib/openstack/context.py	2016-05-18 10:01:02 +0000
@@ -14,12 +14,13 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import glob
 import json
 import os
 import re
 import time
 from base64 import b64decode
-from subprocess import check_call
+from subprocess import check_call, CalledProcessError
 
 import six
 import yaml
@@ -44,16 +45,20 @@
     INFO,
     WARNING,
     ERROR,
+    status_set,
 )
 
 from charmhelpers.core.sysctl import create as sysctl_create
 from charmhelpers.core.strutils import bool_from_string
 
 from charmhelpers.core.host import (
+    get_bond_master,
+    is_phy_iface,
     list_nics,
     get_nic_hwaddr,
     mkdir,
     write_file,
+    pwgen,
 )
 from charmhelpers.contrib.hahelpers.cluster import (
     determine_apache_port,
@@ -84,6 +89,14 @@
     is_bridge_member,
 )
 from charmhelpers.contrib.openstack.utils import get_host_ip
+from charmhelpers.core.unitdata import kv
+
+try:
+    import psutil
+except ImportError:
+    apt_install('python-psutil', fatal=True)
+    import psutil
+
 CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
 ADDRESS_TYPES = ['admin', 'internal', 'public']
 
@@ -192,10 +205,50 @@
 class OSContextGenerator(object):
     """Base class for all context generators."""
     interfaces = []
+    related = False
+    complete = False
+    missing_data = []
 
     def __call__(self):
         raise NotImplementedError
 
+    def context_complete(self, ctxt):
+        """Check for missing data for the required context data.
+        Set self.missing_data if it exists and return False.
+        Set self.complete if no missing data and return True.
+        """
+        # Fresh start
+        self.complete = False
+        self.missing_data = []
+        for k, v in six.iteritems(ctxt):
+            if v is None or v == '':
+                if k not in self.missing_data:
+                    self.missing_data.append(k)
+
+        if self.missing_data:
+            self.complete = False
+            log('Missing required data: %s' % ' '.join(self.missing_data), level=INFO)
+        else:
+            self.complete = True
+        return self.complete
+
+    def get_related(self):
+        """Check if any of the context interfaces have relation ids.
+        Set self.related and return True if one of the interfaces
+        has relation ids.
+        """
+        # Fresh start
+        self.related = False
+        try:
+            for interface in self.interfaces:
+                if relation_ids(interface):
+                    self.related = True
+            return self.related
+        except AttributeError as e:
+            log("{} {}"
+                "".format(self, e), 'INFO')
+            return self.related
+
 
 class SharedDBContext(OSContextGenerator):
     interfaces = ['shared-db']
@@ -211,6 +264,7 @@
         self.database = database
         self.user = user
         self.ssl_dir = ssl_dir
+        self.rel_name = self.interfaces[0]
 
     def __call__(self):
         self.database = self.database or config('database')
@@ -244,6 +298,7 @@
             password_setting = self.relation_prefix + '_password'
 
         for rid in relation_ids(self.interfaces[0]):
+            self.related = True
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
                 host = rdata.get('db_host')
@@ -255,7 +310,7 @@
                     'database_password': rdata.get(password_setting),
                     'database_type': 'mysql'
                 }
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     db_ssl(rdata, ctxt, self.ssl_dir)
                     return ctxt
         return {}
@@ -276,6 +331,7 @@
 
         ctxt = {}
         for rid in relation_ids(self.interfaces[0]):
+            self.related = True
             for unit in related_units(rid):
                 rel_host = relation_get('host', rid=rid, unit=unit)
                 rel_user = relation_get('user', rid=rid, unit=unit)
@@ -285,7 +341,7 @@
                         'database_user': rel_user,
                         'database_password': rel_passwd,
                         'database_type': 'postgresql'}
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     return ctxt
 
         return {}
@@ -346,6 +402,7 @@
             ctxt['signing_dir'] = cachedir
 
         for rid in relation_ids(self.rel_name):
+            self.related = True
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
                 serv_host = rdata.get('service_host')
@@ -354,6 +411,7 @@
                 auth_host = format_ipv6_addr(auth_host) or auth_host
                 svc_protocol = rdata.get('service_protocol') or 'http'
                 auth_protocol = rdata.get('auth_protocol') or 'http'
+                api_version = rdata.get('api_version') or '2.0'
                 ctxt.update({'service_port': rdata.get('service_port'),
                              'service_host': serv_host,
                              'auth_host': auth_host,
@@ -362,9 +420,10 @@
                              'admin_user': rdata.get('service_username'),
                              'admin_password': rdata.get('service_password'),
                              'service_protocol': svc_protocol,
-                             'auth_protocol': auth_protocol})
+                             'auth_protocol': auth_protocol,
+                             'api_version': api_version})
 
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     # NOTE(jamespage) this is required for >= icehouse
                     # so a missing value just indicates keystone needs
                     # upgrading
@@ -403,6 +462,7 @@
         ctxt = {}
         for rid in relation_ids(self.rel_name):
             ha_vip_only = False
+            self.related = True
             for unit in related_units(rid):
                 if relation_get('clustered', rid=rid, unit=unit):
                     ctxt['clustered'] = True
@@ -435,7 +495,7 @@
                 ha_vip_only = relation_get('ha-vip-only',
                                            rid=rid, unit=unit) is not None
 
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     if 'rabbit_ssl_ca' in ctxt:
                         if not self.ssl_dir:
                             log("Charm not setup for ssl support but ssl ca "
@@ -467,7 +527,7 @@
             ctxt['oslo_messaging_flags'] = config_flags_parser(
                 oslo_messaging_flags)
 
-        if not context_complete(ctxt):
+        if not self.complete:
             return {}
 
         return ctxt
@@ -483,13 +543,15 @@
 
         log('Generating template context for ceph', level=DEBUG)
         mon_hosts = []
-        auth = None
-        key = None
-        use_syslog = str(config('use-syslog')).lower()
+        ctxt = {
+            'use_syslog': str(config('use-syslog')).lower()
+        }
         for rid in relation_ids('ceph'):
             for unit in related_units(rid):
-                auth = relation_get('auth', rid=rid, unit=unit)
-                key = relation_get('key', rid=rid, unit=unit)
+                if not ctxt.get('auth'):
+                    ctxt['auth'] = relation_get('auth', rid=rid, unit=unit)
+                if not ctxt.get('key'):
+                    ctxt['key'] = relation_get('key', rid=rid, unit=unit)
                 ceph_pub_addr = relation_get('ceph-public-address', rid=rid,
                                              unit=unit)
                 unit_priv_addr = relation_get('private-address', rid=rid,
@@ -498,15 +560,12 @@
                 ceph_addr = format_ipv6_addr(ceph_addr) or ceph_addr
                 mon_hosts.append(ceph_addr)
 
-        ctxt = {'mon_hosts': ' '.join(sorted(mon_hosts)),
-                'auth': auth,
-                'key': key,
-                'use_syslog': use_syslog}
+        ctxt['mon_hosts'] = ' '.join(sorted(mon_hosts))
 
         if not os.path.isdir('/etc/ceph'):
             os.mkdir('/etc/ceph')
 
-        if not context_complete(ctxt):
+        if not self.context_complete(ctxt):
             return {}
 
         ensure_packages(['ceph-common'])
@@ -579,15 +638,28 @@
         if config('haproxy-client-timeout'):
             ctxt['haproxy_client_timeout'] = config('haproxy-client-timeout')
 
+        if config('haproxy-queue-timeout'):
+            ctxt['haproxy_queue_timeout'] = config('haproxy-queue-timeout')
+
+        if config('haproxy-connect-timeout'):
+            ctxt['haproxy_connect_timeout'] = config('haproxy-connect-timeout')
+
         if config('prefer-ipv6'):
             ctxt['ipv6'] = True
             ctxt['local_host'] = 'ip6-localhost'
             ctxt['haproxy_host'] = '::'
-            ctxt['stat_port'] = ':::8888'
         else:
             ctxt['local_host'] = '127.0.0.1'
             ctxt['haproxy_host'] = '0.0.0.0'
-            ctxt['stat_port'] = ':8888'
+
+        ctxt['stat_port'] = '8888'
+
+        db = kv()
+        ctxt['stat_password'] = db.get('stat-password')
+        if not ctxt['stat_password']:
+            ctxt['stat_password'] = db.set('stat-password',
+                                           pwgen(32))
+            db.flush()
 
         for frontend in cluster_hosts:
             if (len(cluster_hosts[frontend]['backends']) > 1 or
@@ -878,19 +950,6 @@
 
         return calico_ctxt
 
-    def pg_ctxt(self):
-        driver = neutron_plugin_attribute(self.plugin, 'driver',
-                                          self.network_manager)
-        config = neutron_plugin_attribute(self.plugin, 'config',
-                                          self.network_manager)
-        pg_ctxt = {'core_plugin': driver,
-                    'neutron_plugin': 'plumgrid',
-                    'neutron_security_groups': self.neutron_security_groups,
-                    'local_ip': unit_private_ip(),
-                    'config': config}
-
-        return pg_ctxt
-
     def neutron_ctxt(self):
         if https():
             proto = 'https'
@@ -906,6 +965,31 @@
                 'neutron_url': '%s://%s:%s' % (proto, host, '9696')}
         return ctxt
 
+    def pg_ctxt(self):
+        driver = neutron_plugin_attribute(self.plugin, 'driver',
+                                          self.network_manager)
+        config = neutron_plugin_attribute(self.plugin, 'config',
+                                          self.network_manager)
+        ovs_ctxt = {'core_plugin': driver,
+                    'neutron_plugin': 'plumgrid',
+                    'neutron_security_groups': self.neutron_security_groups,
+                    'local_ip': unit_private_ip(),
+                    'config': config}
+        return ovs_ctxt
+
+    def midonet_ctxt(self):
+        driver = neutron_plugin_attribute(self.plugin, 'driver',
+                                          self.network_manager)
+        midonet_config = neutron_plugin_attribute(self.plugin, 'config',
+                                                  self.network_manager)
+        mido_ctxt = {'core_plugin': driver,
+                     'neutron_plugin': 'midonet',
+                     'neutron_security_groups': self.neutron_security_groups,
+                     'local_ip': unit_private_ip(),
+                     'config': midonet_config}
+
+        return mido_ctxt
+
     def __call__(self):
         if self.network_manager not in ['quantum', 'neutron']:
             return {}
@@ -927,6 +1011,8 @@
             ctxt.update(self.nuage_ctxt())
         elif self.plugin == 'plumgrid':
             ctxt.update(self.pg_ctxt())
+        elif self.plugin == 'midonet':
+            ctxt.update(self.midonet_ctxt())
 
         alchemy_flags = config('neutron-alchemy-flags')
         if alchemy_flags:
@@ -938,7 +1024,6 @@
 
 
 class NeutronPortContext(OSContextGenerator):
-    NIC_PREFIXES = ['eth', 'bond']
 
     def resolve_ports(self, ports):
         """Resolve NICs not yet bound to bridge(s)
@@ -950,7 +1035,18 @@
 
         hwaddr_to_nic = {}
         hwaddr_to_ip = {}
-        for nic in list_nics(self.NIC_PREFIXES):
+        for nic in list_nics():
+            # Ignore virtual interfaces (bond masters will be identified from
+            # their slaves)
+            if not is_phy_iface(nic):
+                continue
+
+            _nic = get_bond_master(nic)
+            if _nic:
+                log("Replacing iface '%s' with bond master '%s'" % (nic, _nic),
+                    level=DEBUG)
+                nic = _nic
+
             hwaddr = get_nic_hwaddr(nic)
             hwaddr_to_nic[hwaddr] = nic
             addresses = get_ipv4_addr(nic, fatal=False)
@@ -976,7 +1072,8 @@
                 # trust it to be the real external network).
                 resolved.append(entry)
 
-        return resolved
+        # Ensure no duplicates
+        return list(set(resolved))
 
 
 class OSConfigFlagContext(OSContextGenerator):
@@ -1016,6 +1113,20 @@
                 config_flags_parser(config_flags)}
 
 
+class LibvirtConfigFlagsContext(OSContextGenerator):
+    """
+    This context provides support for extending
+    the libvirt section through user-defined flags.
+    """
+    def __call__(self):
+        ctxt = {}
+        libvirt_flags = config('libvirt-flags')
+        if libvirt_flags:
+            ctxt['libvirt_flags'] = config_flags_parser(
+                libvirt_flags)
+        return ctxt
+
+
 class SubordinateConfigContext(OSContextGenerator):
 
     """
@@ -1048,7 +1159,7 @@
 
         ctxt = {
             ... other context ...
-            'subordinate_config': {
+            'subordinate_configuration': {
                 'DEFAULT': {
                     'key1': 'value1',
                 },
@@ -1066,13 +1177,22 @@
         :param config_file : Service's config file to query sections
         :param interface   : Subordinate interface to inspect
         """
-        self.service = service
         self.config_file = config_file
-        self.interface = interface
+        if isinstance(service, list):
+            self.services = service
+        else:
+            self.services = [service]
+        if isinstance(interface, list):
+            self.interfaces = interface
+        else:
+            self.interfaces = [interface]
 
     def __call__(self):
         ctxt = {'sections': {}}
-        for rid in relation_ids(self.interface):
+        rids = []
+        for interface in self.interfaces:
+            rids.extend(relation_ids(interface))
+        for rid in rids:
             for unit in related_units(rid):
                 sub_config = relation_get('subordinate_configuration',
                                           rid=rid, unit=unit)
@@ -1080,33 +1200,37 @@
                     try:
                         sub_config = json.loads(sub_config)
                     except:
-                        log('Could not parse JSON from subordinate_config '
-                            'setting from %s' % rid, level=ERROR)
-                        continue
-
-                    if self.service not in sub_config:
-                        log('Found subordinate_config on %s but it contained'
-                            'nothing for %s service' % (rid, self.service),
-                            level=INFO)
-                        continue
-
-                    sub_config = sub_config[self.service]
-                    if self.config_file not in sub_config:
-                        log('Found subordinate_config on %s but it contained'
-                            'nothing for %s' % (rid, self.config_file),
-                            level=INFO)
-                        continue
-
-                    sub_config = sub_config[self.config_file]
-                    for k, v in six.iteritems(sub_config):
-                        if k == 'sections':
-                            for section, config_dict in six.iteritems(v):
-                                log("adding section '%s'" % (section),
-                                    level=DEBUG)
-                                ctxt[k][section] = config_dict
-                        else:
-                            ctxt[k] = v
-
+                        log('Could not parse JSON from '
+                            'subordinate_configuration setting from %s'
+                            % rid, level=ERROR)
+                        continue
+
+                    for service in self.services:
+                        if service not in sub_config:
+                            log('Found subordinate_configuration on %s but it '
+                                'contained nothing for %s service'
+                                % (rid, service), level=INFO)
+                            continue
+
+                        sub_config = sub_config[service]
+                        if self.config_file not in sub_config:
+                            log('Found subordinate_configuration on %s but it '
+                                'contained nothing for %s'
+                                % (rid, self.config_file), level=INFO)
+                            continue
+
+                        sub_config = sub_config[self.config_file]
+                        for k, v in six.iteritems(sub_config):
+                            if k == 'sections':
+                                for section, config_list in six.iteritems(v):
+                                    log("adding section '%s'" % (section),
+                                        level=DEBUG)
+                                    if ctxt[k].get(section):
+                                        ctxt[k][section].extend(config_list)
+                                    else:
+                                        ctxt[k][section] = config_list
+                            else:
+                                ctxt[k] = v
         log("%d section(s) found" % (len(ctxt['sections'])), level=DEBUG)
         return ctxt
 
@@ -1143,13 +1267,11 @@
 
     @property
     def num_cpus(self):
-        try:
-            from psutil import NUM_CPUS
-        except ImportError:
-            apt_install('python-psutil', fatal=True)
-            from psutil import NUM_CPUS
-
-        return NUM_CPUS
+        # NOTE: use cpu_count if present (16.04 support)
+        if hasattr(psutil, 'cpu_count'):
+            return psutil.cpu_count()
+        else:
+            return psutil.NUM_CPUS
 
     def __call__(self):
         multiplier = config('worker-multiplier') or 0
@@ -1283,15 +1405,19 @@
     def __call__(self):
         ports = config('data-port')
         if ports:
+            # Map of {port/mac:bridge}
             portmap = parse_data_port_mappings(ports)
-            ports = portmap.values()
+            ports = portmap.keys()
+            # Resolve provided ports or mac addresses and filter out those
+            # already attached to a bridge.
             resolved = self.resolve_ports(ports)
+            # FIXME: is this necessary?
             normalized = {get_nic_hwaddr(port): port for port in resolved
                           if port not in ports}
             normalized.update({port: port for port in resolved
                                if port in ports})
             if resolved:
-                return {bridge: normalized[port] for bridge, port in
+                return {normalized[port]: bridge for port, bridge in
                         six.iteritems(portmap) if port in normalized.keys()}
 
         return None
@@ -1302,12 +1428,22 @@
     def __call__(self):
         ctxt = {}
         mappings = super(PhyNICMTUContext, self).__call__()
-        if mappings and mappings.values():
-            ports = mappings.values()
+        if mappings and mappings.keys():
+            ports = sorted(mappings.keys())
             napi_settings = NeutronAPIContext()()
             mtu = napi_settings.get('network_device_mtu')
+            all_ports = set()
+            # If any of ports is a vlan device, its underlying device must have
+            # mtu applied first.
+            for port in ports:
+                for lport in glob.glob("/sys/class/net/%s/lower_*" % port):
+                    lport = os.path.basename(lport)
+                    all_ports.add(lport.split('_')[1])
+
+            all_ports = list(all_ports)
+            all_ports.extend(ports)
             if mtu:
-                ctxt["devs"] = '\\n'.join(ports)
+                ctxt["devs"] = '\\n'.join(all_ports)
                 ctxt['mtu'] = mtu
 
         return ctxt
@@ -1338,7 +1474,110 @@
                     rdata.get('service_protocol') or 'http',
                     'auth_protocol':
                     rdata.get('auth_protocol') or 'http',
+                    'api_version':
+                    rdata.get('api_version') or '2.0',
                 }
-                if context_complete(ctxt):
+                if self.context_complete(ctxt):
                     return ctxt
         return {}
+
+
+class InternalEndpointContext(OSContextGenerator):
+    """Internal endpoint context.
+
+    This context provides the endpoint type used for communication between
+    services e.g. between Nova and Cinder internally. Openstack uses Public
+    endpoints by default so this allows admins to optionally use internal
+    endpoints.
+    """
+    def __call__(self):
+        return {'use_internal_endpoints': config('use-internal-endpoints')}
+
+
+class AppArmorContext(OSContextGenerator):
+    """Base class for apparmor contexts."""
+
+    def __init__(self):
+        self._ctxt = None
+        self.aa_profile = None
+        self.aa_utils_packages = ['apparmor-utils']
+
+    @property
+    def ctxt(self):
+        if self._ctxt is not None:
+            return self._ctxt
+        self._ctxt = self._determine_ctxt()
+        return self._ctxt
+
+    def _determine_ctxt(self):
+        """
+        Validate aa-profile-mode settings is disable, enforce, or complain.
+
+        :return ctxt: Dictionary of the apparmor profile or None
+        """
+        if config('aa-profile-mode') in ['disable', 'enforce', 'complain']:
+            ctxt = {'aa-profile-mode': config('aa-profile-mode')}
+        else:
+            ctxt = None
+        return ctxt
+
+    def __call__(self):
+        return self.ctxt
+
+    def install_aa_utils(self):
+        """
+        Install packages required for apparmor configuration.
+        """
+        log("Installing apparmor utils.")
+        ensure_packages(self.aa_utils_packages)
+
+    def manually_disable_aa_profile(self):
+        """
+        Manually disable an apparmor profile.
+
+        If aa-profile-mode is set to disabled (default) this is required as the
+        template has been written but apparmor is yet unaware of the profile
+        and aa-disable aa-profile fails. Without this the profile would kick
+        into enforce mode on the next service restart.
+
+        """
+        profile_path = '/etc/apparmor.d'
+        disable_path = '/etc/apparmor.d/disable'
+        if not os.path.lexists(os.path.join(disable_path, self.aa_profile)):
+            os.symlink(os.path.join(profile_path, self.aa_profile),
+                       os.path.join(disable_path, self.aa_profile))
+
+    def setup_aa_profile(self):
+        """
+        Setup an apparmor profile.
+        The ctxt dictionary will contain the apparmor profile mode and
+        the apparmor profile name.
+        Makes calls out to aa-disable, aa-complain, or aa-enforce to setup
+        the apparmor profile.
+        """
+        self()
+        if not self.ctxt:
+            log("Not enabling apparmor Profile")
+            return
+        self.install_aa_utils()
+        cmd = ['aa-{}'.format(self.ctxt['aa-profile-mode'])]
+        cmd.append(self.ctxt['aa-profile'])
+        log("Setting up the apparmor profile for {} in {} mode."
+            "".format(self.ctxt['aa-profile'], self.ctxt['aa-profile-mode']))
+        try:
+            check_call(cmd)
+        except CalledProcessError as e:
+            # If aa-profile-mode is set to disabled (default) manual
+            # disabling is required as the template has been written but
+            # apparmor is yet unaware of the profile and aa-disable aa-profile
+            # fails. If aa-disable learns to read profile files first this can
+            # be removed.
+            if self.ctxt['aa-profile-mode'] == 'disable':
+                log("Manually disabling the apparmor profile for {}."
+                    "".format(self.ctxt['aa-profile']))
+                self.manually_disable_aa_profile()
+                return
+            status_set('blocked', "Apparmor profile {} failed to be set to {}."
+                                  "".format(self.ctxt['aa-profile'],
+                                            self.ctxt['aa-profile-mode']))
+            raise e
 
=== modified file 'hooks/charmhelpers/contrib/openstack/ip.py'
--- hooks/charmhelpers/contrib/openstack/ip.py	2015-07-29 18:07:31 +0000
+++ hooks/charmhelpers/contrib/openstack/ip.py	2016-05-18 10:01:02 +0000
@@ -14,16 +14,19 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+
 from charmhelpers.core.hookenv import (
     config,
     unit_get,
     service_name,
+    network_get_primary_address,
 )
 from charmhelpers.contrib.network.ip import (
     get_address_in_network,
     is_address_in_network,
     is_ipv6,
     get_ipv6_addr,
+    resolve_network_cidr,
 )
 from charmhelpers.contrib.hahelpers.cluster import is_clustered
 
@@ -33,16 +36,19 @@
 
 ADDRESS_MAP = {
     PUBLIC: {
+        'binding': 'public',
         'config': 'os-public-network',
         'fallback': 'public-address',
         'override': 'os-public-hostname',
     },
     INTERNAL: {
+        'binding': 'internal',
         'config': 'os-internal-network',
         'fallback': 'private-address',
         'override': 'os-internal-hostname',
     },
     ADMIN: {
+        'binding': 'admin',
         'config': 'os-admin-network',
         'fallback': 'private-address',
         'override': 'os-admin-hostname',
@@ -110,7 +116,7 @@
     correct network. If clustered with no nets defined, return primary vip.
 
     If not clustered, return unit address ensuring address is on configured net
-    split if one is configured.
+    split if one is configured, or a Juju 2.0 extra-binding has been used.
 
     :param endpoint_type: Network endpoing type
     """
@@ -125,23 +131,45 @@
     net_type = ADDRESS_MAP[endpoint_type]['config']
     net_addr = config(net_type)
     net_fallback = ADDRESS_MAP[endpoint_type]['fallback']
+    binding = ADDRESS_MAP[endpoint_type]['binding']
     clustered = is_clustered()
-    if clustered:
-        if not net_addr:
-            # If no net-splits defined, we expect a single vip
-            resolved_address = vips[0]
-        else:
+
+    if clustered and vips:
+        if net_addr:
             for vip in vips:
                 if is_address_in_network(net_addr, vip):
                     resolved_address = vip
                     break
+        else:
+            # NOTE: endeavour to check vips against network space
+            #       bindings
+            try:
+                bound_cidr = resolve_network_cidr(
+                    network_get_primary_address(binding)
+                )
+                for vip in vips:
+                    if is_address_in_network(bound_cidr, vip):
+                        resolved_address = vip
+                        break
+            except NotImplementedError:
+                # If no net-splits configured and no support for extra
+                # bindings/network spaces so we expect a single vip
+                resolved_address = vips[0]
     else:
         if config('prefer-ipv6'):
             fallback_addr = get_ipv6_addr(exc_list=vips)[0]
         else:
             fallback_addr = unit_get(net_fallback)
 
-        resolved_address = get_address_in_network(net_addr, fallback_addr)
+        if net_addr:
+            resolved_address = get_address_in_network(net_addr, fallback_addr)
+        else:
+            # NOTE: only try to use extra bindings if legacy network
+            #       configuration is not in use
+            try:
+                resolved_address = network_get_primary_address(binding)
+            except NotImplementedError:
+                resolved_address = fallback_addr
 
     if resolved_address is None:
         raise ValueError("Unable to resolve a suitable IP address based on "
 
=== modified file 'hooks/charmhelpers/contrib/openstack/neutron.py'
--- hooks/charmhelpers/contrib/openstack/neutron.py	2015-10-02 15:06:23 +0000
+++ hooks/charmhelpers/contrib/openstack/neutron.py	2016-05-18 10:01:02 +0000
@@ -50,7 +50,7 @@
     if kernel_version() >= (3, 13):
         return []
     else:
-        return ['openvswitch-datapath-dkms']
+        return [headers_package(), 'openvswitch-datapath-dkms']
 
 
 # legacy
@@ -70,7 +70,7 @@
                                         relation_prefix='neutron',
                                         ssl_dir=QUANTUM_CONF_DIR)],
             'services': ['quantum-plugin-openvswitch-agent'],
-            'packages': [[headers_package()] + determine_dkms_package(),
+            'packages': [determine_dkms_package(),
                          ['quantum-plugin-openvswitch-agent']],
             'server_packages': ['quantum-server',
                                 'quantum-plugin-openvswitch'],
@@ -111,7 +111,7 @@
                                         relation_prefix='neutron',
                                         ssl_dir=NEUTRON_CONF_DIR)],
             'services': ['neutron-plugin-openvswitch-agent'],
-            'packages': [[headers_package()] + determine_dkms_package(),
+            'packages': [determine_dkms_package(),
                          ['neutron-plugin-openvswitch-agent']],
             'server_packages': ['neutron-server',
                                 'neutron-plugin-openvswitch'],
@@ -155,7 +155,7 @@
                                         relation_prefix='neutron',
                                         ssl_dir=NEUTRON_CONF_DIR)],
             'services': [],
-            'packages': [[headers_package()] + determine_dkms_package(),
+            'packages': [determine_dkms_package(),
                          ['neutron-plugin-cisco']],
             'server_packages': ['neutron-server',
                                 'neutron-plugin-cisco'],
@@ -174,7 +174,7 @@
                          'neutron-dhcp-agent',
                          'nova-api-metadata',
                          'etcd'],
-            'packages': [[headers_package()] + determine_dkms_package(),
+            'packages': [determine_dkms_package(),
                          ['calico-compute',
                           'bird',
                           'neutron-dhcp-agent',
@@ -209,6 +209,20 @@
             'server_packages': ['neutron-server',
                                 'neutron-plugin-plumgrid'],
             'server_services': ['neutron-server']
+        },
+        'midonet': {
+            'config': '/etc/neutron/plugins/midonet/midonet.ini',
+            'driver': 'midonet.neutron.plugin.MidonetPluginV2',
+            'contexts': [
+                context.SharedDBContext(user=config('neutron-database-user'),
+                                        database=config('neutron-database'),
+                                        relation_prefix='neutron',
+                                        ssl_dir=NEUTRON_CONF_DIR)],
+            'services': [],
+            'packages': [determine_dkms_package()],
+            'server_packages': ['neutron-server',
+                                'python-neutron-plugin-midonet'],
+            'server_services': ['neutron-server']
         }
     }
     if release >= 'icehouse':
@@ -219,6 +233,20 @@
                                              'neutron-plugin-ml2']
         # NOTE: patch in vmware renames nvp->nsx for icehouse onwards
         plugins['nvp'] = plugins['nsx']
+    if release >= 'kilo':
+        plugins['midonet']['driver'] = (
+            'neutron.plugins.midonet.plugin.MidonetPluginV2')
+    if release >= 'liberty':
+        plugins['midonet']['driver'] = (
+            'midonet.neutron.plugin_v1.MidonetPluginV2')
+        plugins['midonet']['server_packages'].remove(
+            'python-neutron-plugin-midonet')
+        plugins['midonet']['server_packages'].append(
+            'python-networking-midonet')
+        plugins['plumgrid']['driver'] = (
+            'networking_plumgrid.neutron.plugins.plugin.NeutronPluginPLUMgridV2')
+        plugins['plumgrid']['server_packages'].remove(
+            'neutron-plugin-plumgrid')
     return plugins
 
 
@@ -269,17 +297,30 @@
         return 'neutron'
 
 
-def parse_mappings(mappings):
+def parse_mappings(mappings, key_rvalue=False):
+    """By default mappings are lvalue keyed.
+
+    If key_rvalue is True, the mapping will be reversed to allow multiple
+    configs for the same lvalue.
+    """
     parsed = {}
     if mappings:
         mappings = mappings.split()
         for m in mappings:
             p = m.partition(':')
-            key = p[0].strip()
-            if p[1]:
-                parsed[key] = p[2].strip()
+
+            if key_rvalue:
+                key_index = 2
+                val_index = 0
+                # if there is no rvalue skip to next
+                if not p[1]:
+                    continue
             else:
-                parsed[key] = ''
+                key_index = 0
+                val_index = 2
+
+            key = p[key_index].strip()
+            parsed[key] = p[val_index].strip()
 
     return parsed
 
@@ -297,25 +338,25 @@
 def parse_data_port_mappings(mappings, default_bridge='br-data'):
     """Parse data port mappings.
 
-    Mappings must be a space-delimited list of bridge:port mappings.
+    Mappings must be a space-delimited list of bridge:port.
 
-    Returns dict of the form {bridge:port}.
+    Returns dict of the form {port:bridge} where ports may be mac addresses or
+    interface names.
     """
-    _mappings = parse_mappings(mappings)
+
+    # NOTE(dosaboy): we use rvalue for key to allow multiple values to be
+    # proposed for <port> since it may be a mac address which will differ
+    # across units this allowing first-known-good to be chosen.
+    _mappings = parse_mappings(mappings, key_rvalue=True)
     if not _mappings or list(_mappings.values()) == ['']:
         if not mappings:
             return {}
 
         # For backwards-compatibility we need to support port-only provided in
         # config.
-        _mappings = {default_bridge: mappings.split()[0]}
-
-    bridges = _mappings.keys()
-    ports = _mappings.values()
-    if len(set(bridges)) != len(bridges):
-        raise Exception("It is not allowed to have more than one port "
-                        "configured on the same bridge")
-
+        _mappings = {mappings.split()[0]: default_bridge}
+
+    ports = _mappings.keys()
     if len(set(ports)) != len(ports):
         raise Exception("It is not allowed to have the same port configured "
                         "on more than one bridge")
 
=== modified file 'hooks/charmhelpers/contrib/openstack/templating.py'
--- hooks/charmhelpers/contrib/openstack/templating.py	2015-07-29 18:07:31 +0000
+++ hooks/charmhelpers/contrib/openstack/templating.py	2016-05-18 10:01:02 +0000
@@ -18,7 +18,7 @@
 
 import six
 
-from charmhelpers.fetch import apt_install
+from charmhelpers.fetch import apt_install, apt_update
 from charmhelpers.core.hookenv import (
     log,
     ERROR,
@@ -29,6 +29,7 @@
 try:
     from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
 except ImportError:
+    apt_update(fatal=True)
     apt_install('python-jinja2', fatal=True)
     from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
 
@@ -112,7 +113,7 @@
 
     def complete_contexts(self):
         '''
-        Return a list of interfaces that have atisfied contexts.
+        Return a list of interfaces that have satisfied contexts.
         '''
         if self._complete_contexts:
             return self._complete_contexts
@@ -293,3 +294,30 @@
         [interfaces.extend(i.complete_contexts())
          for i in six.itervalues(self.templates)]
         return interfaces
+
+    def get_incomplete_context_data(self, interfaces):
+        '''
+        Return dictionary of relation status of interfaces and any missing
+        required context data. Example:
+            {'amqp': {'missing_data': ['rabbitmq_password'], 'related': True},
+             'zeromq-configuration': {'related': False}}
+        '''
+        incomplete_context_data = {}
+
+        for i in six.itervalues(self.templates):
+            for context in i.contexts:
+                for interface in interfaces:
+                    related = False
+                    if interface in context.interfaces:
+                        related = context.get_related()
+                        missing_data = context.missing_data
+                        if missing_data:
+                            incomplete_context_data[interface] = {'missing_data': missing_data}
+                        if related:
+                            if incomplete_context_data.get(interface):
+                                incomplete_context_data[interface].update({'related': True})
+                            else:
+                                incomplete_context_data[interface] = {'related': True}
+                        else:
+                            incomplete_context_data[interface] = {'related': False}
+        return incomplete_context_data
 
=== modified file 'hooks/charmhelpers/contrib/openstack/utils.py'
--- hooks/charmhelpers/contrib/openstack/utils.py	2015-07-29 18:07:31 +0000
+++ hooks/charmhelpers/contrib/openstack/utils.py	2016-05-18 10:01:02 +0000
@@ -1,5 +1,3 @@
-#!/usr/bin/python
-
 # Copyright 2014-2015 Canonical Limited.
 #
 # This file is part of charm-helpers.
@@ -24,8 +22,14 @@
 import json
 import os
 import sys
+import re
+import itertools
+import functools
 
 import six
+import tempfile
+import traceback
+import uuid
 import yaml
 
 from charmhelpers.contrib.network import ip
@@ -35,12 +39,18 @@
 )
 
 from charmhelpers.core.hookenv import (
+    action_fail,
+    action_set,
     config,
     log as juju_log,
     charm_dir,
+    DEBUG,
     INFO,
+    related_units,
     relation_ids,
-    relation_set
+    relation_set,
+    status_set,
+    hook_name
 )
 
 from charmhelpers.contrib.storage.linux.lvm import (
@@ -50,7 +60,9 @@
 )
 
 from charmhelpers.contrib.network.ip import (
-    get_ipv6_addr
+    get_ipv6_addr,
+    is_ipv6,
+    port_has_listener,
 )
 
 from charmhelpers.contrib.python.packages import (
@@ -58,7 +70,15 @@
     pip_install,
 )
 
-from charmhelpers.core.host import lsb_release, mounts, umount
+from charmhelpers.core.host import (
+    lsb_release,
+    mounts,
+    umount,
+    service_running,
+    service_pause,
+    service_resume,
+    restart_on_change_helper,
+)
 from charmhelpers.fetch import apt_install, apt_cache, install_remote
 from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
 from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
@@ -69,7 +89,6 @@
 DISTRO_PROPOSED = ('deb http://archive.ubuntu.com/ubuntu/ %s-proposed '
                    'restricted main multiverse universe')
 
-
 UBUNTU_OPENSTACK_RELEASE = OrderedDict([
     ('oneiric', 'diablo'),
     ('precise', 'essex'),
@@ -80,6 +99,7 @@
     ('utopic', 'juno'),
     ('vivid', 'kilo'),
     ('wily', 'liberty'),
+    ('xenial', 'mitaka'),
 ])
 
 
@@ -93,31 +113,74 @@
     ('2014.2', 'juno'),
     ('2015.1', 'kilo'),
     ('2015.2', 'liberty'),
+    ('2016.1', 'mitaka'),
 ])
 
-# The ugly duckling
+# The ugly duckling - must list releases oldest to newest
 SWIFT_CODENAMES = OrderedDict([
-    ('1.4.3', 'diablo'),
-    ('1.4.8', 'essex'),
-    ('1.7.4', 'folsom'),
-    ('1.8.0', 'grizzly'),
-    ('1.7.7', 'grizzly'),
-    ('1.7.6', 'grizzly'),
-    ('1.10.0', 'havana'),
-    ('1.9.1', 'havana'),
-    ('1.9.0', 'havana'),
-    ('1.13.1', 'icehouse'),
-    ('1.13.0', 'icehouse'),
-    ('1.12.0', 'icehouse'),
-    ('1.11.0', 'icehouse'),
-    ('2.0.0', 'juno'),
-    ('2.1.0', 'juno'),
-    ('2.2.0', 'juno'),
-    ('2.2.1', 'kilo'),
-    ('2.2.2', 'kilo'),
-    ('2.3.0', 'liberty'),
+    ('diablo',
+        ['1.4.3']),
+    ('essex',
+        ['1.4.8']),
+    ('folsom',
+        ['1.7.4']),
+    ('grizzly',
+        ['1.7.6', '1.7.7', '1.8.0']),
+    ('havana',
+        ['1.9.0', '1.9.1', '1.10.0']),
+    ('icehouse',
+        ['1.11.0', '1.12.0', '1.13.0', '1.13.1']),
+    ('juno',
+        ['2.0.0', '2.1.0', '2.2.0']),
+    ('kilo',
+        ['2.2.1', '2.2.2']),
+    ('liberty',
+        ['2.3.0', '2.4.0', '2.5.0']),
+    ('mitaka',
+        ['2.5.0', '2.6.0', '2.7.0']),
 ])
 
+# >= Liberty version->codename mapping
+PACKAGE_CODENAMES = {
+    'nova-common': OrderedDict([
+        ('12.0', 'liberty'),
+        ('13.0', 'mitaka'),
+    ]),
+    'neutron-common': OrderedDict([
+        ('7.0', 'liberty'),
+        ('8.0', 'mitaka'),
+    ]),
+    'cinder-common': OrderedDict([
+        ('7.0', 'liberty'),
+        ('8.0', 'mitaka'),
+    ]),
+    'keystone': OrderedDict([
+        ('8.0', 'liberty'),
+        ('8.1', 'liberty'),
+        ('9.0', 'mitaka'),
+    ]),
+    'horizon-common': OrderedDict([
+        ('8.0', 'liberty'),
+        ('9.0', 'mitaka'),
+    ]),
+    'ceilometer-common': OrderedDict([
+        ('5.0', 'liberty'),
+        ('6.0', 'mitaka'),
+    ]),
+    'heat-common': OrderedDict([
+        ('5.0', 'liberty'),
+        ('6.0', 'mitaka'),
+    ]),
+    'glance-common': OrderedDict([
+        ('11.0', 'liberty'),
+        ('12.0', 'mitaka'),
+    ]),
+    'openstack-dashboard': OrderedDict([
+        ('8.0', 'liberty'),
+        ('9.0', 'mitaka'),
+    ]),
+}
+
 DEFAULT_LOOPBACK_SIZE = '5G'
 
 
@@ -167,9 +230,9 @@
         error_out(e)
 
 
-def get_os_version_codename(codename):
+def get_os_version_codename(codename, version_map=OPENSTACK_CODENAMES):
     '''Determine OpenStack version number from codename.'''
-    for k, v in six.iteritems(OPENSTACK_CODENAMES):
+    for k, v in six.iteritems(version_map):
         if v == codename:
             return k
     e = 'Could not derive OpenStack version for '\
@@ -177,6 +240,33 @@
     error_out(e)
 
 
+def get_os_version_codename_swift(codename):
+    '''Determine OpenStack version number of swift from codename.'''
+    for k, v in six.iteritems(SWIFT_CODENAMES):
+        if k == codename:
+            return v[-1]
+    e = 'Could not derive swift version for '\
+        'codename: %s' % codename
+    error_out(e)
+
+
+def get_swift_codename(version):
+    '''Determine OpenStack codename that corresponds to swift version.'''
+    codenames = [k for k, v in six.iteritems(SWIFT_CODENAMES) if version in v]
+    if len(codenames) > 1:
+        # If more than one release codename contains this version we determine
+        # the actual codename based on the highest available install source.
+        for codename in reversed(codenames):
+            releases = UBUNTU_OPENSTACK_RELEASE
+            release = [k for k, v in six.iteritems(releases) if codename in v]
+            ret = subprocess.check_output(['apt-cache', 'policy', 'swift'])
+            if codename in ret or release[0] in ret:
+                return codename
+    elif len(codenames) == 1:
+        return codenames[0]
+    return None
+
+
 def get_os_codename_package(package, fatal=True):
     '''Derive OpenStack release codename from an installed package.'''
     import apt_pkg as apt
@@ -201,20 +291,33 @@
         error_out(e)
 
     vers = apt.upstream_version(pkg.current_ver.ver_str)
-
-    try:
-        if 'swift' in pkg.name:
-            swift_vers = vers[:5]
-            if swift_vers not in SWIFT_CODENAMES:
-                # Deal with 1.10.0 upward
-                swift_vers = vers[:6]
-            return SWIFT_CODENAMES[swift_vers]
-        else:
-            vers = vers[:6]
-            return OPENSTACK_CODENAMES[vers]
-    except KeyError:
-        e = 'Could not determine OpenStack codename for version %s' % vers
-        error_out(e)
+    if 'swift' in pkg.name:
+        # Fully x.y.z match for swift versions
+        match = re.match('^(\d+)\.(\d+)\.(\d+)', vers)
+    else:
+        # x.y match only for 20XX.X
+        # and ignore patch level for other packages
+        match = re.match('^(\d+)\.(\d+)', vers)
+
+    if match:
+        vers = match.group(0)
+
+    # >= Liberty independent project versions
+    if (package in PACKAGE_CODENAMES and
+            vers in PACKAGE_CODENAMES[package]):
+        return PACKAGE_CODENAMES[package][vers]
+    else:
+        # < Liberty co-ordinated project versions
+        try:
+            if 'swift' in pkg.name:
+                return get_swift_codename(vers)
+            else:
+                return OPENSTACK_CODENAMES[vers]
+        except KeyError:
+            if not fatal:
+                return None
+            e = 'Could not determine OpenStack codename for version %s' % vers
+            error_out(e)
 
 
 def get_os_version_package(pkg, fatal=True):
@@ -226,12 +329,14 @@
 
     if 'swift' in pkg:
         vers_map = SWIFT_CODENAMES
+        for cname, version in six.iteritems(vers_map):
+            if cname == codename:
+                return version[-1]
     else:
         vers_map = OPENSTACK_CODENAMES
-
-    for version, cname in six.iteritems(vers_map):
-        if cname == codename:
-            return version
+        for version, cname in six.iteritems(vers_map):
+            if cname == codename:
+                return version
     # e = "Could not determine OpenStack version for package: %s" % pkg
     # error_out(e)
 
@@ -256,12 +361,42 @@
 
 
 def import_key(keyid):
-    cmd = "apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 " \
-          "--recv-keys %s" % keyid
-    try:
-        subprocess.check_call(cmd.split(' '))
-    except subprocess.CalledProcessError:
-        error_out("Error importing repo key %s" % keyid)
+    key = keyid.strip()
+    if (key.startswith('-----BEGIN PGP PUBLIC KEY BLOCK-----') and
+            key.endswith('-----END PGP PUBLIC KEY BLOCK-----')):
+        juju_log("PGP key found (looks like ASCII Armor format)", level=DEBUG)
+        juju_log("Importing ASCII Armor PGP key", level=DEBUG)
+        with tempfile.NamedTemporaryFile() as keyfile:
+            with open(keyfile.name, 'w') as fd:
+                fd.write(key)
+                fd.write("\n")
+
+            cmd = ['apt-key', 'add', keyfile.name]
+            try:
+                subprocess.check_call(cmd)
+            except subprocess.CalledProcessError:
+                error_out("Error importing PGP key '%s'" % key)
+    else:
+        juju_log("PGP key found (looks like Radix64 format)", level=DEBUG)
+        juju_log("Importing PGP key from keyserver", level=DEBUG)
+        cmd = ['apt-key', 'adv', '--keyserver',
+               'hkp://keyserver.ubuntu.com:80', '--recv-keys', key]
+        try:
+            subprocess.check_call(cmd)
+        except subprocess.CalledProcessError:
+            error_out("Error importing PGP key '%s'" % key)
+
+
+def get_source_and_pgp_key(input):
+    """Look for a pgp key ID or ascii-armor key in the given input."""
+    index = input.strip()
+    index = input.rfind('|')
+    if index < 0:
+        return input, None
+
+    key = input[index + 1:].strip('|')
+    source = input[:index]
+    return source, key
 
 
 def configure_installation_source(rel):
@@ -273,16 +408,16 @@
         with open('/etc/apt/sources.list.d/juju_deb.list', 'w') as f:
             f.write(DISTRO_PROPOSED % ubuntu_rel)
     elif rel[:4] == "ppa:":
-        src = rel
+        src, key = get_source_and_pgp_key(rel)
+        if key:
+            import_key(key)
+
         subprocess.check_call(["add-apt-repository", "-y", src])
     elif rel[:3] == "deb":
-        l = len(rel.split('|'))
-        if l == 2:
-            src, key = rel.split('|')
-            juju_log("Importing PPA key from keyserver for %s" % src)
+        src, key = get_source_and_pgp_key(rel)
+        if key:
             import_key(key)
-        elif l == 1:
-            src = rel
+
         with open('/etc/apt/sources.list.d/juju_deb.list', 'w') as f:
             f.write(src)
     elif rel[:6] == 'cloud:':
@@ -327,6 +462,9 @@
             'liberty': 'trusty-updates/liberty',
             'liberty/updates': 'trusty-updates/liberty',
             'liberty/proposed': 'trusty-proposed/liberty',
+            'mitaka': 'trusty-updates/mitaka',
+            'mitaka/updates': 'trusty-updates/mitaka',
+            'mitaka/proposed': 'trusty-proposed/mitaka',
         }
 
         try:
@@ -392,9 +530,18 @@
     import apt_pkg as apt
     src = config('openstack-origin')
     cur_vers = get_os_version_package(package)
-    available_vers = get_os_version_install_source(src)
+    if "swift" in package:
+        codename = get_os_codename_install_source(src)
+        avail_vers = get_os_version_codename_swift(codename)
+    else:
+        avail_vers = get_os_version_install_source(src)
     apt.init()
-    return apt.version_compare(available_vers, cur_vers) == 1
+    if "swift" in package:
+        major_cur_vers = cur_vers.split('.', 1)[0]
+        major_avail_vers = avail_vers.split('.', 1)[0]
+        major_diff = apt.version_compare(major_avail_vers, major_cur_vers)
+        return avail_vers > cur_vers and (major_diff == 1 or major_diff == 0)
+    return apt.version_compare(avail_vers, cur_vers) == 1
 
 
 def ensure_block_device(block_device):
@@ -469,6 +616,12 @@
                                       relation_prefix=None):
     hosts = get_ipv6_addr(dynamic_only=False)
 
+    if config('vip'):
+        vips = config('vip').split()
+        for vip in vips:
+            if vip and is_ipv6(vip):
+                hosts.append(vip)
+
     kwargs = {'database': database,
               'username': database_user,
               'hostname': json.dumps(hosts)}
@@ -517,7 +670,7 @@
     return yaml.load(projects_yaml)
 
 
-def git_clone_and_install(projects_yaml, core_project, depth=1):
+def git_clone_and_install(projects_yaml, core_project):
     """
     Clone/install all specified OpenStack repositories.
 
@@ -567,6 +720,9 @@
     for p in projects['repositories']:
         repo = p['repository']
         branch = p['branch']
+        depth = '1'
+        if 'depth' in p.keys():
+            depth = p['depth']
         if p['name'] == 'requirements':
             repo_dir = _git_clone_and_install_single(repo, branch, depth,
                                                      parent_dir, http_proxy,
@@ -611,19 +767,14 @@
     """
     Clone and install a single git repository.
     """
-    dest_dir = os.path.join(parent_dir, os.path.basename(repo))
-
     if not os.path.exists(parent_dir):
         juju_log('Directory already exists at {}. '
                  'No need to create directory.'.format(parent_dir))
         os.mkdir(parent_dir)
 
-    if not os.path.exists(dest_dir):
-        juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
-        repo_dir = install_remote(repo, dest=parent_dir, branch=branch,
-                                  depth=depth)
-    else:
-        repo_dir = dest_dir
+    juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
+    repo_dir = install_remote(
+        repo, dest=parent_dir, branch=branch, depth=depth)
 
     venv = os.path.join(parent_dir, 'venv')
 
@@ -704,3 +855,721 @@
         return projects[key]
 
     return None
+
+
+def os_workload_status(configs, required_interfaces, charm_func=None):
+    """
+    Decorator to set workload status based on complete contexts
+    """
+    def wrap(f):
+        @wraps(f)
+        def wrapped_f(*args, **kwargs):
+            # Run the original function first
+            f(*args, **kwargs)
+            # Set workload status now that contexts have been
+            # acted on
+            set_os_workload_status(configs, required_interfaces, charm_func)
+        return wrapped_f
+    return wrap
+
+
+def set_os_workload_status(configs, required_interfaces, charm_func=None,
+                           services=None, ports=None):