Merge lp:~saviq/charms/trusty/openstack-dashboard/simplify-settings into lp:~sdn-charmers/charms/trusty/openstack-dashboard/add-settings

[Michał Sawicz]

Let plugin subordinates handle the plugin files themselves

[lp:~saviq/charms/trusty/openstack-dashboard/simplify-settings]

83. By Michał Sawicz on 2015-08-27

Drop empty line

84. By Michał Sawicz on 2015-08-27

Merge next

Unmerged revisions

84. By Michał Sawicz on 2015-08-27

Merge next

83. By Michał Sawicz on 2015-08-27

Drop empty line

82. By Michał Sawicz on 2015-08-27

Undo test changes

81. By Michał Sawicz on 2015-08-27

Revert support for plugin files, let subordinates write those themselves

80. By Michał Sawicz on 2015-08-27

Revert charmhelpers' pattern support

=== modified file 'charm-helpers-hooks.yaml'
--- charm-helpers-hooks.yaml	2015-04-07 13:58:41 +0000
+++ charm-helpers-hooks.yaml	2015-08-27 15:02:41 +0000
@@ -2,6 +2,7 @@
 destination: hooks/charmhelpers
 include:
     - core
+    - cli
     - fetch
     - contrib.openstack|inc=*
     - contrib.storage.linux

=== modified file 'config.yaml'
--- config.yaml	2015-04-13 15:34:13 +0000
+++ config.yaml	2015-08-27 15:02:41 +0000
@@ -7,9 +7,7 @@
     type: boolean
     default: False
     description: |
-      By default, all services will log into their corresponding log
-      files. Setting this to True will force all services to log to the
-      syslog.
+      Setting this to True will allow supporting services to log to syslog.
   openstack-origin:
     default: distro
     type: string
@@ -18,17 +16,27 @@
       distro (default), ppa:somecustom/ppa, a deb url sources entry,
       or a supported Cloud Archive release pocket.
 
-      Supported Cloud Archive sources include: cloud:precise-folsom,
-      cloud:precise-folsom/updates, cloud:precise-folsom/staging,
-      cloud:precise-folsom/proposed.
-
-      Note that updating this setting to a source that is known to
-      provide a later version of OpenStack will trigger a software
-      upgrade.
-
-      Note that when openstack-origin-git is specified, openstack
-      specific packages will be installed from source rather than
-      from the openstack-origin repository.
+      Supported Cloud Archive sources include:
+
+      cloud:<series>-<openstack-release>
+      cloud:<series>-<openstack-release>/updates
+      cloud:<series>-<openstack-release>/staging
+      cloud:<series>-<openstack-release>/proposed
+
+      For series=Precise we support cloud archives for openstack-release:
+         * icehouse
+
+      For series=Trusty we support cloud archives for openstack-release:
+         * juno
+         * kilo
+         * ...
+
+      NOTE: updating this setting to a source that is known to provide
+      a later version of OpenStack will trigger a software upgrade.
+
+      NOTE: when openstack-origin-git is specified, openstack specific
+      packages will be installed from source rather than from the
+      openstack-origin repository.
   openstack-origin-git:
     default:
     type: string
@@ -88,10 +96,9 @@
     default:
     description: |
       Base64-encoded SSL certificate to install and use for Horizon.
-      .
-         juju set openstack-dashbaord ssl_cert="$(cat cert| base64)" \
-                                      ssl_key="$(cat key| base64)"
-      .
+
+       juju set openstack-dashbaord ssl_cert="$(cat cert| base64)" \
+                                    ssl_key="$(cat key| base64)"
   ssl_key:
     type: string
     default:

=== added directory 'hooks/charmhelpers/cli'
=== added file 'hooks/charmhelpers/cli/__init__.py'
--- hooks/charmhelpers/cli/__init__.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/cli/__init__.py	2015-08-27 15:02:41 +0000
@@ -0,0 +1,191 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+import inspect
+import argparse
+import sys
+
+from six.moves import zip
+
+from charmhelpers.core import unitdata
+
+
+class OutputFormatter(object):
+    def __init__(self, outfile=sys.stdout):
+        self.formats = (
+            "raw",
+            "json",
+            "py",
+            "yaml",
+            "csv",
+            "tab",
+        )
+        self.outfile = outfile
+
+    def add_arguments(self, argument_parser):
+        formatgroup = argument_parser.add_mutually_exclusive_group()
+        choices = self.supported_formats
+        formatgroup.add_argument("--format", metavar='FMT',
+                                 help="Select output format for returned data, "
+                                      "where FMT is one of: {}".format(choices),
+                                 choices=choices, default='raw')
+        for fmt in self.formats:
+            fmtfunc = getattr(self, fmt)
+            formatgroup.add_argument("-{}".format(fmt[0]),
+                                     "--{}".format(fmt), action='store_const',
+                                     const=fmt, dest='format',
+                                     help=fmtfunc.__doc__)
+
+    @property
+    def supported_formats(self):
+        return self.formats
+
+    def raw(self, output):
+        """Output data as raw string (default)"""
+        if isinstance(output, (list, tuple)):
+            output = '\n'.join(map(str, output))
+        self.outfile.write(str(output))
+
+    def py(self, output):
+        """Output data as a nicely-formatted python data structure"""
+        import pprint
+        pprint.pprint(output, stream=self.outfile)
+
+    def json(self, output):
+        """Output data in JSON format"""
+        import json
+        json.dump(output, self.outfile)
+
+    def yaml(self, output):
+        """Output data in YAML format"""
+        import yaml
+        yaml.safe_dump(output, self.outfile)
+
+    def csv(self, output):
+        """Output data as excel-compatible CSV"""
+        import csv
+        csvwriter = csv.writer(self.outfile)
+        csvwriter.writerows(output)
+
+    def tab(self, output):
+        """Output data in excel-compatible tab-delimited format"""
+        import csv
+        csvwriter = csv.writer(self.outfile, dialect=csv.excel_tab)
+        csvwriter.writerows(output)
+
+    def format_output(self, output, fmt='raw'):
+        fmtfunc = getattr(self, fmt)
+        fmtfunc(output)
+
+
+class CommandLine(object):
+    argument_parser = None
+    subparsers = None
+    formatter = None
+    exit_code = 0
+
+    def __init__(self):
+        if not self.argument_parser:
+            self.argument_parser = argparse.ArgumentParser(description='Perform common charm tasks')
+        if not self.formatter:
+            self.formatter = OutputFormatter()
+            self.formatter.add_arguments(self.argument_parser)
+        if not self.subparsers:
+            self.subparsers = self.argument_parser.add_subparsers(help='Commands')
+
+    def subcommand(self, command_name=None):
+        """
+        Decorate a function as a subcommand. Use its arguments as the
+        command-line arguments"""
+        def wrapper(decorated):
+            cmd_name = command_name or decorated.__name__
+            subparser = self.subparsers.add_parser(cmd_name,
+                                                   description=decorated.__doc__)
+            for args, kwargs in describe_arguments(decorated):
+                subparser.add_argument(*args, **kwargs)
+            subparser.set_defaults(func=decorated)
+            return decorated
+        return wrapper
+
+    def test_command(self, decorated):
+        """
+        Subcommand is a boolean test function, so bool return values should be
+        converted to a 0/1 exit code.
+        """
+        decorated._cli_test_command = True
+        return decorated
+
+    def no_output(self, decorated):
+        """
+        Subcommand is not expected to return a value, so don't print a spurious None.
+        """
+        decorated._cli_no_output = True
+        return decorated
+
+    def subcommand_builder(self, command_name, description=None):
+        """
+        Decorate a function that builds a subcommand. Builders should accept a
+        single argument (the subparser instance) and return the function to be
+        run as the command."""
+        def wrapper(decorated):
+            subparser = self.subparsers.add_parser(command_name)
+            func = decorated(subparser)
+            subparser.set_defaults(func=func)
+            subparser.description = description or func.__doc__
+        return wrapper
+
+    def run(self):
+        "Run cli, processing arguments and executing subcommands."
+        arguments = self.argument_parser.parse_args()
+        argspec = inspect.getargspec(arguments.func)
+        vargs = []
+        for arg in argspec.args:
+            vargs.append(getattr(arguments, arg))
+        if argspec.varargs:
+            vargs.extend(getattr(arguments, argspec.varargs))
+        output = arguments.func(*vargs)
+        if getattr(arguments.func, '_cli_test_command', False):
+            self.exit_code = 0 if output else 1
+            output = ''
+        if getattr(arguments.func, '_cli_no_output', False):
+            output = ''
+        self.formatter.format_output(output, arguments.format)
+        if unitdata._KV:
+            unitdata._KV.flush()
+
+
+cmdline = CommandLine()
+
+
+def describe_arguments(func):
+    """
+    Analyze a function's signature and return a data structure suitable for
+    passing in as arguments to an argparse parser's add_argument() method."""
+
+    argspec = inspect.getargspec(func)
+    # we should probably raise an exception somewhere if func includes **kwargs
+    if argspec.defaults:
+        positional_args = argspec.args[:-len(argspec.defaults)]
+        keyword_names = argspec.args[-len(argspec.defaults):]
+        for arg, default in zip(keyword_names, argspec.defaults):
+            yield ('--{}'.format(arg),), {'default': default}
+    else:
+        positional_args = argspec.args
+
+    for arg in positional_args:
+        yield (arg,), {}
+    if argspec.varargs:
+        yield (argspec.varargs,), {'nargs': '*'}

=== added file 'hooks/charmhelpers/cli/benchmark.py'
--- hooks/charmhelpers/cli/benchmark.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/cli/benchmark.py	2015-08-27 15:02:41 +0000
@@ -0,0 +1,36 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from . import cmdline
+from charmhelpers.contrib.benchmark import Benchmark
+
+
[e-mail](command_name='benchmark-start')
+def start():
+    Benchmark.start()
+
+
[e-mail](command_name='benchmark-finish')
+def finish():
+    Benchmark.finish()
+
+
[e-mail]_builder('benchmark-composite', description="Set the benchmark composite score")
+def service(subparser):
+    subparser.add_argument("value", help="The composite score.")
+    subparser.add_argument("units", help="The units the composite score represents, i.e., 'reads/sec'.")
+    subparser.add_argument("direction", help="'asc' if a lower score is better, 'desc' if a higher score is better.")
+    return Benchmark.set_composite_score

=== added file 'hooks/charmhelpers/cli/commands.py'
--- hooks/charmhelpers/cli/commands.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/cli/commands.py	2015-08-27 15:02:41 +0000
@@ -0,0 +1,32 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+This module loads sub-modules into the python runtime so they can be
+discovered via the inspect module. In order to prevent flake8 from (rightfully)
+telling us these are unused modules, throw a ' # noqa' at the end of each import
+so that the warning is suppressed.
+"""
+
+from . import CommandLine  # noqa
+
+"""
+Import the sub-modules which have decorated subcommands to register with chlp.
+"""
+from . import host  # noqa
+from . import benchmark  # noqa
+from . import unitdata  # noqa
+from . import hookenv  # noqa

=== added file 'hooks/charmhelpers/cli/hookenv.py'
--- hooks/charmhelpers/cli/hookenv.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/cli/hookenv.py	2015-08-27 15:02:41 +0000
@@ -0,0 +1,23 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from . import cmdline
+from charmhelpers.core import hookenv
+
+
+cmdline.subcommand('relation-id')(hookenv.relation_id._wrapped)
+cmdline.subcommand('service-name')(hookenv.service_name)
+cmdline.subcommand('remote-service-name')(hookenv.remote_service_name._wrapped)

=== added file 'hooks/charmhelpers/cli/host.py'
--- hooks/charmhelpers/cli/host.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/cli/host.py	2015-08-27 15:02:41 +0000
@@ -0,0 +1,31 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from . import cmdline
+from charmhelpers.core import host
+
+
[e-mail]()
+def mounts():
+    "List mounts"
+    return host.mounts()
+
+
[e-mail]_builder('service', description="Control system services")
+def service(subparser):
+    subparser.add_argument("action", help="The action to perform (start, stop, etc...)")
+    subparser.add_argument("service_name", help="Name of the service to control")
+    return host.service

=== added file 'hooks/charmhelpers/cli/unitdata.py'
--- hooks/charmhelpers/cli/unitdata.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/cli/unitdata.py	2015-08-27 15:02:41 +0000
@@ -0,0 +1,39 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from . import cmdline
+from charmhelpers.core import unitdata
+
+
[e-mail]_builder('unitdata', description="Store and retrieve data")
+def unitdata_cmd(subparser):
+    nested = subparser.add_subparsers()
+    get_cmd = nested.add_parser('get', help='Retrieve data')
+    get_cmd.add_argument('key', help='Key to retrieve the value of')
+    get_cmd.set_defaults(action='get', value=None)
+    set_cmd = nested.add_parser('set', help='Store data')
+    set_cmd.add_argument('key', help='Key to set')
+    set_cmd.add_argument('value', help='Value to store')
+    set_cmd.set_defaults(action='set')
+
+    def _unitdata_cmd(action, key, value):
+        if action == 'get':
+            return unitdata.kv().get(key)
+        elif action == 'set':
+            unitdata.kv().set(key, value)
+            unitdata.kv().flush()
+            return ''
+    return _unitdata_cmd

=== modified file 'hooks/charmhelpers/contrib/hahelpers/cluster.py'
--- hooks/charmhelpers/contrib/hahelpers/cluster.py	2015-02-26 10:11:26 +0000
+++ hooks/charmhelpers/contrib/hahelpers/cluster.py	2015-08-27 15:02:41 +0000
@@ -44,6 +44,7 @@
     ERROR,
     WARNING,
     unit_get,
+    is_leader as juju_is_leader
 )
 from charmhelpers.core.decorators import (
     retry_on_exception,
@@ -52,6 +53,8 @@
     bool_from_string,
 )
 
+DC_RESOURCE_NAME = 'DC'
+
 
 class HAIncompleteConfig(Exception):
     pass
@@ -61,17 +64,30 @@
     pass
 
 
+class CRMDCNotFound(Exception):
+    pass
+
+
 def is_elected_leader(resource):
     """
     Returns True if the charm executing this is the elected cluster leader.
 
     It relies on two mechanisms to determine leadership:
-        1. If the charm is part of a corosync cluster, call corosync to
+        1. If juju is sufficiently new and leadership election is supported,
+        the is_leader command will be used.
+        2. If the charm is part of a corosync cluster, call corosync to
         determine leadership.
-        2. If the charm is not part of a corosync cluster, the leader is
+        3. If the charm is not part of a corosync cluster, the leader is
         determined as being "the alive unit with the lowest unit numer". In
         other words, the oldest surviving unit.
     """
+    try:
+        return juju_is_leader()
+    except NotImplementedError:
+        log('Juju leadership election feature not enabled'
+            ', using fallback support',
+            level=WARNING)
+
     if is_clustered():
         if not is_crm_leader(resource):
             log('Deferring action to CRM leader.', level=INFO)
@@ -95,7 +111,33 @@
     return False
 
 
-@retry_on_exception(5, base_delay=2, exc_type=CRMResourceNotFound)
+def is_crm_dc():
+    """
+    Determine leadership by querying the pacemaker Designated Controller
+    """
+    cmd = ['crm', 'status']
+    try:
+        status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+        if not isinstance(status, six.text_type):
+            status = six.text_type(status, "utf-8")
+    except subprocess.CalledProcessError as ex:
+        raise CRMDCNotFound(str(ex))
+
+    current_dc = ''
+    for line in status.split('\n'):
+        if line.startswith('Current DC'):
+            # Current DC: juju-lytrusty-machine-2 (168108163) - partition with quorum
+            current_dc = line.split(':')[1].split()[0]
+    if current_dc == get_unit_hostname():
+        return True
+    elif current_dc == 'NONE':
+        raise CRMDCNotFound('Current DC: NONE')
+
+    return False
+
+
+@retry_on_exception(5, base_delay=2,
+                    exc_type=(CRMResourceNotFound, CRMDCNotFound))
 def is_crm_leader(resource, retry=False):
     """
     Returns True if the charm calling this is the elected corosync leader,
@@ -104,6 +146,8 @@
     We allow this operation to be retried to avoid the possibility of getting a
     false negative. See LP #1396246 for more info.
     """
+    if resource == DC_RESOURCE_NAME:
+        return is_crm_dc()
     cmd = ['crm', 'resource', 'show', resource]
     try:
         status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)

=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/deployment.py'
--- hooks/charmhelpers/contrib/openstack/amulet/deployment.py	2015-04-23 14:53:21 +0000
+++ hooks/charmhelpers/contrib/openstack/amulet/deployment.py	2015-08-27 15:02:41 +0000
@@ -44,7 +44,7 @@
            Determine if the local branch being tested is derived from its
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
-        base_charms = ['mysql', 'mongodb']
+        base_charms = ['mysql', 'mongodb', 'nrpe']
 
         if self.series in ['precise', 'trusty']:
             base_series = self.series
@@ -79,9 +79,9 @@
         services.append(this_service)
         use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
                       'ceph-osd', 'ceph-radosgw']
-        # Openstack subordinate charms do not expose an origin option as that
-        # is controlled by the principle
-        ignore = ['neutron-openvswitch']
+        # Most OpenStack subordinate charms do not expose an origin option
+        # as that is controlled by the principle.
+        ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe']
 
         if self.openstack:
             for svc in services:
@@ -110,7 +110,8 @@
         (self.precise_essex, self.precise_folsom, self.precise_grizzly,
          self.precise_havana, self.precise_icehouse,
          self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
-         self.trusty_kilo, self.vivid_kilo) = range(10)
+         self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
+         self.wily_liberty) = range(12)
 
         releases = {
             ('precise', None): self.precise_essex,
@@ -121,8 +122,10 @@
             ('trusty', None): self.trusty_icehouse,
             ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
             ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
+            ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
             ('utopic', None): self.utopic_juno,
-            ('vivid', None): self.vivid_kilo}
+            ('vivid', None): self.vivid_kilo,
+            ('wily', None): self.wily_liberty}
         return releases[(self.series, self.openstack)]
 
     def _get_openstack_release_string(self):
@@ -138,9 +141,43 @@
             ('trusty', 'icehouse'),
             ('utopic', 'juno'),
             ('vivid', 'kilo'),
+            ('wily', 'liberty'),
         ])
         if self.openstack:
             os_origin = self.openstack.split(':')[1]
             return os_origin.split('%s-' % self.series)[1].split('/')[0]
         else:
             return releases[self.series]
+
+    def get_ceph_expected_pools(self, radosgw=False):
+        """Return a list of expected ceph pools in a ceph + cinder + glance
+        test scenario, based on OpenStack release and whether ceph radosgw
+        is flagged as present or not."""
+
+        if self._get_openstack_release() >= self.trusty_kilo:
+            # Kilo or later
+            pools = [
+                'rbd',
+                'cinder',
+                'glance'
+            ]
+        else:
+            # Juno or earlier
+            pools = [
+                'data',
+                'metadata',
+                'rbd',
+                'cinder',
+                'glance'
+            ]
+
+        if radosgw:
+            pools.extend([
+                '.rgw.root',
+                '.rgw.control',
+                '.rgw',
+                '.rgw.gc',
+                '.users.uid'
+            ])
+
+        return pools

=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/utils.py'
--- hooks/charmhelpers/contrib/openstack/amulet/utils.py	2015-01-26 09:46:38 +0000
+++ hooks/charmhelpers/contrib/openstack/amulet/utils.py	2015-08-27 15:02:41 +0000
@@ -14,16 +14,20 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import amulet
+import json
 import logging
 import os
+import six
 import time
 import urllib
 
+import cinderclient.v1.client as cinder_client
 import glanceclient.v1.client as glance_client
+import heatclient.v1.client as heat_client
 import keystoneclient.v2_0 as keystone_client
 import novaclient.v1_1.client as nova_client
-
-import six
+import swiftclient
 
 from charmhelpers.contrib.amulet.utils import (
     AmuletUtils
@@ -37,7 +41,7 @@
     """OpenStack amulet utilities.
 
        This class inherits from AmuletUtils and has additional support
-       that is specifically for use by OpenStack charms.
+       that is specifically for use by OpenStack charm tests.
        """
 
     def __init__(self, log_level=ERROR):
@@ -51,6 +55,8 @@
            Validate actual endpoint data vs expected endpoint data. The ports
            are used to find the matching endpoint.
            """
+        self.log.debug('Validating endpoint data...')
+        self.log.debug('actual: {}'.format(repr(endpoints)))
         found = False
         for ep in endpoints:
             self.log.debug('endpoint: {}'.format(repr(ep)))
@@ -77,6 +83,7 @@
            Validate a list of actual service catalog endpoints vs a list of
            expected service catalog endpoints.
            """
+        self.log.debug('Validating service catalog endpoint data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for k, v in six.iteritems(expected):
             if k in actual:
@@ -93,6 +100,7 @@
            Validate a list of actual tenant data vs list of expected tenant
            data.
            """
+        self.log.debug('Validating tenant data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -114,6 +122,7 @@
            Validate a list of actual role data vs a list of expected role
            data.
            """
+        self.log.debug('Validating role data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -134,6 +143,7 @@
            Validate a list of actual user data vs a list of expected user
            data.
            """
+        self.log.debug('Validating user data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -155,17 +165,30 @@
 
            Validate a list of actual flavors vs a list of expected flavors.
            """
+        self.log.debug('Validating flavor data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         act = [a.name for a in actual]
         return self._validate_list_data(expected, act)
 
     def tenant_exists(self, keystone, tenant):
         """Return True if tenant exists."""
+        self.log.debug('Checking if tenant exists ({})...'.format(tenant))
         return tenant in [t.name for t in keystone.tenants.list()]
 
+    def authenticate_cinder_admin(self, keystone_sentry, username,
+                                  password, tenant):
+        """Authenticates admin user with cinder."""
+        # NOTE(beisner): cinder python client doesn't accept tokens.
+        service_ip = \
+            keystone_sentry.relation('shared-db',
+                                     'mysql:shared-db')['private-address']
+        ept = "http://{}:5000/v2.0".format(service_ip.strip().decode('utf-8'))
+        return cinder_client.Client(username, password, tenant, ept)
+
     def authenticate_keystone_admin(self, keystone_sentry, user, password,
                                     tenant):
         """Authenticates admin user with the keystone admin endpoint."""
+        self.log.debug('Authenticating keystone admin...')
         unit = keystone_sentry
         service_ip = unit.relation('shared-db',
                                    'mysql:shared-db')['private-address']
@@ -175,6 +198,7 @@
 
     def authenticate_keystone_user(self, keystone, user, password, tenant):
         """Authenticates a regular user with the keystone public endpoint."""
+        self.log.debug('Authenticating keystone user ({})...'.format(user))
         ep = keystone.service_catalog.url_for(service_type='identity',
                                               endpoint_type='publicURL')
         return keystone_client.Client(username=user, password=password,
@@ -182,19 +206,49 @@
 
     def authenticate_glance_admin(self, keystone):
         """Authenticates admin user with glance."""
+        self.log.debug('Authenticating glance admin...')
         ep = keystone.service_catalog.url_for(service_type='image',
                                               endpoint_type='adminURL')
         return glance_client.Client(ep, token=keystone.auth_token)
 
+    def authenticate_heat_admin(self, keystone):
+        """Authenticates the admin user with heat."""
+        self.log.debug('Authenticating heat admin...')
+        ep = keystone.service_catalog.url_for(service_type='orchestration',
+                                              endpoint_type='publicURL')
+        return heat_client.Client(endpoint=ep, token=keystone.auth_token)
+
     def authenticate_nova_user(self, keystone, user, password, tenant):
         """Authenticates a regular user with nova-api."""
+        self.log.debug('Authenticating nova user ({})...'.format(user))
         ep = keystone.service_catalog.url_for(service_type='identity',
                                               endpoint_type='publicURL')
         return nova_client.Client(username=user, api_key=password,
                                   project_id=tenant, auth_url=ep)
 
+    def authenticate_swift_user(self, keystone, user, password, tenant):
+        """Authenticates a regular user with swift api."""
+        self.log.debug('Authenticating swift user ({})...'.format(user))
+        ep = keystone.service_catalog.url_for(service_type='identity',
+                                              endpoint_type='publicURL')
+        return swiftclient.Connection(authurl=ep,
+                                      user=user,
+                                      key=password,
+                                      tenant_name=tenant,
+                                      auth_version='2.0')
+
     def create_cirros_image(self, glance, image_name):
-        """Download the latest cirros image and upload it to glance."""
+        """Download the latest cirros image and upload it to glance,
+        validate and return a resource pointer.
+
+        :param glance: pointer to authenticated glance connection
+        :param image_name: display name for new image
+        :returns: glance image pointer
+        """
+        self.log.debug('Creating glance cirros image '
+                       '({})...'.format(image_name))
+
+        # Download cirros image
         http_proxy = os.getenv('AMULET_HTTP_PROXY')
         self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
         if http_proxy:
@@ -203,57 +257,67 @@
         else:
             opener = urllib.FancyURLopener()
 
-        f = opener.open("http://download.cirros-cloud.net/version/released")
+        f = opener.open('http://download.cirros-cloud.net/version/released')
         version = f.read().strip()
-        cirros_img = "cirros-{}-x86_64-disk.img".format(version)
+        cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
         local_path = os.path.join('tests', cirros_img)
 
         if not os.path.exists(local_path):
-            cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net",
+            cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
                                                   version, cirros_img)
             opener.retrieve(cirros_url, local_path)
         f.close()
 
+        # Create glance image
         with open(local_path) as f:
             image = glance.images.create(name=image_name, is_public=True,
                                          disk_format='qcow2',
                                          container_format='bare', data=f)
-        count = 1
-        status = image.status
-        while status != 'active' and count < 10:
-            time.sleep(3)
-            image = glance.images.get(image.id)
-            status = image.status
-            self.log.debug('image status: {}'.format(status))
-            count += 1
-
-        if status != 'active':
-            self.log.error('image creation timed out')
-            return None
+
+        # Wait for image to reach active status
+        img_id = image.id
+        ret = self.resource_reaches_status(glance.images, img_id,
+                                           expected_stat='active',
+                                           msg='Image status wait')
+        if not ret:
+            msg = 'Glance image failed to reach expected state.'
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Re-validate new image
+        self.log.debug('Validating image attributes...')
+        val_img_name = glance.images.get(img_id).name
+        val_img_stat = glance.images.get(img_id).status
+        val_img_pub = glance.images.get(img_id).is_public
+        val_img_cfmt = glance.images.get(img_id).container_format
+        val_img_dfmt = glance.images.get(img_id).disk_format
+        msg_attr = ('Image attributes - name:{} public:{} id:{} stat:{} '
+                    'container fmt:{} disk fmt:{}'.format(
+                        val_img_name, val_img_pub, img_id,
+                        val_img_stat, val_img_cfmt, val_img_dfmt))
+
+        if val_img_name == image_name and val_img_stat == 'active' \
+                and val_img_pub is True and val_img_cfmt == 'bare' \
+                and val_img_dfmt == 'qcow2':
+            self.log.debug(msg_attr)
+        else:
+            msg = ('Volume validation failed, {}'.format(msg_attr))
+            amulet.raise_status(amulet.FAIL, msg=msg)
 
         return image
 
     def delete_image(self, glance, image):
         """Delete the specified image."""
-        num_before = len(list(glance.images.list()))
-        glance.images.delete(image)
-
-        count = 1
-        num_after = len(list(glance.images.list()))
-        while num_after != (num_before - 1) and count < 10:
-            time.sleep(3)
-            num_after = len(list(glance.images.list()))
-            self.log.debug('number of images: {}'.format(num_after))
-            count += 1
-
-        if num_after != (num_before - 1):
-            self.log.error('image deletion timed out')
-            return False
-
-        return True
+
+        # /!\ DEPRECATION WARNING
+        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+                      'delete_resource instead of delete_image.')
+        self.log.debug('Deleting glance image ({})...'.format(image))
+        return self.delete_resource(glance.images, image, msg='glance image')
 
     def create_instance(self, nova, image_name, instance_name, flavor):
         """Create the specified instance."""
+        self.log.debug('Creating instance '
+                       '({}|{}|{})'.format(instance_name, image_name, flavor))
         image = nova.images.find(name=image_name)
         flavor = nova.flavors.find(name=flavor)
         instance = nova.servers.create(name=instance_name, image=image,
@@ -276,19 +340,265 @@
 
     def delete_instance(self, nova, instance):
         """Delete the specified instance."""
-        num_before = len(list(nova.servers.list()))
-        nova.servers.delete(instance)
-
-        count = 1
-        num_after = len(list(nova.servers.list()))
-        while num_after != (num_before - 1) and count < 10:
-            time.sleep(3)
-            num_after = len(list(nova.servers.list()))
-            self.log.debug('number of instances: {}'.format(num_after))
-            count += 1
-
-        if num_after != (num_before - 1):
-            self.log.error('instance deletion timed out')
-            return False
-
-        return True
+
+        # /!\ DEPRECATION WARNING
+        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+                      'delete_resource instead of delete_instance.')
+        self.log.debug('Deleting instance ({})...'.format(instance))
+        return self.delete_resource(nova.servers, instance,
+                                    msg='nova instance')
+
+    def create_or_get_keypair(self, nova, keypair_name="testkey"):
+        """Create a new keypair, or return pointer if it already exists."""
+        try:
+            _keypair = nova.keypairs.get(keypair_name)
+            self.log.debug('Keypair ({}) already exists, '
+                           'using it.'.format(keypair_name))
+            return _keypair
+        except:
+            self.log.debug('Keypair ({}) does not exist, '
+                           'creating it.'.format(keypair_name))
+
+        _keypair = nova.keypairs.create(name=keypair_name)
+        return _keypair
+
+    def create_cinder_volume(self, cinder, vol_name="demo-vol", vol_size=1,
+                             img_id=None, src_vol_id=None, snap_id=None):
+        """Create cinder volume, optionally from a glance image, OR
+        optionally as a clone of an existing volume, OR optionally
+        from a snapshot.  Wait for the new volume status to reach
+        the expected status, validate and return a resource pointer.
+
+        :param vol_name: cinder volume display name
+        :param vol_size: size in gigabytes
+        :param img_id: optional glance image id
+        :param src_vol_id: optional source volume id to clone
+        :param snap_id: optional snapshot id to use
+        :returns: cinder volume pointer
+        """
+        # Handle parameter input and avoid impossible combinations
+        if img_id and not src_vol_id and not snap_id:
+            # Create volume from image
+            self.log.debug('Creating cinder volume from glance image...')
+            bootable = 'true'
+        elif src_vol_id and not img_id and not snap_id:
+            # Clone an existing volume
+            self.log.debug('Cloning cinder volume...')
+            bootable = cinder.volumes.get(src_vol_id).bootable
+        elif snap_id and not src_vol_id and not img_id:
+            # Create volume from snapshot
+            self.log.debug('Creating cinder volume from snapshot...')
+            snap = cinder.volume_snapshots.find(id=snap_id)
+            vol_size = snap.size
+            snap_vol_id = cinder.volume_snapshots.get(snap_id).volume_id
+            bootable = cinder.volumes.get(snap_vol_id).bootable
+        elif not img_id and not src_vol_id and not snap_id:
+            # Create volume
+            self.log.debug('Creating cinder volume...')
+            bootable = 'false'
+        else:
+            # Impossible combination of parameters
+            msg = ('Invalid method use - name:{} size:{} img_id:{} '
+                   'src_vol_id:{} snap_id:{}'.format(vol_name, vol_size,
+                                                     img_id, src_vol_id,
+                                                     snap_id))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Create new volume
+        try:
+            vol_new = cinder.volumes.create(display_name=vol_name,
+                                            imageRef=img_id,
+                                            size=vol_size,
+                                            source_volid=src_vol_id,
+                                            snapshot_id=snap_id)
+            vol_id = vol_new.id
+        except Exception as e:
+            msg = 'Failed to create volume: {}'.format(e)
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Wait for volume to reach available status
+        ret = self.resource_reaches_status(cinder.volumes, vol_id,
+                                           expected_stat="available",
+                                           msg="Volume status wait")
+        if not ret:
+            msg = 'Cinder volume failed to reach expected state.'
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Re-validate new volume
+        self.log.debug('Validating volume attributes...')
+        val_vol_name = cinder.volumes.get(vol_id).display_name
+        val_vol_boot = cinder.volumes.get(vol_id).bootable
+        val_vol_stat = cinder.volumes.get(vol_id).status
+        val_vol_size = cinder.volumes.get(vol_id).size
+        msg_attr = ('Volume attributes - name:{} id:{} stat:{} boot:'
+                    '{} size:{}'.format(val_vol_name, vol_id,
+                                        val_vol_stat, val_vol_boot,
+                                        val_vol_size))
+
+        if val_vol_boot == bootable and val_vol_stat == 'available' \
+                and val_vol_name == vol_name and val_vol_size == vol_size:
+            self.log.debug(msg_attr)
+        else:
+            msg = ('Volume validation failed, {}'.format(msg_attr))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        return vol_new
+
+    def delete_resource(self, resource, resource_id,
+                        msg="resource", max_wait=120):
+        """Delete one openstack resource, such as one instance, keypair,
+        image, volume, stack, etc., and confirm deletion within max wait time.
+
+        :param resource: pointer to os resource type, ex:glance_client.images
+        :param resource_id: unique name or id for the openstack resource
+        :param msg: text to identify purpose in logging
+        :param max_wait: maximum wait time in seconds
+        :returns: True if successful, otherwise False
+        """
+        self.log.debug('Deleting OpenStack resource '
+                       '{} ({})'.format(resource_id, msg))
+        num_before = len(list(resource.list()))
+        resource.delete(resource_id)
+
+        tries = 0
+        num_after = len(list(resource.list()))
+        while num_after != (num_before - 1) and tries < (max_wait / 4):
+            self.log.debug('{} delete check: '
+                           '{} [{}:{}] {}'.format(msg, tries,
+                                                  num_before,
+                                                  num_after,
+                                                  resource_id))
+            time.sleep(4)
+            num_after = len(list(resource.list()))
+            tries += 1
+
+        self.log.debug('{}:  expected, actual count = {}, '
+                       '{}'.format(msg, num_before - 1, num_after))
+
+        if num_after == (num_before - 1):
+            return True
+        else:
+            self.log.error('{} delete timed out'.format(msg))
+            return False
+
+    def resource_reaches_status(self, resource, resource_id,
+                                expected_stat='available',
+                                msg='resource', max_wait=120):
+        """Wait for an openstack resources status to reach an
+           expected status within a specified time.  Useful to confirm that
+           nova instances, cinder vols, snapshots, glance images, heat stacks
+           and other resources eventually reach the expected status.
+
+        :param resource: pointer to os resource type, ex: heat_client.stacks
+        :param resource_id: unique id for the openstack resource
+        :param expected_stat: status to expect resource to reach
+        :param msg: text to identify purpose in logging
+        :param max_wait: maximum wait time in seconds
+        :returns: True if successful, False if status is not reached
+        """
+
+        tries = 0
+        resource_stat = resource.get(resource_id).status
+        while resource_stat != expected_stat and tries < (max_wait / 4):
+            self.log.debug('{} status check: '
+                           '{} [{}:{}] {}'.format(msg, tries,
+                                                  resource_stat,
+                                                  expected_stat,
+                                                  resource_id))
+            time.sleep(4)
+            resource_stat = resource.get(resource_id).status
+            tries += 1
+
+        self.log.debug('{}:  expected, actual status = {}, '
+                       '{}'.format(msg, resource_stat, expected_stat))
+
+        if resource_stat == expected_stat:
+            return True
+        else:
+            self.log.debug('{} never reached expected status: '
+                           '{}'.format(resource_id, expected_stat))
+            return False
+
+    def get_ceph_osd_id_cmd(self, index):
+        """Produce a shell command that will return a ceph-osd id."""
+        return ("`initctl list | grep 'ceph-osd ' | "
+                "awk 'NR=={} {{ print $2 }}' | "
+                "grep -o '[0-9]*'`".format(index + 1))
+
+    def get_ceph_pools(self, sentry_unit):
+        """Return a dict of ceph pools from a single ceph unit, with
+        pool name as keys, pool id as vals."""
+        pools = {}
+        cmd = 'sudo ceph osd lspools'
+        output, code = sentry_unit.run(cmd)
+        if code != 0:
+            msg = ('{} `{}` returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, code, output))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance,
+        for pool in str(output).split(','):
+            pool_id_name = pool.split(' ')
+            if len(pool_id_name) == 2:
+                pool_id = pool_id_name[0]
+                pool_name = pool_id_name[1]
+                pools[pool_name] = int(pool_id)
+
+        self.log.debug('Pools on {}: {}'.format(sentry_unit.info['unit_name'],
+                                                pools))
+        return pools
+
+    def get_ceph_df(self, sentry_unit):
+        """Return dict of ceph df json output, including ceph pool state.
+
+        :param sentry_unit: Pointer to amulet sentry instance (juju unit)
+        :returns: Dict of ceph df output
+        """
+        cmd = 'sudo ceph df --format=json'
+        output, code = sentry_unit.run(cmd)
+        if code != 0:
+            msg = ('{} `{}` returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, code, output))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+        return json.loads(output)
+
+    def get_ceph_pool_sample(self, sentry_unit, pool_id=0):
+        """Take a sample of attributes of a ceph pool, returning ceph
+        pool name, object count and disk space used for the specified
+        pool ID number.
+
+        :param sentry_unit: Pointer to amulet sentry instance (juju unit)
+        :param pool_id: Ceph pool ID
+        :returns: List of pool name, object count, kb disk space used
+        """
+        df = self.get_ceph_df(sentry_unit)
+        pool_name = df['pools'][pool_id]['name']
+        obj_count = df['pools'][pool_id]['stats']['objects']
+        kb_used = df['pools'][pool_id]['stats']['kb_used']
+        self.log.debug('Ceph {} pool (ID {}): {} objects, '
+                       '{} kb used'.format(pool_name, pool_id,
+                                           obj_count, kb_used))
+        return pool_name, obj_count, kb_used
+
+    def validate_ceph_pool_samples(self, samples, sample_type="resource pool"):
+        """Validate ceph pool samples taken over time, such as pool
+        object counts or pool kb used, before adding, after adding, and
+        after deleting items which affect those pool attributes.  The
+        2nd element is expected to be greater than the 1st; 3rd is expected
+        to be less than the 2nd.
+
+        :param samples: List containing 3 data samples
+        :param sample_type: String for logging and usage context
+        :returns: None if successful, Failure message otherwise
+        """
+        original, created, deleted = range(3)
+        if samples[created] <= samples[original] or \
+                samples[deleted] >= samples[created]:
+            return ('Ceph {} samples ({}) '
+                    'unexpected.'.format(sample_type, samples))
+        else:
+            self.log.debug('Ceph {} samples (OK): '
+                           '{}'.format(sample_type, samples))
+            return None

=== modified file 'hooks/charmhelpers/contrib/openstack/context.py'
--- hooks/charmhelpers/contrib/openstack/context.py	2015-06-02 12:12:49 +0000
+++ hooks/charmhelpers/contrib/openstack/context.py	2015-08-27 15:02:41 +0000
@@ -50,6 +50,8 @@
 from charmhelpers.core.strutils import bool_from_string
 
 from charmhelpers.core.host import (
+    get_bond_master,
+    is_phy_iface,
     list_nics,
     get_nic_hwaddr,
     mkdir,
@@ -122,21 +124,24 @@
          of specifying multiple key value pairs within the same string. For
          example, a string in the format of 'key1=value1, key2=value2' will
          return a dict of:
-         {'key1': 'value1',
-          'key2': 'value2'}.
+
+             {'key1': 'value1',
+              'key2': 'value2'}.
 
       2. A string in the above format, but supporting a comma-delimited list
          of values for the same key. For example, a string in the format of
          'key1=value1, key2=value3,value4,value5' will return a dict of:
-         {'key1', 'value1',
-          'key2', 'value2,value3,value4'}
+
+             {'key1', 'value1',
+              'key2', 'value2,value3,value4'}
 
       3. A string containing a colon character (:) prior to an equal
          character (=) will be treated as yaml and parsed as such. This can be
          used to specify more complex key value pairs. For example,
          a string in the format of 'key1: subkey1=value1, subkey2=value2' will
          return a dict of:
-         {'key1', 'subkey1=value1, subkey2=value2'}
+
+             {'key1', 'subkey1=value1, subkey2=value2'}
 
     The provided config_flags string may be a list of comma-separated values
     which themselves may be comma-separated list of values.
@@ -194,15 +199,6 @@
         raise NotImplementedError
 
 
-class OSPatternContextGenerator(OSContextGenerator):
-    """Base class for pattern context generators.
-
-    __call__ should return a dictionary of { tuple: dict }, where the tuple
-    will be used as input for format() for the filename pattern as registered
-    by OSConfigRenderer.register_pattern().
-    """
-
-
 class SharedDBContext(OSContextGenerator):
     interfaces = ['shared-db']
 
@@ -249,7 +245,7 @@
         if self.relation_prefix:
             password_setting = self.relation_prefix + '_password'
 
-        for rid in relation_ids('shared-db'):
+        for rid in relation_ids(self.interfaces[0]):
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
                 host = rdata.get('db_host')
@@ -900,8 +896,6 @@
         return ctxt
 
     def __call__(self):
-        self._ensure_packages()
-
         if self.network_manager not in ['quantum', 'neutron']:
             return {}
 
@@ -931,7 +925,6 @@
 
 
 class NeutronPortContext(OSContextGenerator):
-    NIC_PREFIXES = ['eth', 'bond']
 
     def resolve_ports(self, ports):
         """Resolve NICs not yet bound to bridge(s)
@@ -943,7 +936,18 @@
 
         hwaddr_to_nic = {}
         hwaddr_to_ip = {}
-        for nic in list_nics(self.NIC_PREFIXES):
+        for nic in list_nics():
+            # Ignore virtual interfaces (bond masters will be identified from
+            # their slaves)
+            if not is_phy_iface(nic):
+                continue
+
+            _nic = get_bond_master(nic)
+            if _nic:
+                log("Replacing iface '%s' with bond master '%s'" % (nic, _nic),
+                    level=DEBUG)
+                nic = _nic
+
             hwaddr = get_nic_hwaddr(nic)
             hwaddr_to_nic[hwaddr] = nic
             addresses = get_ipv4_addr(nic, fatal=False)
@@ -969,7 +973,8 @@
                 # trust it to be the real external network).
                 resolved.append(entry)
 
-        return resolved
+        # Ensure no duplicates
+        return list(set(resolved))
 
 
 class OSConfigFlagContext(OSContextGenerator):
@@ -1059,13 +1064,22 @@
         :param config_file : Service's config file to query sections
         :param interface   : Subordinate interface to inspect
         """
-        self.service = service
         self.config_file = config_file
-        self.interface = interface
+        if isinstance(service, list):
+            self.services = service
+        else:
+            self.services = [service]
+        if isinstance(interface, list):
+            self.interfaces = interface
+        else:
+            self.interfaces = [interface]
 
     def __call__(self):
         ctxt = {'sections': {}}
-        for rid in relation_ids(self.interface):
+        rids = []
+        for interface in self.interfaces:
+            rids.extend(relation_ids(interface))
+        for rid in rids:
             for unit in related_units(rid):
                 sub_config = relation_get('subordinate_configuration',
                                           rid=rid, unit=unit)
@@ -1077,29 +1091,32 @@
                             'setting from %s' % rid, level=ERROR)
                         continue
 
-                    if self.service not in sub_config:
-                        log('Found subordinate_config on %s but it contained'
-                            'nothing for %s service' % (rid, self.service),
-                            level=INFO)
-                        continue
-
-                    sub_config = sub_config[self.service]
-                    if self.config_file not in sub_config:
-                        log('Found subordinate_config on %s but it contained'
-                            'nothing for %s' % (rid, self.config_file),
-                            level=INFO)
-                        continue
-
-                    sub_config = sub_config[self.config_file]
-                    for k, v in six.iteritems(sub_config):
-                        if k == 'sections':
-                            for section, config_dict in six.iteritems(v):
-                                log("adding section '%s'" % (section),
-                                    level=DEBUG)
-                                ctxt[k][section] = config_dict
-                        else:
-                            ctxt[k] = v
-
+                    for service in self.services:
+                        if service not in sub_config:
+                            log('Found subordinate_config on %s but it contained'
+                                'nothing for %s service' % (rid, service),
+                                level=INFO)
+                            continue
+
+                        sub_config = sub_config[service]
+                        if self.config_file not in sub_config:
+                            log('Found subordinate_config on %s but it contained'
+                                'nothing for %s' % (rid, self.config_file),
+                                level=INFO)
+                            continue
+
+                        sub_config = sub_config[self.config_file]
+                        for k, v in six.iteritems(sub_config):
+                            if k == 'sections':
+                                for section, config_list in six.iteritems(v):
+                                    log("adding section '%s'" % (section),
+                                        level=DEBUG)
+                                    if ctxt[k].get(section):
+                                        ctxt[k][section].extend(config_list)
+                                    else:
+                                        ctxt[k][section] = config_list
+                            else:
+                                ctxt[k] = v
         log("%d section(s) found" % (len(ctxt['sections'])), level=DEBUG)
         return ctxt
 
@@ -1276,15 +1293,19 @@
     def __call__(self):
         ports = config('data-port')
         if ports:
+            # Map of {port/mac:bridge}
             portmap = parse_data_port_mappings(ports)
-            ports = portmap.values()
+            ports = portmap.keys()
+            # Resolve provided ports or mac addresses and filter out those
+            # already attached to a bridge.
             resolved = self.resolve_ports(ports)
+            # FIXME: is this necessary?
             normalized = {get_nic_hwaddr(port): port for port in resolved
                           if port not in ports}
             normalized.update({port: port for port in resolved
                                if port in ports})
             if resolved:
-                return {bridge: normalized[port] for bridge, port in
+                return {bridge: normalized[port] for port, bridge in
                         six.iteritems(portmap) if port in normalized.keys()}
 
         return None

=== modified file 'hooks/charmhelpers/contrib/openstack/ip.py'
--- hooks/charmhelpers/contrib/openstack/ip.py	2015-02-26 10:11:26 +0000
+++ hooks/charmhelpers/contrib/openstack/ip.py	2015-08-27 15:02:41 +0000
@@ -17,6 +17,7 @@
 from charmhelpers.core.hookenv import (
     config,
     unit_get,
+    service_name,
 )
 from charmhelpers.contrib.network.ip import (
     get_address_in_network,
@@ -26,8 +27,6 @@
 )
 from charmhelpers.contrib.hahelpers.cluster import is_clustered
 
-from functools import partial
-
 PUBLIC = 'public'
 INTERNAL = 'int'
 ADMIN = 'admin'
@@ -35,15 +34,18 @@
 ADDRESS_MAP = {
     PUBLIC: {
         'config': 'os-public-network',
-        'fallback': 'public-address'
+        'fallback': 'public-address',
+        'override': 'os-public-hostname',
     },
     INTERNAL: {
         'config': 'os-internal-network',
-        'fallback': 'private-address'
+        'fallback': 'private-address',
+        'override': 'os-internal-hostname',
     },
     ADMIN: {
         'config': 'os-admin-network',
-        'fallback': 'private-address'
+        'fallback': 'private-address',
+        'override': 'os-admin-hostname',
     }
 }
 
@@ -57,15 +59,50 @@
     :param endpoint_type: str endpoint type to resolve.
     :param returns: str base URL for services on the current service unit.
     """
-    scheme = 'http'
-    if 'https' in configs.complete_contexts():
-        scheme = 'https'
+    scheme = _get_scheme(configs)
+
     address = resolve_address(endpoint_type)
     if is_ipv6(address):
         address = "[{}]".format(address)
+
     return '%s://%s' % (scheme, address)
 
 
+def _get_scheme(configs):
+    """Returns the scheme to use for the url (either http or https)
+    depending upon whether https is in the configs value.
+
+    :param configs: OSTemplateRenderer config templating object to inspect
+                    for a complete https context.
+    :returns: either 'http' or 'https' depending on whether https is
+              configured within the configs context.
+    """
+    scheme = 'http'
+    if configs and 'https' in configs.complete_contexts():
+        scheme = 'https'
+    return scheme
+
+
+def _get_address_override(endpoint_type=PUBLIC):
+    """Returns any address overrides that the user has defined based on the
+    endpoint type.
+
+    Note: this function allows for the service name to be inserted into the
+    address if the user specifies {service_name}.somehost.org.
+
+    :param endpoint_type: the type of endpoint to retrieve the override
+                          value for.
+    :returns: any endpoint address or hostname that the user has overridden
+              or None if an override is not present.
+    """
+    override_key = ADDRESS_MAP[endpoint_type]['override']
+    addr_override = config(override_key)
+    if not addr_override:
+        return None
+    else:
+        return addr_override.format(service_name=service_name())
+
+
 def resolve_address(endpoint_type=PUBLIC):
     """Return unit address depending on net config.
 
@@ -77,7 +114,10 @@
 
     :param endpoint_type: Network endpoing type
     """
-    resolved_address = None
+    resolved_address = _get_address_override(endpoint_type)
+    if resolved_address:
+        return resolved_address
+
     vips = config('vip')
     if vips:
         vips = vips.split()
@@ -109,38 +149,3 @@
                          "clustered=%s)" % (net_type, clustered))
 
     return resolved_address
-
-
-def endpoint_url(configs, url_template, port, endpoint_type=PUBLIC,
-                 override=None):
-    """Returns the correct endpoint URL to advertise to Keystone.
-
-    This method provides the correct endpoint URL which should be advertised to
-    the keystone charm for endpoint creation. This method allows for the url to
-    be overridden to force a keystone endpoint to have specific URL for any of
-    the defined scopes (admin, internal, public).
-
-    :param configs: OSTemplateRenderer config templating object to inspect
-                    for a complete https context.
-    :param url_template: str format string for creating the url template. Only
-                         two values will be passed - the scheme+hostname
-                        returned by the canonical_url and the port.
-    :param endpoint_type: str endpoint type to resolve.
-    :param override: str the name of the config option which overrides the
-                     endpoint URL defined by the charm itself. None will
-                     disable any overrides (default).
-    """
-    if override:
-        # Return any user-defined overrides for the keystone endpoint URL.
-        user_value = config(override)
-        if user_value:
-            return user_value.strip()
-
-    return url_template % (canonical_url(configs, endpoint_type), port)
-
-
-public_endpoint = partial(endpoint_url, endpoint_type=PUBLIC)
-
-internal_endpoint = partial(endpoint_url, endpoint_type=INTERNAL)
-
-admin_endpoint = partial(endpoint_url, endpoint_type=ADMIN)

=== modified file 'hooks/charmhelpers/contrib/openstack/neutron.py'
--- hooks/charmhelpers/contrib/openstack/neutron.py	2015-04-16 20:24:28 +0000
+++ hooks/charmhelpers/contrib/openstack/neutron.py	2015-08-27 15:02:41 +0000
@@ -172,14 +172,16 @@
             'services': ['calico-felix',
                          'bird',
                          'neutron-dhcp-agent',
-                         'nova-api-metadata'],
+                         'nova-api-metadata',
+                         'etcd'],
             'packages': [[headers_package()] + determine_dkms_package(),
                          ['calico-compute',
                           'bird',
                           'neutron-dhcp-agent',
-                          'nova-api-metadata']],
-            'server_packages': ['neutron-server', 'calico-control'],
-            'server_services': ['neutron-server']
+                          'nova-api-metadata',
+                          'etcd']],
+            'server_packages': ['neutron-server', 'calico-control', 'etcd'],
+            'server_services': ['neutron-server', 'etcd']
         },
         'vsp': {
             'config': '/etc/neutron/plugins/nuage/nuage_plugin.ini',
@@ -253,14 +255,30 @@
         return 'neutron'
 
 
-def parse_mappings(mappings):
+def parse_mappings(mappings, key_rvalue=False):
+    """By default mappings are lvalue keyed.
+
+    If key_rvalue is True, the mapping will be reversed to allow multiple
+    configs for the same lvalue.
+    """
     parsed = {}
     if mappings:
-        mappings = mappings.split(' ')
+        mappings = mappings.split()
         for m in mappings:
             p = m.partition(':')
-            if p[1] == ':':
-                parsed[p[0].strip()] = p[2].strip()
+
+            if key_rvalue:
+                key_index = 2
+                val_index = 0
+                # if there is no rvalue skip to next
+                if not p[1]:
+                    continue
+            else:
+                key_index = 0
+                val_index = 2
+
+            key = p[key_index].strip()
+            parsed[key] = p[val_index].strip()
 
     return parsed
 
@@ -278,25 +296,25 @@
 def parse_data_port_mappings(mappings, default_bridge='br-data'):
     """Parse data port mappings.
 
-    Mappings must be a space-delimited list of bridge:port mappings.
+    Mappings must be a space-delimited list of port:bridge mappings.
 
-    Returns dict of the form {bridge:port}.
+    Returns dict of the form {port:bridge} where port may be an mac address or
+    interface name.
     """
-    _mappings = parse_mappings(mappings)
-    if not _mappings:
+
+    # NOTE(dosaboy): we use rvalue for key to allow multiple values to be
+    # proposed for <port> since it may be a mac address which will differ
+    # across units this allowing first-known-good to be chosen.
+    _mappings = parse_mappings(mappings, key_rvalue=True)
+    if not _mappings or list(_mappings.values()) == ['']:
         if not mappings:
             return {}
 
         # For backwards-compatibility we need to support port-only provided in
         # config.
-        _mappings = {default_bridge: mappings.split(' ')[0]}
-
-    bridges = _mappings.keys()
-    ports = _mappings.values()
-    if len(set(bridges)) != len(bridges):
-        raise Exception("It is not allowed to have more than one port "
-                        "configured on the same bridge")
-
+        _mappings = {mappings.split()[0]: default_bridge}
+
+    ports = _mappings.keys()
     if len(set(ports)) != len(ports):
         raise Exception("It is not allowed to have the same port configured "
                         "on more than one bridge")
@@ -309,6 +327,8 @@
 
     Mappings must be a space-delimited list of provider:start:end mappings.
 
+    The start:end range is optional and may be omitted.
+
     Returns dict of the form {provider: (start, end)}.
     """
     _mappings = parse_mappings(mappings)

=== modified file 'hooks/charmhelpers/contrib/openstack/templates/ceph.conf'
--- hooks/charmhelpers/contrib/openstack/templates/ceph.conf	2015-02-24 05:48:43 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/ceph.conf	2015-08-27 15:02:41 +0000
@@ -5,11 +5,11 @@
 ###############################################################################
 [global]
 {% if auth -%}
- auth_supported = {{ auth }}
- keyring = /etc/ceph/$cluster.$name.keyring
- mon host = {{ mon_hosts }}
+auth_supported = {{ auth }}
+keyring = /etc/ceph/$cluster.$name.keyring
+mon host = {{ mon_hosts }}
 {% endif -%}
- log to syslog = {{ use_syslog }}
- err to syslog = {{ use_syslog }}
- clog to syslog = {{ use_syslog }}
+log to syslog = {{ use_syslog }}
+err to syslog = {{ use_syslog }}
+clog to syslog = {{ use_syslog }}
 

=== modified file 'hooks/charmhelpers/contrib/openstack/templating.py'
--- hooks/charmhelpers/contrib/openstack/templating.py	2015-06-02 12:12:49 +0000
+++ hooks/charmhelpers/contrib/openstack/templating.py	2015-08-27 15:02:41 +0000
@@ -14,9 +14,7 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
-import glob
 import os
-import string
 
 import six
 
@@ -27,13 +25,12 @@
     INFO
 )
 from charmhelpers.contrib.openstack.utils import OPENSTACK_CODENAMES
-from charmhelpers.contrib.openstack.context import OSPatternContextGenerator
 
 try:
     from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
 except ImportError:
-    # python-jinja2 may not be installed yet, or we're running unittests.
-    FileSystemLoader = ChoiceLoader = Environment = exceptions = None
+    apt_install('python-jinja2', fatal=True)
+    from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
 
 
 class OSConfigException(Exception):
@@ -99,7 +96,7 @@
         else:
             self.contexts = contexts
 
-        self._complete_contexts = set()
+        self._complete_contexts = []
 
     def context(self):
         ctxt = {}
@@ -108,7 +105,9 @@
             if _ctxt:
                 ctxt.update(_ctxt)
                 # track interfaces for every complete context.
-                self._complete_contexts.update(context.interfaces)
+                [self._complete_contexts.append(interface)
+                 for interface in context.interfaces
+                 if interface not in self._complete_contexts]
         return ctxt
 
     def complete_contexts(self):
@@ -121,41 +120,6 @@
         return self._complete_contexts
 
 
-class OSPatternConfigTemplate(OSConfigTemplate):
-    """
-    Associates a config pattern template with a list of context generators.
-    Responsible for constructing a template context based on those generators.
-    """
-    def __init__(self, pattern, contexts):
-        self.pattern = pattern
-        super(OSPatternConfigTemplate, self).__init__(config_file=None,
-                                                      contexts=contexts)
-
-    def context(self):
-        base_ctxt = {}
-        ctxt = {}
-        for context in self.contexts:
-            _ctxt = context()
-            if not _ctxt:
-                continue
-            elif isinstance(context, OSPatternContextGenerator):
-                # for each returned key initialize its context with base_ctxt
-                # if not defined before and update with new data
-                for k, v in _ctxt.items():
-                    if k not in ctxt:
-                        ctxt[k] = base_ctxt.copy()
-                    ctxt[k].update(v)
-            else:
-                # update the base context and all pre-existing file-specific
-                # contexts
-                base_ctxt.update(_ctxt)
-                for key in ctxt:
-                    ctxt[key].update(_ctxt)
-            # track interfaces for every complete context.
-            self._complete_contexts.update(context.interfaces)
-        return ctxt
-
-
 class OSConfigRenderer(object):
     """
     This class provides a common templating system to be used by OpenStack
@@ -168,13 +132,6 @@
         # import some common context generates from charmhelpers
         from charmhelpers.contrib.openstack import context
 
-        # or create your own
-        class SimpleContextGenerator(OSContextGenerator):
-            def __call__():
-                return {
-                    'key': 'value'
-                }
-
         # Create a renderer object for a specific OS release.
         configs = OSConfigRenderer(templates_dir='/tmp/templates',
                                    openstack_release='folsom')
@@ -185,31 +142,12 @@
         configs.register(config_file='/etc/nova/api-paste.ini',
                          contexts=[context.IdentityServiceContext()])
         configs.register(config_file='/etc/haproxy/haproxy.conf',
-                         contexts=[context.HAProxyContext(),
-                                   SimpleContextGenerator()])
+                         contexts=[context.HAProxyContext()])
         # write out a single config
         configs.write('/etc/nova/nova.conf')
         # write out all registered configs
         configs.write_all()
 
-    Using patterns::
-        class DashboardContextGenerator(OSPatternContextGenerator):
-            def __call__():
-                return {
-                    (40, 'router'): { 'DISABLED': True }
-                }
-
-        configs.register_pattern(
-            pattern='/usr/share/openstack-dashboard/openstack_dashboard'
-                    '/enabled/_{}_juju_{}.py',
-            contexts=[
-                DashboardContextGenerator()
-            ])
-        # delete all files matching the pattern and
-        # write _40_juju_router.py anew
-        configs.write('/usr/share/openstack-dashboard/openstack_dashboard'
-                      '/enabled/_{}_juju_{}.py')
-
     **OpenStack Releases and template loading**
 
     When the object is instantiated, it is associated with a specific OS
@@ -281,15 +219,6 @@
                                                        contexts=contexts)
         log('Registered config file: %s' % config_file, level=INFO)
 
-    def register_pattern(self, pattern, contexts):
-        """
-        Register a config file name pattern with a list of context generators
-        to be called during rendering. Use standard format() specification.
-        """
-        self.templates[pattern] = OSPatternConfigTemplate(pattern=pattern,
-                                                          contexts=contexts)
-        log('Registered config pattern: %s' % pattern, level=INFO)
-
     def _get_tmpl_env(self):
         if not self._tmpl_env:
             loader = get_loader(self.templates_dir, self.openstack_release)
@@ -305,8 +234,7 @@
         if config_file not in self.templates:
             log('Config not registered: %s' % config_file, level=ERROR)
             raise OSConfigException
-        ostemplate = self.templates[config_file]
-        ctxt = ostemplate.context()
+        ctxt = self.templates[config_file].context()
 
         _tmpl = os.path.basename(config_file)
         try:
@@ -325,14 +253,7 @@
                 raise e
 
         log('Rendering from template: %s' % _tmpl, level=INFO)
-
-        if not isinstance(ostemplate, OSPatternConfigTemplate):
-            ctxt = {(): ctxt}
-
-        renders = {}
-        for args, file_ctxt in ctxt.items():
-            renders[args] = template.render(file_ctxt)
-        return renders
+        return template.render(ctxt)
 
     def write(self, config_file):
         """
@@ -342,16 +263,10 @@
             log('Config not registered: %s' % config_file, level=ERROR)
             raise OSConfigException
 
-        renders = self.render(config_file)
-
-        files = glob.glob(''.join([t[0] + ('' if t[1] is None else '*')
-                          for t in string.Formatter().parse(config_file)]))
-        for name in files:
-            os.unlink(name)
-
-        for args, render in renders.items():
-            with open(config_file.format(*args), 'wb') as out:
-                out.write(render)
+        _out = self.render(config_file)
+
+        with open(config_file, 'wb') as out:
+            out.write(_out)
 
         log('Wrote template %s.' % config_file, level=INFO)
 

=== modified file 'hooks/charmhelpers/contrib/openstack/utils.py'
--- hooks/charmhelpers/contrib/openstack/utils.py	2015-04-16 20:24:28 +0000
+++ hooks/charmhelpers/contrib/openstack/utils.py	2015-08-27 15:02:41 +0000
@@ -24,6 +24,7 @@
 import json
 import os
 import sys
+import re
 
 import six
 import yaml
@@ -53,9 +54,13 @@
     get_ipv6_addr
 )
 
+from charmhelpers.contrib.python.packages import (
+    pip_create_virtualenv,
+    pip_install,
+)
+
 from charmhelpers.core.host import lsb_release, mounts, umount
 from charmhelpers.fetch import apt_install, apt_cache, install_remote
-from charmhelpers.contrib.python.packages import pip_install
 from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
 from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
 
@@ -65,7 +70,6 @@
 DISTRO_PROPOSED = ('deb http://archive.ubuntu.com/ubuntu/ %s-proposed '
                    'restricted main multiverse universe')
 
-
 UBUNTU_OPENSTACK_RELEASE = OrderedDict([
     ('oneiric', 'diablo'),
     ('precise', 'essex'),
@@ -75,6 +79,7 @@
     ('trusty', 'icehouse'),
     ('utopic', 'juno'),
     ('vivid', 'kilo'),
+    ('wily', 'liberty'),
 ])
 
 
@@ -87,6 +92,7 @@
     ('2014.1', 'icehouse'),
     ('2014.2', 'juno'),
     ('2015.1', 'kilo'),
+    ('2015.2', 'liberty'),
 ])
 
 # The ugly duckling
@@ -109,8 +115,37 @@
     ('2.2.0', 'juno'),
     ('2.2.1', 'kilo'),
     ('2.2.2', 'kilo'),
+    ('2.3.0', 'liberty'),
 ])
 
+# >= Liberty version->codename mapping
+PACKAGE_CODENAMES = {
+    'nova-common': OrderedDict([
+        ('12.0.0', 'liberty'),
+    ]),
+    'neutron-common': OrderedDict([
+        ('7.0.0', 'liberty'),
+    ]),
+    'cinder-common': OrderedDict([
+        ('7.0.0', 'liberty'),
+    ]),
+    'keystone': OrderedDict([
+        ('8.0.0', 'liberty'),
+    ]),
+    'horizon-common': OrderedDict([
+        ('8.0.0', 'liberty'),
+    ]),
+    'ceilometer-common': OrderedDict([
+        ('5.0.0', 'liberty'),
+    ]),
+    'heat-common': OrderedDict([
+        ('5.0.0', 'liberty'),
+    ]),
+    'glance-common': OrderedDict([
+        ('11.0.0', 'liberty'),
+    ]),
+}
+
 DEFAULT_LOOPBACK_SIZE = '5G'
 
 
@@ -194,20 +229,29 @@
         error_out(e)
 
     vers = apt.upstream_version(pkg.current_ver.ver_str)
+    match = re.match('^(\d)\.(\d)\.(\d)', vers)
+    if match:
+        vers = match.group(0)
 
-    try:
-        if 'swift' in pkg.name:
-            swift_vers = vers[:5]
-            if swift_vers not in SWIFT_CODENAMES:
-                # Deal with 1.10.0 upward
-                swift_vers = vers[:6]
-            return SWIFT_CODENAMES[swift_vers]
-        else:
-            vers = vers[:6]
-            return OPENSTACK_CODENAMES[vers]
-    except KeyError:
-        e = 'Could not determine OpenStack codename for version %s' % vers
-        error_out(e)
+    # >= Liberty independent project versions
+    if (package in PACKAGE_CODENAMES and
+            vers in PACKAGE_CODENAMES[package]):
+        return PACKAGE_CODENAMES[package][vers]
+    else:
+        # < Liberty co-ordinated project versions
+        try:
+            if 'swift' in pkg.name:
+                swift_vers = vers[:5]
+                if swift_vers not in SWIFT_CODENAMES:
+                    # Deal with 1.10.0 upward
+                    swift_vers = vers[:6]
+                return SWIFT_CODENAMES[swift_vers]
+            else:
+                vers = vers[:6]
+                return OPENSTACK_CODENAMES[vers]
+        except KeyError:
+            e = 'Could not determine OpenStack codename for version %s' % vers
+            error_out(e)
 
 
 def get_os_version_package(pkg, fatal=True):
@@ -317,6 +361,9 @@
             'kilo': 'trusty-updates/kilo',
             'kilo/updates': 'trusty-updates/kilo',
             'kilo/proposed': 'trusty-proposed/kilo',
+            'liberty': 'trusty-updates/liberty',
+            'liberty/updates': 'trusty-updates/liberty',
+            'liberty/proposed': 'trusty-proposed/liberty',
         }
 
         try:
@@ -497,11 +544,22 @@
 requirements_dir = None
 
 
-def git_clone_and_install(projects_yaml, core_project):
+def _git_yaml_load(projects_yaml):
+    """
+    Load the specified yaml into a dictionary.
+    """
+    if not projects_yaml:
+        return None
+
+    return yaml.load(projects_yaml)
+
+
+def git_clone_and_install(projects_yaml, core_project, depth=1):
     """
     Clone/install all specified OpenStack repositories.
 
     The expected format of projects_yaml is:
+
         repositories:
           - {name: keystone,
              repository: 'git://git.openstack.org/openstack/keystone.git',
@@ -509,24 +567,25 @@
           - {name: requirements,
              repository: 'git://git.openstack.org/openstack/requirements.git',
              branch: 'stable/icehouse'}
+
         directory: /mnt/openstack-git
-        http_proxy: http://squid.internal:3128
-        https_proxy: https://squid.internal:3128
-
-        The directory, http_proxy, and https_proxy keys are optional.
+        http_proxy: squid-proxy-url
+        https_proxy: squid-proxy-url
+
+    The directory, http_proxy, and https_proxy keys are optional.
+
     """
     global requirements_dir
     parent_dir = '/mnt/openstack-git'
-
-    if not projects_yaml:
-        return
-
-    projects = yaml.load(projects_yaml)
+    http_proxy = None
+
+    projects = _git_yaml_load(projects_yaml)
     _git_validate_projects_yaml(projects, core_project)
 
     old_environ = dict(os.environ)
 
     if 'http_proxy' in projects.keys():
+        http_proxy = projects['http_proxy']
         os.environ['http_proxy'] = projects['http_proxy']
     if 'https_proxy' in projects.keys():
         os.environ['https_proxy'] = projects['https_proxy']
@@ -534,15 +593,25 @@
     if 'directory' in projects.keys():
         parent_dir = projects['directory']
 
+    pip_create_virtualenv(os.path.join(parent_dir, 'venv'))
+
+    # Upgrade setuptools and pip from default virtualenv versions. The default
+    # versions in trusty break master OpenStack branch deployments.
+    for p in ['pip', 'setuptools']:
+        pip_install(p, upgrade=True, proxy=http_proxy,
+                    venv=os.path.join(parent_dir, 'venv'))
+
     for p in projects['repositories']:
         repo = p['repository']
         branch = p['branch']
         if p['name'] == 'requirements':
-            repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
+            repo_dir = _git_clone_and_install_single(repo, branch, depth,
+                                                     parent_dir, http_proxy,
                                                      update_requirements=False)
             requirements_dir = repo_dir
         else:
-            repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
+            repo_dir = _git_clone_and_install_single(repo, branch, depth,
+                                                     parent_dir, http_proxy,
                                                      update_requirements=True)
 
     os.environ = old_environ
@@ -574,7 +643,8 @@
         error_out('openstack-origin-git key \'{}\' is missing'.format(key))
 
 
-def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements):
+def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy,
+                                  update_requirements):
     """
     Clone and install a single git repository.
     """
@@ -587,23 +657,29 @@
 
     if not os.path.exists(dest_dir):
         juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
-        repo_dir = install_remote(repo, dest=parent_dir, branch=branch)
+        repo_dir = install_remote(repo, dest=parent_dir, branch=branch,
+                                  depth=depth)
     else:
         repo_dir = dest_dir
 
+    venv = os.path.join(parent_dir, 'venv')
+
     if update_requirements:
         if not requirements_dir:
             error_out('requirements repo must be cloned before '
                       'updating from global requirements.')
-        _git_update_requirements(repo_dir, requirements_dir)
+        _git_update_requirements(venv, repo_dir, requirements_dir)
 
     juju_log('Installing git repo from dir: {}'.format(repo_dir))
-    pip_install(repo_dir)
+    if http_proxy:
+        pip_install(repo_dir, proxy=http_proxy, venv=venv)
+    else:
+        pip_install(repo_dir, venv=venv)
 
     return repo_dir
 
 
-def _git_update_requirements(package_dir, reqs_dir):
+def _git_update_requirements(venv, package_dir, reqs_dir):
     """
     Update from global requirements.
 
@@ -612,25 +688,38 @@
     """
     orig_dir = os.getcwd()
     os.chdir(reqs_dir)
-    cmd = ['python', 'update.py', package_dir]
+    python = os.path.join(venv, 'bin/python')
+    cmd = [python, 'update.py', package_dir]
     try:
         subprocess.check_call(cmd)
     except subprocess.CalledProcessError:
         package = os.path.basename(package_dir)
-        error_out("Error updating {} from global-requirements.txt".format(package))
+        error_out("Error updating {} from "
+                  "global-requirements.txt".format(package))
     os.chdir(orig_dir)
 
 
+def git_pip_venv_dir(projects_yaml):
+    """
+    Return the pip virtualenv path.
+    """
+    parent_dir = '/mnt/openstack-git'
+
+    projects = _git_yaml_load(projects_yaml)
+
+    if 'directory' in projects.keys():
+        parent_dir = projects['directory']
+
+    return os.path.join(parent_dir, 'venv')
+
+
 def git_src_dir(projects_yaml, project):
     """
     Return the directory where the specified project's source is located.
     """
     parent_dir = '/mnt/openstack-git'
 
-    if not projects_yaml:
-        return
-
-    projects = yaml.load(projects_yaml)
+    projects = _git_yaml_load(projects_yaml)
 
     if 'directory' in projects.keys():
         parent_dir = projects['directory']
@@ -640,3 +729,15 @@
             return os.path.join(parent_dir, os.path.basename(p['repository']))
 
     return None
+
+
+def git_yaml_value(projects_yaml, key):
+    """
+    Return the value in projects_yaml for the specified key.
+    """
+    projects = _git_yaml_load(projects_yaml)
+
+    if key in projects.keys():
+        return projects[key]
+
+    return None

=== modified file 'hooks/charmhelpers/contrib/python/packages.py'
--- hooks/charmhelpers/contrib/python/packages.py	2015-02-26 10:11:26 +0000
+++ hooks/charmhelpers/contrib/python/packages.py	2015-08-27 15:02:41 +0000
@@ -17,8 +17,11 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
+import subprocess
+
 from charmhelpers.fetch import apt_install, apt_update
-from charmhelpers.core.hookenv import log
+from charmhelpers.core.hookenv import charm_dir, log
 
 try:
     from pip import main as pip_execute
@@ -33,6 +36,8 @@
 def parse_options(given, available):
     """Given a set of options, check if available"""
     for key, value in sorted(given.items()):
+        if not value:
+            continue
         if key in available:
             yield "--{0}={1}".format(key, value)
 
@@ -51,11 +56,15 @@
     pip_execute(command)
 
 
-def pip_install(package, fatal=False, upgrade=False, **options):
+def pip_install(package, fatal=False, upgrade=False, venv=None, **options):
     """Install a python package"""
-    command = ["install"]
+    if venv:
+        venv_python = os.path.join(venv, 'bin/pip')
+        command = [venv_python, "install"]
+    else:
+        command = ["install"]
 
-    available_options = ('proxy', 'src', 'log', "index-url", )
+    available_options = ('proxy', 'src', 'log', 'index-url', )
     for option in parse_options(options, available_options):
         command.append(option)
 
@@ -69,7 +78,10 @@
 
     log("Installing {} package with options: {}".format(package,
                                                         command))
-    pip_execute(command)
+    if venv:
+        subprocess.check_call(command)
+    else:
+        pip_execute(command)
 
 
 def pip_uninstall(package, **options):
@@ -94,3 +106,16 @@
     """Returns the list of current python installed packages
     """
     return pip_execute(["list"])
+
+
+def pip_create_virtualenv(path=None):
+    """Create an isolated Python environment."""
+    apt_install('python-virtualenv')
+
+    if path:
+        venv_path = path
+    else:
+        venv_path = os.path.join(charm_dir(), 'venv')
+
+    if not os.path.exists(venv_path):
+        subprocess.check_call(['virtualenv', venv_path])

=== modified file 'hooks/charmhelpers/contrib/storage/linux/ceph.py'
--- hooks/charmhelpers/contrib/storage/linux/ceph.py	2015-01-26 09:46:38 +0000
+++ hooks/charmhelpers/contrib/storage/linux/ceph.py	2015-08-27 15:02:41 +0000
@@ -60,12 +60,12 @@
 KEYFILE = '/etc/ceph/ceph.client.{}.key'
 
 CEPH_CONF = """[global]
- auth supported = {auth}
- keyring = {keyring}
- mon host = {mon_hosts}
- log to syslog = {use_syslog}
- err to syslog = {use_syslog}
- clog to syslog = {use_syslog}
+auth supported = {auth}
+keyring = {keyring}
+mon host = {mon_hosts}
+log to syslog = {use_syslog}
+err to syslog = {use_syslog}
+clog to syslog = {use_syslog}
 """
 
 

=== modified file 'hooks/charmhelpers/contrib/storage/linux/utils.py'
--- hooks/charmhelpers/contrib/storage/linux/utils.py	2015-01-26 09:46:38 +0000
+++ hooks/charmhelpers/contrib/storage/linux/utils.py	2015-08-27 15:02:41 +0000
@@ -43,9 +43,10 @@
 
     :param block_device: str: Full path of block device to clean.
     '''
+    # https://github.com/ceph/ceph/commit/fdd7f8d83afa25c4e09aaedd90ab93f3b64a677b
     # sometimes sgdisk exits non-zero; this is OK, dd will clean up
-    call(['sgdisk', '--zap-all', '--mbrtogpt',
-          '--clear', block_device])
+    call(['sgdisk', '--zap-all', '--', block_device])
+    call(['sgdisk', '--clear', '--mbrtogpt', '--', block_device])
     dev_end = check_output(['blockdev', '--getsz',
                             block_device]).decode('UTF-8')
     gpt_end = int(dev_end.split()[0]) - 100
@@ -67,4 +68,4 @@
     out = check_output(['mount']).decode('UTF-8')
     if is_partition:
         return bool(re.search(device + r"\b", out))
-    return bool(re.search(device + r"[0-9]+\b", out))
+    return bool(re.search(device + r"[0-9]*\b", out))

=== added file 'hooks/charmhelpers/core/files.py'
--- hooks/charmhelpers/core/files.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/files.py	2015-08-27 15:02:41 +0000
@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = 'Jorge Niedbalski <[e-mail]>'
+
+import os
+import subprocess
+
+
+def sed(filename, before, after, flags='g'):
+    """
+    Search and replaces the given pattern on filename.
+
+    :param filename: relative or absolute file path.
+    :param before: expression to be replaced (see 'man sed')
+    :param after: expression to replace with (see 'man sed')
+    :param flags: sed-compatible regex flags in example, to make
+    the  search and replace case insensitive, specify ``flags="i"``.
+    The ``g`` flag is always specified regardless, so you do not
+    need to remember to include it when overriding this parameter.
+    :returns: If the sed command exit code was zero then return,
+    otherwise raise CalledProcessError.
+    """
+    expression = r's/{0}/{1}/{2}'.format(before,
+                                         after, flags)
+
+    return subprocess.check_call(["sed", "-i", "-r", "-e",
+                                  expression,
+                                  os.path.expanduser(filename)])

=== modified file 'hooks/charmhelpers/core/hookenv.py'
--- hooks/charmhelpers/core/hookenv.py	2015-04-16 20:24:28 +0000
+++ hooks/charmhelpers/core/hookenv.py	2015-08-27 15:02:41 +0000
@@ -21,12 +21,17 @@
 #  Charm Helpers Developers <[e-mail]>
 
 from __future__ import print_function
+import copy
+from distutils.version import LooseVersion
+from functools import wraps
+import glob
 import os
 import json
 import yaml
 import subprocess
 import sys
 import errno
+import tempfile
 from subprocess import CalledProcessError
 
 import six
@@ -58,15 +63,18 @@
 
     will cache the result of unit_get + 'test' for future calls.
     """
+    @wraps(func)
     def wrapper(*args, **kwargs):
         global cache
         key = str((func, args, kwargs))
         try:
             return cache[key]
         except KeyError:
-            res = func(*args, **kwargs)
-            cache[key] = res
-            return res
+            pass  # Drop out of the exception handler scope.
+        res = func(*args, **kwargs)
+        cache[key] = res
+        return res
+    wrapper._wrapped = func
     return wrapper
 
 
@@ -166,9 +174,19 @@
     return os.environ.get('JUJU_RELATION', None)
 
 
-def relation_id():
-    """The relation ID for the current relation hook"""
-    return os.environ.get('JUJU_RELATION_ID', None)
+@cached
+def relation_id(relation_name=None, service_or_unit=None):
+    """The relation ID for the current or a specified relation"""
+    if not relation_name and not service_or_unit:
+        return os.environ.get('JUJU_RELATION_ID', None)
+    elif relation_name and service_or_unit:
+        service_name = service_or_unit.split('/')[0]
+        for relid in relation_ids(relation_name):
+            remote_service = remote_service_name(relid)
+            if remote_service == service_name:
+                return relid
+    else:
+        raise ValueError('Must specify neither or both of relation_name and service_or_unit')
 
 
 def local_unit():
@@ -178,7 +196,7 @@
 
 def remote_unit():
     """The remote unit for the current relation hook"""
-    return os.environ['JUJU_REMOTE_UNIT']
+    return os.environ.get('JUJU_REMOTE_UNIT', None)
 
 
 def service_name():
@@ -186,9 +204,20 @@
     return local_unit().split('/')[0]
 
 
+@cached
+def remote_service_name(relid=None):
+    """The remote service name for a given relation-id (or the current relation)"""
+    if relid is None:
+        unit = remote_unit()
+    else:
+        units = related_units(relid)
+        unit = units[0] if units else None
+    return unit.split('/')[0] if unit else None
+
+
 def hook_name():
     """The name of the currently executing hook"""
-    return os.path.basename(sys.argv[0])
+    return os.environ.get('JUJU_HOOK_NAME', os.path.basename(sys.argv[0]))
 
 
 class Config(dict):
@@ -238,23 +267,7 @@
         self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
         if os.path.exists(self.path):
             self.load_previous()
-
-    def __getitem__(self, key):
-        """For regular dict lookups, check the current juju config first,
-        then the previous (saved) copy. This ensures that user-saved values
-        will be returned by a dict lookup.
-
-        """
-        try:
-            return dict.__getitem__(self, key)
-        except KeyError:
-            return (self._prev_dict or {})[key]
-
-    def keys(self):
-        prev_keys = []
-        if self._prev_dict is not None:
-            prev_keys = self._prev_dict.keys()
-        return list(set(prev_keys + list(dict.keys(self))))
+        atexit(self._implicit_save)
 
     def load_previous(self, path=None):
         """Load previous copy of config from disk.
@@ -273,6 +286,9 @@
         self.path = path or self.path
         with open(self.path) as f:
             self._prev_dict = json.load(f)
+        for k, v in copy.deepcopy(self._prev_dict).items():
+            if k not in self:
+                self[k] = v
 
     def changed(self, key):
         """Return True if the current value for this key is different from
@@ -304,13 +320,13 @@
         instance.
 
         """
-        if self._prev_dict:
-            for k, v in six.iteritems(self._prev_dict):
-                if k not in self:
-                    self[k] = v
         with open(self.path, 'w') as f:
             json.dump(self, f)
 
+    def _implicit_save(self):
+        if self.implicit_save:
+            self.save()
+
 
 @cached
 def config(scope=None):
@@ -353,18 +369,49 @@
     """Set relation information for the current unit"""
     relation_settings = relation_settings if relation_settings else {}
     relation_cmd_line = ['relation-set']
+    accepts_file = "--file" in subprocess.check_output(
+        relation_cmd_line + ["--help"], universal_newlines=True)
     if relation_id is not None:
         relation_cmd_line.extend(('-r', relation_id))
-    for k, v in (list(relation_settings.items()) + list(kwargs.items())):
-        if v is None:
-            relation_cmd_line.append('{}='.format(k))
-        else:
-            relation_cmd_line.append('{}={}'.format(k, v))
-    subprocess.check_call(relation_cmd_line)
+    settings = relation_settings.copy()
+    settings.update(kwargs)
+    for key, value in settings.items():
+        # Force value to be a string: it always should, but some call
+        # sites pass in things like dicts or numbers.
+        if value is not None:
+            settings[key] = "{}".format(value)
+    if accepts_file:
+        # --file was introduced in Juju 1.23.2. Use it by default if
+        # available, since otherwise we'll break if the relation data is
+        # too big. Ideally we should tell relation-set to read the data from
+        # stdin, but that feature is broken in 1.23.2: Bug #1454678.
+        with tempfile.NamedTemporaryFile(delete=False) as settings_file:
+            settings_file.write(yaml.safe_dump(settings).encode("utf-8"))
+        subprocess.check_call(
+            relation_cmd_line + ["--file", settings_file.name])
+        os.remove(settings_file.name)
+    else:
+        for key, value in settings.items():
+            if value is None:
+                relation_cmd_line.append('{}='.format(key))
+            else:
+                relation_cmd_line.append('{}={}'.format(key, value))
+        subprocess.check_call(relation_cmd_line)
     # Flush cache of any relation-gets for local unit
     flush(local_unit())
 
 
+def relation_clear(r_id=None):
+    ''' Clears any relation data already set on relation r_id '''
+    settings = relation_get(rid=r_id,
+                            unit=local_unit())
+    for setting in settings:
+        if setting not in ['public-address', 'private-address']:
+            settings[setting] = None
+    relation_set(relation_id=r_id,
+                 **settings)
+
+
 @cached
 def relation_ids(reltype=None):
     """A list of relation_ids"""
@@ -444,6 +491,63 @@
 
 
 @cached
+def relation_to_interface(relation_name):
+    """
+    Given the name of a relation, return the interface that relation uses.
+
+    :returns: The interface name, or ``None``.
+    """
+    return relation_to_role_and_interface(relation_name)[1]
+
+
+@cached
+def relation_to_role_and_interface(relation_name):
+    """
+    Given the name of a relation, return the role and the name of the interface
+    that relation uses (where role is one of ``provides``, ``requires``, or ``peer``).
+
+    :returns: A tuple containing ``(role, interface)``, or ``(None, None)``.
+    """
+    _metadata = metadata()
+    for role in ('provides', 'requires', 'peer'):
+        interface = _metadata.get(role, {}).get(relation_name, {}).get('interface')
+        if interface:
+            return role, interface
+    return None, None
+
+
+@cached
+def role_and_interface_to_relations(role, interface_name):
+    """
+    Given a role and interface name, return a list of relation names for the
+    current charm that use that interface under that role (where role is one
+    of ``provides``, ``requires``, or ``peer``).
+
+    :returns: A list of relation names.
+    """
+    _metadata = metadata()
+    results = []
+    for relation_name, relation in _metadata.get(role, {}).items():
+        if relation['interface'] == interface_name:
+            results.append(relation_name)
+    return results
+
+
+@cached
+def interface_to_relations(interface_name):
+    """
+    Given an interface, return a list of relation names for the current
+    charm that use that interface.
+
+    :returns: A list of relation names.
+    """
+    results = []
+    for role in ('provides', 'requires', 'peer'):
+        results.extend(role_and_interface_to_relations(role, interface_name))
+    return results
+
+
+@cached
 def charm_name():
     """Get the name of the current charm as is specified on metadata.yaml"""
     return metadata().get('name')
@@ -509,6 +613,11 @@
         return None
 
 
+def unit_public_ip():
+    """Get this unit's public IP address"""
+    return unit_get('public-address')
+
+
 def unit_private_ip():
     """Get this unit's private IP address"""
     return unit_get('private-address')
@@ -541,10 +650,14 @@
             hooks.execute(sys.argv)
     """
 
-    def __init__(self, config_save=True):
+    def __init__(self, config_save=None):
         super(Hooks, self).__init__()
         self._hooks = {}
-        self._config_save = config_save
+
+        # For unknown reasons, we allow the Hooks constructor to override
+        # config().implicit_save.
+        if config_save is not None:
+            config().implicit_save = config_save
 
     def register(self, name, function):
         """Register a hook"""
@@ -552,13 +665,16 @@
 
     def execute(self, args):
         """Execute a registered hook based on args[0]"""
+        _run_atstart()
         hook_name = os.path.basename(args[0])
         if hook_name in self._hooks:
-            self._hooks[hook_name]()
-            if self._config_save:
-                cfg = config()
-                if cfg.implicit_save:
-                    cfg.save()
+            try:
+                self._hooks[hook_name]()
+            except SystemExit as x:
+                if x.code is None or x.code == 0:
+                    _run_atexit()
+                raise
+            _run_atexit()
         else:
             raise UnregisteredHookError(hook_name)
 
@@ -605,3 +721,176 @@
 
     The results set by action_set are preserved."""
     subprocess.check_call(['action-fail', message])
+
+
+def action_name():
+    """Get the name of the currently executing action."""
+    return os.environ.get('JUJU_ACTION_NAME')
+
+
+def action_uuid():
+    """Get the UUID of the currently executing action."""
+    return os.environ.get('JUJU_ACTION_UUID')
+
+
+def action_tag():
+    """Get the tag for the currently executing action."""
+    return os.environ.get('JUJU_ACTION_TAG')
+
+
+def status_set(workload_state, message):
+    """Set the workload state with a message
+
+    Use status-set to set the workload state with a message which is visible
+    to the user via juju status. If the status-set command is not found then
+    assume this is juju < 1.23 and juju-log the message unstead.
+
+    workload_state -- valid juju workload state.
+    message        -- status update message
+    """
+    valid_states = ['maintenance', 'blocked', 'waiting', 'active']
+    if workload_state not in valid_states:
+        raise ValueError(
+            '{!r} is not a valid workload state'.format(workload_state)
+        )
+    cmd = ['status-set', workload_state, message]
+    try:
+        ret = subprocess.call(cmd)
+        if ret == 0:
+            return
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
+    log_message = 'status-set failed: {} {}'.format(workload_state,
+                                                    message)
+    log(log_message, level='INFO')
+
+
+def status_get():
+    """Retrieve the previously set juju workload state
+
+    If the status-set command is not found then assume this is juju < 1.23 and
+    return 'unknown'
+    """
+    cmd = ['status-get']
+    try:
+        raw_status = subprocess.check_output(cmd, universal_newlines=True)
+        status = raw_status.rstrip()
+        return status
+    except OSError as e:
+        if e.errno == errno.ENOENT:
+            return 'unknown'
+        else:
+            raise
+
+
+def translate_exc(from_exc, to_exc):
+    def inner_translate_exc1(f):
+        def inner_translate_exc2(*args, **kwargs):
+            try:
+                return f(*args, **kwargs)
+            except from_exc:
+                raise to_exc
+
+        return inner_translate_exc2
+
+    return inner_translate_exc1
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def is_leader():
+    """Does the current unit hold the juju leadership
+
+    Uses juju to determine whether the current unit is the leader of its peers
+    """
+    cmd = ['is-leader', '--format=json']
+    return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def leader_get(attribute=None):
+    """Juju leader get value(s)"""
+    cmd = ['leader-get', '--format=json'] + [attribute or '-']
+    return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def leader_set(settings=None, **kwargs):
+    """Juju leader set value(s)"""
+    # Don't log secrets.
+    # log("Juju leader-set '%s'" % (settings), level=DEBUG)
+    cmd = ['leader-set']
+    settings = settings or {}
+    settings.update(kwargs)
+    for k, v in settings.items():
+        if v is None:
+            cmd.append('{}='.format(k))
+        else:
+            cmd.append('{}={}'.format(k, v))
+    subprocess.check_call(cmd)
+
+
+@cached
+def juju_version():
+    """Full version string (eg. '1.23.3.1-trusty-amd64')"""
+    # Per https://bugs.launchpad.net/juju-core/+bug/1455368/comments/1
+    jujud = glob.glob('/var/lib/juju/tools/machine-*/jujud')[0]
+    return subprocess.check_output([jujud, 'version'],
+                                   universal_newlines=True).strip()
+
+
+@cached
+def has_juju_version(minimum_version):
+    """Return True if the Juju version is at least the provided version"""
+    return LooseVersion(juju_version()) >= LooseVersion(minimum_version)
+
+
+_atexit = []
+_atstart = []
+
+
+def atstart(callback, *args, **kwargs):
+    '''Schedule a callback to run before the main hook.
+
+    Callbacks are run in the order they were added.
+
+    This is useful for modules and classes to perform initialization
+    and inject behavior. In particular:
+
+        - Run common code before all of your hooks, such as logging
+          the hook name or interesting relation data.
+        - Defer object or module initialization that requires a hook
+          context until we know there actually is a hook context,
+          making testing easier.
+        - Rather than requiring charm authors to include boilerplate to
+          invoke your helper's behavior, have it run automatically if
+          your object is instantiated or module imported.
+
+    This is not at all useful after your hook framework as been launched.
+    '''
+    global _atstart
+    _atstart.append((callback, args, kwargs))
+
+
+def atexit(callback, *args, **kwargs):
+    '''Schedule a callback to run on successful hook completion.
+
+    Callbacks are run in the reverse order that they were added.'''
+    _atexit.append((callback, args, kwargs))
+
+
+def _run_atstart():
+    '''Hook frameworks must invoke this before running the main hook body.'''
+    global _atstart
+    for callback, args, kwargs in _atstart:
+        callback(*args, **kwargs)
+    del _atstart[:]
+
+
+def _run_atexit():
+    '''Hook frameworks must invoke this after the main hook body has
+    successfully completed. Do not invoke it if the hook fails.'''
+    global _atexit
+    for callback, args, kwargs in reversed(_atexit):
+        callback(*args, **kwargs)
+    del _atexit[:]

=== modified file 'hooks/charmhelpers/core/host.py'
--- hooks/charmhelpers/core/host.py	2015-06-02 12:12:49 +0000
+++ hooks/charmhelpers/core/host.py	2015-08-27 15:02:41 +0000
@@ -63,6 +63,36 @@
     return service_result
 
 
+def service_pause(service_name, init_dir=None):
+    """Pause a system service.
+
+    Stop it, and prevent it from starting again at boot."""
+    if init_dir is None:
+        init_dir = "/etc/init"
+    stopped = service_stop(service_name)
+    # XXX: Support systemd too
+    override_path = os.path.join(
+        init_dir, '{}.override'.format(service_name))
+    with open(override_path, 'w') as fh:
+        fh.write("manual\n")
+    return stopped
+
+
+def service_resume(service_name, init_dir=None):
+    """Resume a system service.
+
+    Reenable starting again at boot. Start the service"""
+    # XXX: Support systemd too
+    if init_dir is None:
+        init_dir = "/etc/init"
+    override_path = os.path.join(
+        init_dir, '{}.override'.format(service_name))
+    if os.path.exists(override_path):
+        os.unlink(override_path)
+    started = service_start(service_name)
+    return started
+
+
 def service(action, service_name):
     """Control a system service"""
     cmd = ['service', service_name, action]
@@ -91,7 +121,7 @@
             ['service', service_name, 'status'],
             stderr=subprocess.STDOUT).decode('UTF-8')
     except subprocess.CalledProcessError as e:
-        return 'unrecognized service' not in e.output
+        return b'unrecognized service' not in e.output
     else:
         return True
 
@@ -118,6 +148,16 @@
     return user_info
 
 
+def user_exists(username):
+    """Check if a user exists"""
+    try:
+        pwd.getpwnam(username)
+        user_exists = True
+    except KeyError:
+        user_exists = False
+    return user_exists
+
+
 def add_group(group_name, system_group=False):
     """Add a group to the system"""
     try:
@@ -140,11 +180,7 @@
 
 def add_user_to_group(username, group):
     """Add a user to a group"""
-    cmd = [
-        'gpasswd', '-a',
-        username,
-        group
-    ]
+    cmd = ['gpasswd', '-a', username, group]
     log("Adding user {} to group {}".format(username, group))
     subprocess.check_call(cmd)
 
@@ -254,6 +290,17 @@
     return system_mounts
 
 
+def fstab_mount(mountpoint):
+    """Mount filesystem using fstab"""
+    cmd_args = ['mount', mountpoint]
+    try:
+        subprocess.check_output(cmd_args)
+    except subprocess.CalledProcessError as e:
+        log('Error unmounting {}\n{}'.format(mountpoint, e.output))
+        return False
+    return True
+
+
 def file_hash(path, hash_type='md5'):
     """
     Generate a hash checksum of the contents of 'path' or None if not found.
@@ -370,25 +417,80 @@
     return(''.join(random_chars))
 
 
-def list_nics(nic_type):
+def is_phy_iface(interface):
+    """Returns True if interface is not virtual, otherwise False."""
+    if interface:
+        sys_net = '/sys/class/net'
+        if os.path.isdir(sys_net):
+            for iface in glob.glob(os.path.join(sys_net, '*')):
+                if '/virtual/' in os.path.realpath(iface):
+                    continue
+
+                if interface == os.path.basename(iface):
+                    return True
+
+    return False
+
+
+def get_bond_master(interface):
+    """Returns bond master if interface is bond slave otherwise None.
+
+    NOTE: the provided interface is expected to be physical
+    """
+    if interface:
+        iface_path = '/sys/class/net/%s' % (interface)
+        if os.path.exists(iface_path):
+            if '/virtual/' in os.path.realpath(iface_path):
+                return None
+
+            master = os.path.join(iface_path, 'master')
+            if os.path.exists(master):
+                master = os.path.realpath(master)
+                # make sure it is a bond master
+                if os.path.exists(os.path.join(master, 'bonding')):
+                    return os.path.basename(master)
+
+    return None
+
+
+def list_nics(nic_type=None):
     '''Return a list of nics of given type(s)'''
     if isinstance(nic_type, six.string_types):
         int_types = [nic_type]
     else:
         int_types = nic_type
+
     interfaces = []
-    for int_type in int_types:
-        cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
+    if nic_type:
+        for int_type in int_types:
+            cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
+            ip_output = subprocess.check_output(cmd).decode('UTF-8')
+            ip_output = ip_output.split('\n')
+            ip_output = (line for line in ip_output if line)
+            for line in ip_output:
+                if line.split()[1].startswith(int_type):
+                    matched = re.search('.*: (' + int_type +
+                                        r'[0-9]+\.[0-9]+)@.*', line)
+                    if matched:
+                        iface = matched.groups()[0]
+                    else:
+                        iface = line.split()[1].replace(":", "")
+
+                    if iface not in interfaces:
+                        interfaces.append(iface)
+    else:
+        cmd = ['ip', 'a']
         ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
-        ip_output = (line for line in ip_output if line)
+        ip_output = (line.strip() for line in ip_output if line)
+
+        key = re.compile('^[0-9]+:\s+(.+):')
         for line in ip_output:
-            if line.split()[1].startswith(int_type):
-                matched = re.search('.*: (' + int_type + r'[0-9]+\.[0-9]+)@.*', line)
-                if matched:
-                    interface = matched.groups()[0]
-                else:
-                    interface = line.split()[1].replace(":", "")
-                interfaces.append(interface)
+            matched = re.search(key, line)
+            if matched:
+                iface = matched.group(1)
+                iface = iface.partition("@")[0]
+                if iface not in interfaces:
+                    interfaces.append(iface)
 
     return interfaces
 

=== added file 'hooks/charmhelpers/core/hugepage.py'
--- hooks/charmhelpers/core/hugepage.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/hugepage.py	2015-08-27 15:02:41 +0000
@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+import yaml
+from charmhelpers.core import fstab
+from charmhelpers.core import sysctl
+from charmhelpers.core.host import (
+    add_group,
+    add_user_to_group,
+    fstab_mount,
+    mkdir,
+)
+
+
+def hugepage_support(user, group='hugetlb', nr_hugepages=256,
+                     max_map_count=65536, mnt_point='/run/hugepages/kvm',
+                     pagesize='2MB', mount=True):
+    """Enable hugepages on system.
+
+    Args:
+    user (str)  -- Username to allow access to hugepages to
+    group (str) -- Group name to own hugepages
+    nr_hugepages (int) -- Number of pages to reserve
+    max_map_count (int) -- Number of Virtual Memory Areas a process can own
+    mnt_point (str) -- Directory to mount hugepages on
+    pagesize (str) -- Size of hugepages
+    mount (bool) -- Whether to Mount hugepages
+    """
+    group_info = add_group(group)
+    gid = group_info.gr_gid
+    add_user_to_group(user, group)
+    sysctl_settings = {
+        'vm.nr_hugepages': nr_hugepages,
+        'vm.max_map_count': max_map_count,
+        'vm.hugetlb_shm_group': gid,
+    }
+    sysctl.create(yaml.dump(sysctl_settings), '/etc/sysctl.d/10-hugepage.conf')
+    mkdir(mnt_point, owner='root', group='root', perms=0o755, force=False)
+    lfstab = fstab.Fstab()
+    fstab_entry = lfstab.get_entry_by_attr('mountpoint', mnt_point)
+    if fstab_entry:
+        lfstab.remove_entry(fstab_entry)
+    entry = lfstab.Entry('nodev', mnt_point, 'hugetlbfs',
+                         'mode=1770,gid={},pagesize={}'.format(gid, pagesize), 0, 0)
+    lfstab.add_entry(entry)
+    if mount:
+        fstab_mount(mnt_point)

=== modified file 'hooks/charmhelpers/core/services/base.py'
--- hooks/charmhelpers/core/services/base.py	2015-01-26 09:46:38 +0000
+++ hooks/charmhelpers/core/services/base.py	2015-08-27 15:02:41 +0000
@@ -15,9 +15,9 @@
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import re
 import json
-from collections import Iterable
+from inspect import getargspec
+from collections import Iterable, OrderedDict
 
 from charmhelpers.core import host
 from charmhelpers.core import hookenv
@@ -119,7 +119,7 @@
         """
         self._ready_file = os.path.join(hookenv.charm_dir(), 'READY-SERVICES.json')
         self._ready = None
-        self.services = {}
+        self.services = OrderedDict()
         for service in services or []:
             service_name = service['service']
             self.services[service_name] = service
@@ -128,15 +128,18 @@
         """
         Handle the current hook by doing The Right Thing with the registered services.
         """
-        hook_name = hookenv.hook_name()
-        if hook_name == 'stop':
-            self.stop_services()
-        else:
-            self.provide_data()
-            self.reconfigure_services()
-        cfg = hookenv.config()
-        if cfg.implicit_save:
-            cfg.save()
+        hookenv._run_atstart()
+        try:
+            hook_name = hookenv.hook_name()
+            if hook_name == 'stop':
+                self.stop_services()
+            else:
+                self.reconfigure_services()
+                self.provide_data()
+        except SystemExit as x:
+            if x.code is None or x.code == 0:
+                hookenv._run_atexit()
+        hookenv._run_atexit()
 
     def provide_data(self):
         """
@@ -145,15 +148,36 @@
         A provider must have a `name` attribute, which indicates which relation
         to set data on, and a `provide_data()` method, which returns a dict of
         data to set.
+
+        The `provide_data()` method can optionally accept two parameters:
+
+          * ``remote_service`` The name of the remote service that the data will
+            be provided to.  The `provide_data()` method will be called once
+            for each connected service (not unit).  This allows the method to
+            tailor its data to the given service.
+          * ``service_ready`` Whether or not the service definition had all of
+            its requirements met, and thus the ``data_ready`` callbacks run.
+
+        Note that the ``provided_data`` methods are now called **after** the
+        ``data_ready`` callbacks are run.  This gives the ``data_ready`` callbacks
+        a chance to generate any data necessary for the providing to the remote
+        services.
         """
-        hook_name = hookenv.hook_name()
-        for service in self.services.values():
+        for service_name, service in self.services.items():
+            service_ready = self.is_ready(service_name)
             for provider in service.get('provided_data', []):
-                if re.match(r'{}-relation-(joined|changed)'.format(provider.name), hook_name):
-                    data = provider.provide_data()
-                    _ready = provider._is_ready(data) if hasattr(provider, '_is_ready') else data
-                    if _ready:
-                        hookenv.relation_set(None, data)
+                for relid in hookenv.relation_ids(provider.name):
+                    units = hookenv.related_units(relid)
+                    if not units:
+                        continue
+                    remote_service = units[0].split('/')[0]
+                    argspec = getargspec(provider.provide_data)
+                    if len(argspec.args) > 1:
+                        data = provider.provide_data(remote_service, service_ready)
+                    else:
+                        data = provider.provide_data()
+                    if data:
+                        hookenv.relation_set(relid, data)
 
     def reconfigure_services(self, *service_names):
         """

=== modified file 'hooks/charmhelpers/core/services/helpers.py'
--- hooks/charmhelpers/core/services/helpers.py	2015-04-16 20:24:28 +0000
+++ hooks/charmhelpers/core/services/helpers.py	2015-08-27 15:02:41 +0000
@@ -16,7 +16,9 @@
 
 import os
 import yaml
+
 from charmhelpers.core import hookenv
+from charmhelpers.core import host
 from charmhelpers.core import templating
 
 from charmhelpers.core.services.base import ManagerCallback
@@ -239,28 +241,42 @@
     action.
 
     :param str source: The template source file, relative to
-    `$CHARM_DIR/templates`
+        `$CHARM_DIR/templates`
 
     :param str target: The target to write the rendered template to
     :param str owner: The owner of the rendered file
     :param str group: The group of the rendered file
     :param int perms: The permissions of the rendered file
+    :param partial on_change_action: functools partial to be executed when
+                                     rendered file changes
     """
     def __init__(self, source, target,
-                 owner='root', group='root', perms=0o444):
+                 owner='root', group='root', perms=0o444,
+                 on_change_action=None):
         self.source = source
         self.target = target
         self.owner = owner
         self.group = group
         self.perms = perms
+        self.on_change_action = on_change_action
 
     def __call__(self, manager, service_name, event_name):
+        pre_checksum = ''
+        if self.on_change_action and os.path.isfile(self.target):
+            pre_checksum = host.file_hash(self.target)
         service = manager.get_service(service_name)
         context = {}
         for ctx in service.get('required_data', []):
             context.update(ctx)
         templating.render(self.source, self.target, context,
                           self.owner, self.group, self.perms)
+        if self.on_change_action:
+            if pre_checksum == host.file_hash(self.target):
+                hookenv.log(
+                    'No change detected: {}'.format(self.target),
+                    hookenv.DEBUG)
+            else:
+                self.on_change_action()
 
 
 # Convenience aliases for templates

=== modified file 'hooks/charmhelpers/core/unitdata.py'
--- hooks/charmhelpers/core/unitdata.py	2015-04-16 20:24:28 +0000
+++ hooks/charmhelpers/core/unitdata.py	2015-08-27 15:02:41 +0000
@@ -152,6 +152,7 @@
 import collections
 import contextlib
 import datetime
+import itertools
 import json
 import os
 import pprint
@@ -164,8 +165,7 @@
 class Storage(object):
     """Simple key value database for local unit state within charms.
 
-    Modifications are automatically committed at hook exit. That's
-    currently regardless of exit code.
+    Modifications are not persisted unless :meth:`flush` is called.
 
     To support dicts, lists, integer, floats, and booleans values
     are automatically json encoded/decoded.
@@ -173,8 +173,11 @@
     def __init__(self, path=None):
         self.db_path = path
         if path is None:
-            self.db_path = os.path.join(
-                os.environ.get('CHARM_DIR', ''), '.unit-state.db')
+            if 'UNIT_STATE_DB' in os.environ:
+                self.db_path = os.environ['UNIT_STATE_DB']
+            else:
+                self.db_path = os.path.join(
+                    os.environ.get('CHARM_DIR', ''), '.unit-state.db')
         self.conn = sqlite3.connect('%s' % self.db_path)
         self.cursor = self.conn.cursor()
         self.revision = None
@@ -189,15 +192,8 @@
         self.conn.close()
         self._closed = True
 
-    def _scoped_query(self, stmt, params=None):
-        if params is None:
-            params = []
-        return stmt, params
-
     def get(self, key, default=None, record=False):
-        self.cursor.execute(
-            *self._scoped_query(
-                'select data from kv where key=?', [key]))
+        self.cursor.execute('select data from kv where key=?', [key])
         result = self.cursor.fetchone()
         if not result:
             return default
@@ -206,33 +202,81 @@
         return json.loads(result[0])
 
     def getrange(self, key_prefix, strip=False):
-        stmt = "select key, data from kv where key like '%s%%'" % key_prefix
-        self.cursor.execute(*self._scoped_query(stmt))
+        """
+        Get a range of keys starting with a common prefix as a mapping of
+        keys to values.
+
+        :param str key_prefix: Common prefix among all keys
+        :param bool strip: Optionally strip the common prefix from the key
+            names in the returned dict
+        :return dict: A (possibly empty) dict of key-value mappings
+        """
+        self.cursor.execute("select key, data from kv where key like ?",
+                            ['%s%%' % key_prefix])
         result = self.cursor.fetchall()
 
         if not result:
-            return None
+            return {}
         if not strip:
             key_prefix = ''
         return dict([
             (k[len(key_prefix):], json.loads(v)) for k, v in result])
 
     def update(self, mapping, prefix=""):
+        """
+        Set the values of multiple keys at once.
+
+        :param dict mapping: Mapping of keys to values
+        :param str prefix: Optional prefix to apply to all keys in `mapping`
+            before setting
+        """
         for k, v in mapping.items():
             self.set("%s%s" % (prefix, k), v)
 
     def unset(self, key):
+        """
+        Remove a key from the database entirely.
+        """
         self.cursor.execute('delete from kv where key=?', [key])
         if self.revision and self.cursor.rowcount:
             self.cursor.execute(
                 'insert into kv_revisions values (?, ?, ?)',
                 [key, self.revision, json.dumps('DELETED')])
 
+    def unsetrange(self, keys=None, prefix=""):
+        """
+        Remove a range of keys starting with a common prefix, from the database
+        entirely.
+
+        :param list keys: List of keys to remove.
+        :param str prefix: Optional prefix to apply to all keys in ``keys``
+            before removing.
+        """
+        if keys is not None:
+            keys = ['%s%s' % (prefix, key) for key in keys]
+            self.cursor.execute('delete from kv where key in (%s)' % ','.join(['?'] * len(keys)), keys)
+            if self.revision and self.cursor.rowcount:
+                self.cursor.execute(
+                    'insert into kv_revisions values %s' % ','.join(['(?, ?, ?)'] * len(keys)),
+                    list(itertools.chain.from_iterable((key, self.revision, json.dumps('DELETED')) for key in keys)))
+        else:
+            self.cursor.execute('delete from kv where key like ?',
+                                ['%s%%' % prefix])
+            if self.revision and self.cursor.rowcount:
+                self.cursor.execute(
+                    'insert into kv_revisions values (?, ?, ?)',
+                    ['%s%%' % prefix, self.revision, json.dumps('DELETED')])
+
     def set(self, key, value):
+        """
+        Set a value in the database.
+
+        :param str key: Key to set the value for
+        :param value: Any JSON-serializable value to be set
+        """
         serialized = json.dumps(value)
 
-        self.cursor.execute(
-            'select data from kv where key=?', [key])
+        self.cursor.execute('select data from kv where key=?', [key])
         exists = self.cursor.fetchone()
 
         # Skip mutations to the same value

=== modified file 'hooks/charmhelpers/fetch/__init__.py'
--- hooks/charmhelpers/fetch/__init__.py	2015-01-26 09:46:38 +0000
+++ hooks/charmhelpers/fetch/__init__.py	2015-08-27 15:02:41 +0000
@@ -90,6 +90,14 @@
     'kilo/proposed': 'trusty-proposed/kilo',
     'trusty-kilo/proposed': 'trusty-proposed/kilo',
     'trusty-proposed/kilo': 'trusty-proposed/kilo',
+    # Liberty
+    'liberty': 'trusty-updates/liberty',
+    'trusty-liberty': 'trusty-updates/liberty',
+    'trusty-liberty/updates': 'trusty-updates/liberty',
+    'trusty-updates/liberty': 'trusty-updates/liberty',
+    'liberty/proposed': 'trusty-proposed/liberty',
+    'trusty-liberty/proposed': 'trusty-proposed/liberty',
+    'trusty-proposed/liberty': 'trusty-proposed/liberty',
 }
 
 # The order of this list is very important. Handlers should be listed in from
@@ -158,7 +166,7 @@
 
 def apt_cache(in_memory=True):
     """Build and return an apt cache"""
-    import apt_pkg
+    from apt import apt_pkg
     apt_pkg.init()
     if in_memory:
         apt_pkg.config.set("Dir::Cache::pkgcache", "")
@@ -215,19 +223,27 @@
     _run_apt_command(cmd, fatal)
 
 
+def apt_mark(packages, mark, fatal=False):
+    """Flag one or more packages using apt-mark"""
+    cmd = ['apt-mark', mark]
+    if isinstance(packages, six.string_types):
+        cmd.append(packages)
+    else:
+        cmd.extend(packages)
+    log("Holding {}".format(packages))
+
+    if fatal:
+        subprocess.check_call(cmd, universal_newlines=True)
+    else:
+        subprocess.call(cmd, universal_newlines=True)
+
+
 def apt_hold(packages, fatal=False):
-    """Hold one or more packages"""
-    cmd = ['apt-mark', 'hold']
-    if isinstance(packages, six.string_types):
-        cmd.append(packages)
-    else:
-        cmd.extend(packages)
-    log("Holding {}".format(packages))
-
-    if fatal:
-        subprocess.check_call(cmd)
-    else:
-        subprocess.call(cmd)
+    return apt_mark(packages, 'hold', fatal=fatal)
+
+
+def apt_unhold(packages, fatal=False):
+    return apt_mark(packages, 'unhold', fatal=fatal)
 
 
 def add_source(source, key=None):
@@ -370,8 +386,9 @@
     for handler in handlers:
         try:
             installed_to = handler.install(source, *args, **kwargs)
-        except UnhandledSource:
-            pass
+        except UnhandledSource as e:
+            log('Install source attempt unsuccessful: {}'.format(e),
+                level='WARNING')
     if not installed_to:
         raise UnhandledSource("No handler found for source {}".format(source))
     return installed_to

=== modified file 'hooks/charmhelpers/fetch/archiveurl.py'
--- hooks/charmhelpers/fetch/archiveurl.py	2015-02-26 10:11:26 +0000
+++ hooks/charmhelpers/fetch/archiveurl.py	2015-08-27 15:02:41 +0000
@@ -77,6 +77,8 @@
     def can_handle(self, source):
         url_parts = self.parse_url(source)
         if url_parts.scheme not in ('http', 'https', 'ftp', 'file'):
+            # XXX: Why is this returning a boolean and a string? It's
+            # doomed to fail since "bool(can_handle('foo://'))"  will be True.
             return "Wrong source type"
         if get_archive_handler(self.base_url(source)):
             return True
@@ -155,7 +157,11 @@
             else:
                 algorithms = hashlib.algorithms_available
             if key in algorithms:
-                check_hash(dld_file, value, key)
+                if len(value) != 1:
+                    raise TypeError(
+                        "Expected 1 hash value, not %d" % len(value))
+                expected = value[0]
+                check_hash(dld_file, expected, key)
         if checksum:
             check_hash(dld_file, checksum, hash_type)
         return extract(dld_file, dest)

=== modified file 'hooks/charmhelpers/fetch/giturl.py'
--- hooks/charmhelpers/fetch/giturl.py	2015-02-26 10:11:26 +0000
+++ hooks/charmhelpers/fetch/giturl.py	2015-08-27 15:02:41 +0000
@@ -45,14 +45,16 @@
         else:
             return True
 
-    def clone(self, source, dest, branch):
+    def clone(self, source, dest, branch, depth=None):
         if not self.can_handle(source):
             raise UnhandledSource("Cannot handle {}".format(source))
 
-        repo = Repo.clone_from(source, dest)
-        repo.git.checkout(branch)
+        if depth:
+            Repo.clone_from(source, dest, branch=branch, depth=depth)
+        else:
+            Repo.clone_from(source, dest, branch=branch)
 
-    def install(self, source, branch="master", dest=None):
+    def install(self, source, branch="master", dest=None, depth=None):
         url_parts = self.parse_url(source)
         branch_name = url_parts.path.strip("/").split("/")[-1]
         if dest:
@@ -63,9 +65,9 @@
         if not os.path.exists(dest_dir):
             mkdir(dest_dir, perms=0o755)
         try:
-            self.clone(source, dest_dir, branch)
+            self.clone(source, dest_dir, branch, depth)
         except GitCommandError as e:
-            raise UnhandledSource(e.message)
+            raise UnhandledSource(e)
         except OSError as e:
             raise UnhandledSource(e.strerror)
         return dest_dir

=== modified file 'hooks/horizon_contexts.py'
--- hooks/horizon_contexts.py	2015-06-03 13:47:12 +0000
+++ hooks/horizon_contexts.py	2015-08-27 15:02:41 +0000
@@ -10,7 +10,6 @@
 )
 from charmhelpers.contrib.openstack.context import (
     OSContextGenerator,
-    OSPatternContextGenerator,
     HAProxyContext,
     context_complete
 )
@@ -28,7 +27,6 @@
 
 from base64 import b64decode
 import os
-import re
 
 
 class HorizonHAProxyContext(HAProxyContext):
@@ -199,27 +197,3 @@
                                     key=lambda r: r[1]['priority'])]
         }
         return ctxt
-
-
-class PluginsContext(OSPatternContextGenerator):
-    def __call__(self):
-
-        plugins = {}
-
-        for rid in relation_ids("plugin"):
-            try:
-                unit = related_units(rid)[0]
-            except IndexError:
-                pass
-            else:
-                rdata = relation_get(unit=unit, rid=rid)
-                try:
-                    if rdata['priority'] is not None and rdata['plugin-file']:
-                        service = re.sub('[^a-z0-9_]', '_', unit.split('/')[0])
-                        plugins[(rdata['priority'], service)] = {
-                            'unit': unit,
-                            'plugin_file': rdata['plugin-file']}
-                except KeyError:
-                    pass
-
-        return plugins

=== modified file 'hooks/horizon_hooks.py'
--- hooks/horizon_hooks.py	2015-06-03 13:07:42 +0000
+++ hooks/horizon_hooks.py	2015-08-27 15:02:41 +0000
@@ -33,7 +33,7 @@
     register_configs,
     restart_map,
     services,
-    LOCAL_SETTINGS, HAPROXY_CONF, PLUGIN_SETTINGS,
+    LOCAL_SETTINGS, HAPROXY_CONF,
     enable_ssl,
     do_openstack_upgrade,
     git_install,
@@ -125,7 +125,7 @@
     open_port(443)
 
     if git_install_requested():
-        git_post_install_late()
+        git_post_install_late(config('openstack-origin-git'))
 
 
 @hooks.hook('identity-service-relation-joined')
@@ -253,7 +253,6 @@
 @restart_on_change(restart_map())
 def update_plugin_config():
     CONFIGS.write(LOCAL_SETTINGS)
-    CONFIGS.write(PLUGIN_SETTINGS)
 
 
 def main():

=== modified file 'hooks/horizon_utils.py'
--- hooks/horizon_utils.py	2015-06-03 13:07:42 +0000
+++ hooks/horizon_utils.py	2015-08-27 15:02:41 +0000
@@ -5,7 +5,6 @@
 import pwd
 import subprocess
 import shutil
-import string
 from collections import OrderedDict
 
 import charmhelpers.contrib.openstack.context as context
@@ -18,9 +17,13 @@
     git_clone_and_install,
     os_release,
     git_src_dir,
+    git_pip_venv_dir,
+    git_yaml_value,
+)
+from charmhelpers.contrib.python.packages import (
+    pip_install,
 )
 from charmhelpers.core.hookenv import (
-    charm_dir,
     config,
     log
 )
@@ -33,7 +36,9 @@
     mkdir,
     service_restart,
 )
-
+from charmhelpers.core.templating import (
+    render,
+)
 from charmhelpers.fetch import (
     apt_upgrade,
     apt_update,
@@ -54,9 +59,14 @@
 BASE_GIT_PACKAGES = [
     'apache2',
     'libapache2-mod-wsgi',
+    'libffi-dev',
+    'libpcre3-dev',
+    'libssl-dev',
     'libxml2-dev',
     'libxslt1-dev',
+    'libyaml-dev',
     'python-dev',
+    'python-lesscpy',
     'python-pip',
     'python-setuptools',
     'zlib1g-dev',
@@ -83,9 +93,6 @@
 APACHE_DEFAULT = "%s/sites-available/default" % (APACHE_CONF_DIR)
 ROUTER_SETTING = \
     "/usr/share/openstack-dashboard/openstack_dashboard/enabled/_40_router.py"
-PLUGIN_SETTINGS = \
-    "/usr/share/openstack-dashboard/openstack_dashboard/local/enabled" \
-    "/_{}_juju_{}.py"
 
 TEMPLATES = 'templates'
 
@@ -137,10 +144,6 @@
         'hook_contexts': [horizon_contexts.RouterSettingContext()],
         'services': ['apache2'],
     }),
-    (PLUGIN_SETTINGS, {
-        'hook_contexts': [horizon_contexts.PluginsContext()],
-        'services': ['apache2'],
-    }),
 ])
 
 
@@ -179,11 +182,6 @@
     if os.path.exists(os.path.dirname(ROUTER_SETTING)):
         configs.register(ROUTER_SETTING,
                          CONFIG_FILES[ROUTER_SETTING]['hook_contexts'])
-
-    if os_release('openstack_dashboard') >= 'icehouse':
-        configs.register_pattern(PLUGIN_SETTINGS,
-                                 CONFIG_FILES[PLUGIN_SETTINGS]
-                                             ['hook_contexts'])
     return configs
 
 
@@ -201,10 +199,7 @@
         for svc in ctxt['services']:
             svcs.append(svc)
         if svcs:
-            # replace all formatting in path with asterisks for glob
-            path = ''.join([t[0] + ('' if t[1] is None else '*')
-                           for t in string.Formatter().parse(f)])
-            _map.append((path, svcs))
+            _map.append((f, svcs))
     return OrderedDict(_map)
 
 
@@ -313,7 +308,6 @@
 def git_post_install(projects_yaml):
     """Perform horizon post-install setup."""
     src_dir = git_src_dir(projects_yaml, 'horizon')
-    templates_dir = os.path.join(charm_dir(), 'templates/git')
     copy_files = {
         'manage': {
             'src': os.path.join(src_dir, 'manage.py'),
@@ -328,10 +322,6 @@
                                 'local_settings.py.example'),
             'dest': '/etc/openstack-dashboard/local_settings.py',
         },
-        'openstack-dashboard': {
-            'src': os.path.join(templates_dir, 'dashboard.conf'),
-            'dest': '/etc/apache2/conf-available/openstack-dashboard.conf',
-        },
     }
 
     for name, files in copy_files.iteritems():
@@ -359,8 +349,8 @@
          'link': '/usr/share/openstack-dashboard/bin/less/lessc'},
         {'src': '/etc/openstack-dashboard/local_settings.py',
          'link': os.path.join(share_dir, 'local/local_settings.py')},
-        {'src':
-            '/usr/local/lib/python2.7/dist-packages/horizon/static/horizon/',
+        {'src': os.path.join(git_pip_venv_dir(projects_yaml),
+         'local/lib/python2.7/site-packages/horizon/static/horizon/'),
          'link': os.path.join(share_dir, 'static/horizon')},
     ]
 
@@ -369,12 +359,25 @@
             os.remove(s['link'])
         os.symlink(s['src'], s['link'])
 
+    render('git/dashboard.conf',
+           '/etc/apache2/conf-available/openstack-dashboard.conf',
+           {'virtualenv': git_pip_venv_dir(projects_yaml)},
+           owner='root', group='root', perms=0o644)
+
     os.chmod('/var/lib/openstack-dashboard', 0o750)
     os.chmod('/usr/share/openstack-dashboard/manage.py', 0o755),
 
-    subprocess.check_call(['/usr/share/openstack-dashboard/manage.py',
+    http_proxy = git_yaml_value(projects_yaml, 'http_proxy')
+    if http_proxy:
+        pip_install('python-memcached', proxy=http_proxy,
+                    venv=git_pip_venv_dir(projects_yaml))
+    else:
+        pip_install('python-memcached',
+                    venv=git_pip_venv_dir(projects_yaml))
+    python = os.path.join(git_pip_venv_dir(projects_yaml), 'bin/python')
+    subprocess.check_call([python, '/usr/share/openstack-dashboard/manage.py',
                            'collectstatic', '--noinput'])
-    subprocess.check_call(['/usr/share/openstack-dashboard/manage.py',
+    subprocess.check_call([python, '/usr/share/openstack-dashboard/manage.py',
                            'compress', '--force'])
 
     uid = pwd.getpwnam('horizon').pw_uid
@@ -396,9 +399,15 @@
     service_restart('apache2')
 
 
-def git_post_install_late():
+def git_post_install_late(projects_yaml):
     """Perform horizon post-install setup."""
-    subprocess.check_call(['/usr/share/openstack-dashboard/manage.py',
+    render('git/dashboard.conf',
+           '/etc/apache2/conf-available/openstack-dashboard.conf',
+           {'virtualenv': git_pip_venv_dir(projects_yaml)},
+           owner='root', group='root', perms=0o644)
+
+    python = os.path.join(git_pip_venv_dir(projects_yaml), 'bin/python')
+    subprocess.check_call([python, '/usr/share/openstack-dashboard/manage.py',
                            'collectstatic', '--noinput'])
-    subprocess.check_call(['/usr/share/openstack-dashboard/manage.py',
+    subprocess.check_call([python, '/usr/share/openstack-dashboard/manage.py',
                            'compress', '--force'])

=== modified file 'templates/git/dashboard.conf'
--- templates/git/dashboard.conf	2015-04-13 14:18:45 +0000
+++ templates/git/dashboard.conf	2015-08-27 15:02:41 +0000
@@ -1,6 +1,7 @@
 WSGIScriptAlias /horizon /usr/share/openstack-dashboard/openstack_dashboard/wsgi/django.wsgi
 WSGIDaemonProcess horizon user=horizon group=horizon processes=3 threads=10
 WSGIProcessGroup horizon
+WSGIPythonHome {{ virtualenv }}
 Alias /static /usr/share/openstack-dashboard/openstack_dashboard/static/
 <Directory /usr/share/openstack-dashboard/openstack_dashboard/wsgi>
   Order allow,deny

=== removed file 'templates/icehouse/_{}_juju_{}.py'
--- templates/icehouse/_{}_juju_{}.py	2015-06-03 13:07:42 +0000
+++ templates/icehouse/_{}_juju_{}.py	1970-01-01 00:00:00 +0000
@@ -1,2 +0,0 @@
-# {{ unit }}
-{{ plugin_file }}
\ No newline at end of file

=== modified file 'tests/00-setup'
--- tests/00-setup	2015-02-11 18:24:07 +0000
+++ tests/00-setup	2015-08-27 15:02:41 +0000
@@ -5,6 +5,7 @@
 sudo add-apt-repository --yes ppa:juju/stable
 sudo apt-get update --yes
 sudo apt-get install --yes python-amulet \
+                           python-distro-info \
                            python-neutronclient \
                            python-keystoneclient \
                            python-novaclient \

=== modified file 'tests/017-basic-trusty-kilo' (properties changed: -x to +x)
=== removed file 'tests/018-basic-utopic-juno'
--- tests/018-basic-utopic-juno	2015-04-16 21:36:17 +0000
+++ tests/018-basic-utopic-juno	1970-01-01 00:00:00 +0000
@@ -1,9 +0,0 @@
-#!/usr/bin/python
-
-"""Amulet tests on a basic openstack-dashboard deployment on utopic-juno."""
-
-from basic_deployment import OpenstackDashboardBasicDeployment
-
-if __name__ == '__main__':
-    deployment = OpenstackDashboardBasicDeployment(series='utopic')
-    deployment.run_tests()

=== added file 'tests/052-basic-trusty-kilo-git'
--- tests/052-basic-trusty-kilo-git	1970-01-01 00:00:00 +0000
+++ tests/052-basic-trusty-kilo-git	2015-08-27 15:02:41 +0000
@@ -0,0 +1,12 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic openstack-dashboard git deployment on trusty-kilo."""
+
+from basic_deployment import OpenstackDashboardBasicDeployment
+
+if __name__ == '__main__':
+    deployment = OpenstackDashboardBasicDeployment(series='trusty',
+                                                   openstack='cloud:trusty-kilo',
+                                                   source='cloud:trusty-updates/kilo',
+                                                   git=True)
+    deployment.run_tests()

=== modified file 'tests/basic_deployment.py'
--- tests/basic_deployment.py	2015-04-15 19:31:27 +0000
+++ tests/basic_deployment.py	2015-08-27 15:02:41 +0000
@@ -59,21 +59,58 @@
         """Configure all of the services."""
         horizon_config = {}
         if self.git:
+            amulet_http_proxy = os.environ.get('AMULET_HTTP_PROXY')
+
+            reqs_repo = 'git://github.com/openstack/requirements'
+            horizon_repo = 'git://github.com/openstack/horizon'
+            if self._get_openstack_release() == self.trusty_icehouse:
+                reqs_repo = 'git://github.com/coreycb/requirements'
+                horizon_repo = 'git://github.com/coreycb/horizon'
+
             branch = 'stable/' + self._get_openstack_release_string()
-            amulet_http_proxy = os.environ.get('AMULET_HTTP_PROXY')
-            openstack_origin_git = {
-                'repositories': [
-                    {'name': 'requirements',
-                     'repository': 'git://git.openstack.org/openstack/requirements',
-                     'branch': branch},
-                    {'name': 'horizon',
-                     'repository': 'git://git.openstack.org/openstack/horizon',
-                     'branch': branch},
-                ],
-                'directory': '/mnt/openstack-git',
-                'http_proxy': amulet_http_proxy,
-                'https_proxy': amulet_http_proxy,
-            }
+
+            if self._get_openstack_release() == self.trusty_juno:
+                openstack_origin_git = {
+                    'repositories': [
+                        {'name': 'requirements',
+                         'repository': reqs_repo,
+                         'branch': branch},
+                        # NOTE(coreycb): Pin oslo libraries here because they're not
+                        # capped and recently released versions causing issues for juno.
+                        {'name': 'oslo-config',
+                         'repository': 'git://github.com/openstack/oslo.config',
+                         'branch': '1.6.0'},
+                        {'name': 'oslo-i18n',
+                         'repository': 'git://github.com/openstack/oslo.i18n',
+                         'branch': '1.3.1'},
+                        {'name': 'oslo-serialization',
+                         'repository': 'git://github.com/openstack/oslo.serialization',
+                         'branch': '1.2.0'},
+                        {'name': 'oslo-utils',
+                         'repository': 'git://github.com/openstack/oslo.utils',
+                         'branch': '1.4.0'},
+                        {'name': 'horizon',
+                         'repository': horizon_repo,
+                         'branch': branch},
+                    ],
+                    'directory': '/mnt/openstack-git',
+                    'http_proxy': amulet_http_proxy,
+                    'https_proxy': amulet_http_proxy,
+                }
+            else:
+                openstack_origin_git = {
+                    'repositories': [
+                        {'name': 'requirements',
+                         'repository': reqs_repo,
+                         'branch': branch},
+                        {'name': 'horizon',
+                         'repository': horizon_repo,
+                         'branch': branch},
+                    ],
+                    'directory': '/mnt/openstack-git',
+                    'http_proxy': amulet_http_proxy,
+                    'https_proxy': amulet_http_proxy,
+                }
             horizon_config['openstack-origin-git'] = yaml.dump(openstack_origin_git)
 
         keystone_config = {'admin-password': 'openstack',
@@ -164,7 +201,7 @@
         conf = '/etc/openstack-dashboard/local_settings.py'
         services = ['apache2']
         self.d.configure('openstack-dashboard', {'use-syslog': 'True'})
-        time = 40
+        time = 120
         for s in services:
             if not u.service_restarted(self.openstack_dashboard_sentry, s, conf,
                                        pgrep_full=True, sleep_time=time):

=== modified file 'tests/charmhelpers/contrib/amulet/utils.py'
--- tests/charmhelpers/contrib/amulet/utils.py	2015-04-23 14:53:21 +0000
+++ tests/charmhelpers/contrib/amulet/utils.py	2015-08-27 15:02:41 +0000
@@ -14,14 +14,23 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
-import ConfigParser
 import io
+import json
 import logging
+import os
 import re
+import subprocess
 import sys
 import time
 
+import amulet
+import distro_info
 import six
+from six.moves import configparser
+if six.PY3:
+    from urllib import parse as urlparse
+else:
+    import urlparse
 
 
 class AmuletUtils(object):
@@ -33,6 +42,7 @@
 
     def __init__(self, log_level=logging.ERROR):
         self.log = self.get_logger(level=log_level)
+        self.ubuntu_releases = self.get_ubuntu_releases()
 
     def get_logger(self, name="amulet-logger", level=logging.DEBUG):
         """Get a logger object that will log to stdout."""
@@ -70,12 +80,44 @@
         else:
             return False
 
+    def get_ubuntu_release_from_sentry(self, sentry_unit):
+        """Get Ubuntu release codename from sentry unit.
+
+        :param sentry_unit: amulet sentry/service unit pointer
+        :returns: list of strings - release codename, failure message
+        """
+        msg = None
+        cmd = 'lsb_release -cs'
+        release, code = sentry_unit.run(cmd)
+        if code == 0:
+            self.log.debug('{} lsb_release: {}'.format(
+                sentry_unit.info['unit_name'], release))
+        else:
+            msg = ('{} `{}` returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, release, code))
+        if release not in self.ubuntu_releases:
+            msg = ("Release ({}) not found in Ubuntu releases "
+                   "({})".format(release, self.ubuntu_releases))
+        return release, msg
+
     def validate_services(self, commands):
-        """Validate services.
-
-           Verify the specified services are running on the corresponding
+        """Validate that lists of commands succeed on service units.  Can be
+           used to verify system services are running on the corresponding
            service units.
-           """
+
+        :param commands: dict with sentry keys and arbitrary command list vals
+        :returns: None if successful, Failure string message otherwise
+        """
+        self.log.debug('Checking status of system services...')
+
+        # /!\ DEPRECATION WARNING (beisner):
+        # New and existing tests should be rewritten to use
+        # validate_services_by_name() as it is aware of init systems.
+        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+                      'validate_services_by_name instead of validate_services '
+                      'due to init system differences.')
+
         for k, v in six.iteritems(commands):
             for cmd in v:
                 output, code = k.run(cmd)
@@ -86,6 +128,45 @@
                     return "command `{}` returned {}".format(cmd, str(code))
         return None
 
+    def validate_services_by_name(self, sentry_services):
+        """Validate system service status by service name, automatically
+           detecting init system based on Ubuntu release codename.
+
+        :param sentry_services: dict with sentry keys and svc list values
+        :returns: None if successful, Failure string message otherwise
+        """
+        self.log.debug('Checking status of system services...')
+
+        # Point at which systemd became a thing
+        systemd_switch = self.ubuntu_releases.index('vivid')
+
+        for sentry_unit, services_list in six.iteritems(sentry_services):
+            # Get lsb_release codename from unit
+            release, ret = self.get_ubuntu_release_from_sentry(sentry_unit)
+            if ret:
+                return ret
+
+            for service_name in services_list:
+                if (self.ubuntu_releases.index(release) >= systemd_switch or
+                        service_name in ['rabbitmq-server', 'apache2']):
+                    # init is systemd (or regular sysv)
+                    cmd = 'sudo service {} status'.format(service_name)
+                    output, code = sentry_unit.run(cmd)
+                    service_running = code == 0
+                elif self.ubuntu_releases.index(release) < systemd_switch:
+                    # init is upstart
+                    cmd = 'sudo status {}'.format(service_name)
+                    output, code = sentry_unit.run(cmd)
+                    service_running = code == 0 and "start/running" in output
+
+                self.log.debug('{} `{}` returned '
+                               '{}'.format(sentry_unit.info['unit_name'],
+                                           cmd, code))
+                if not service_running:
+                    return u"command `{}` returned {} {}".format(
+                        cmd, output, str(code))
+        return None
+
     def _get_config(self, unit, filename):
         """Get a ConfigParser object for parsing a unit's config file."""
         file_contents = unit.file_contents(filename)
@@ -93,7 +174,7 @@
         # NOTE(beisner):  by default, ConfigParser does not handle options
         # with no value, such as the flags used in the mysql my.cnf file.
         # https://bugs.python.org/issue7005
-        config = ConfigParser.ConfigParser(allow_no_value=True)
+        config = configparser.ConfigParser(allow_no_value=True)
         config.readfp(io.StringIO(file_contents))
         return config
 
@@ -103,7 +184,15 @@
 
            Verify that the specified section of the config file contains
            the expected option key:value pairs.
+
+           Compare expected dictionary data vs actual dictionary data.
+           The values in the 'expected' dictionary can be strings, bools, ints,
+           longs, or can be a function that evaluates a variable and returns a
+           bool.
            """
+        self.log.debug('Validating config file data ({} in {} on {})'
+                       '...'.format(section, config_file,
+                                    sentry_unit.info['unit_name']))
         config = self._get_config(sentry_unit, config_file)
 
         if section != 'DEFAULT' and not config.has_section(section):
@@ -112,9 +201,20 @@
         for k in expected.keys():
             if not config.has_option(section, k):
                 return "section [{}] is missing option {}".format(section, k)
-            if config.get(section, k) != expected[k]:
+
+            actual = config.get(section, k)
+            v = expected[k]
+            if (isinstance(v, six.string_types) or
+                    isinstance(v, bool) or
+                    isinstance(v, six.integer_types)):
+                # handle explicit values
+                if actual != v:
+                    return "section [{}] {}:{} != expected {}:{}".format(
+                           section, k, actual, k, expected[k])
+            # handle function pointers, such as not_null or valid_ip
+            elif not v(actual):
                 return "section [{}] {}:{} != expected {}:{}".format(
-                       section, k, config.get(section, k), k, expected[k])
+                       section, k, actual, k, expected[k])
         return None
 
     def _validate_dict_data(self, expected, actual):
@@ -122,7 +222,7 @@
 
            Compare expected dictionary data vs actual dictionary data.
            The values in the 'expected' dictionary can be strings, bools, ints,
-           longs, or can be a function that evaluate a variable and returns a
+           longs, or can be a function that evaluates a variable and returns a
            bool.
            """
         self.log.debug('actual: {}'.format(repr(actual)))
@@ -133,8 +233,10 @@
                 if (isinstance(v, six.string_types) or
                         isinstance(v, bool) or
                         isinstance(v, six.integer_types)):
+                    # handle explicit values
                     if v != actual[k]:
                         return "{}:{}".format(k, actual[k])
+                # handle function pointers, such as not_null or valid_ip
                 elif not v(actual[k]):
                     return "{}:{}".format(k, actual[k])
             else:
@@ -321,3 +423,174 @@
 
     def endpoint_error(self, name, data):
         return 'unexpected endpoint data in {} - {}'.format(name, data)
+
+    def get_ubuntu_releases(self):
+        """Return a list of all Ubuntu releases in order of release."""
+        _d = distro_info.UbuntuDistroInfo()
+        _release_list = _d.all
+        self.log.debug('Ubuntu release list: {}'.format(_release_list))
+        return _release_list
+
+    def file_to_url(self, file_rel_path):
+        """Convert a relative file path to a file URL."""
+        _abs_path = os.path.abspath(file_rel_path)
+        return urlparse.urlparse(_abs_path, scheme='file').geturl()
+
+    def check_commands_on_units(self, commands, sentry_units):
+        """Check that all commands in a list exit zero on all
+        sentry units in a list.
+
+        :param commands:  list of bash commands
+        :param sentry_units:  list of sentry unit pointers
+        :returns: None if successful; Failure message otherwise
+        """
+        self.log.debug('Checking exit codes for {} commands on {} '
+                       'sentry units...'.format(len(commands),
+                                                len(sentry_units)))
+        for sentry_unit in sentry_units:
+            for cmd in commands:
+                output, code = sentry_unit.run(cmd)
+                if code == 0:
+                    self.log.debug('{} `{}` returned {} '
+                                   '(OK)'.format(sentry_unit.info['unit_name'],
+                                                 cmd, code))
+                else:
+                    return ('{} `{}` returned {} '
+                            '{}'.format(sentry_unit.info['unit_name'],
+                                        cmd, code, output))
+        return None
+
+    def get_process_id_list(self, sentry_unit, process_name,
+                            expect_success=True):
+        """Get a list of process ID(s) from a single sentry juju unit
+        for a single process name.
+
+        :param sentry_unit: Amulet sentry instance (juju unit)
+        :param process_name: Process name
+        :param expect_success: If False, expect the PID to be missing,
+            raise if it is present.
+        :returns: List of process IDs
+        """
+        cmd = 'pidof -x {}'.format(process_name)
+        if not expect_success:
+            cmd += " || exit 0 && exit 1"
+        output, code = sentry_unit.run(cmd)
+        if code != 0:
+            msg = ('{} `{}` returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, code, output))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+        return str(output).split()
+
+    def get_unit_process_ids(self, unit_processes, expect_success=True):
+        """Construct a dict containing unit sentries, process names, and
+        process IDs.
+
+        :param unit_processes: A dictionary of Amulet sentry instance
+            to list of process names.
+        :param expect_success: if False expect the processes to not be
+            running, raise if they are.
+        :returns: Dictionary of Amulet sentry instance to dictionary
+            of process names to PIDs.
+        """
+        pid_dict = {}
+        for sentry_unit, process_list in six.iteritems(unit_processes):
+            pid_dict[sentry_unit] = {}
+            for process in process_list:
+                pids = self.get_process_id_list(
+                    sentry_unit, process, expect_success=expect_success)
+                pid_dict[sentry_unit].update({process: pids})
+        return pid_dict
+
+    def validate_unit_process_ids(self, expected, actual):
+        """Validate process id quantities for services on units."""
+        self.log.debug('Checking units for running processes...')
+        self.log.debug('Expected PIDs: {}'.format(expected))
+        self.log.debug('Actual PIDs: {}'.format(actual))
+
+        if len(actual) != len(expected):
+            return ('Unit count mismatch.  expected, actual: {}, '
+                    '{} '.format(len(expected), len(actual)))
+
+        for (e_sentry, e_proc_names) in six.iteritems(expected):
+            e_sentry_name = e_sentry.info['unit_name']
+            if e_sentry in actual.keys():
+                a_proc_names = actual[e_sentry]
+            else:
+                return ('Expected sentry ({}) not found in actual dict data.'
+                        '{}'.format(e_sentry_name, e_sentry))
+
+            if len(e_proc_names.keys()) != len(a_proc_names.keys()):
+                return ('Process name count mismatch.  expected, actual: {}, '
+                        '{}'.format(len(expected), len(actual)))
+
+            for (e_proc_name, e_pids_length), (a_proc_name, a_pids) in \
+                    zip(e_proc_names.items(), a_proc_names.items()):
+                if e_proc_name != a_proc_name:
+                    return ('Process name mismatch.  expected, actual: {}, '
+                            '{}'.format(e_proc_name, a_proc_name))
+
+                a_pids_length = len(a_pids)
+                fail_msg = ('PID count mismatch. {} ({}) expected, actual: '
+                            '{}, {} ({})'.format(e_sentry_name, e_proc_name,
+                                                 e_pids_length, a_pids_length,
+                                                 a_pids))
+
+                # If expected is not bool, ensure PID quantities match
+                if not isinstance(e_pids_length, bool) and \
+                        a_pids_length != e_pids_length:
+                    return fail_msg
+                # If expected is bool True, ensure 1 or more PIDs exist
+                elif isinstance(e_pids_length, bool) and \
+                        e_pids_length is True and a_pids_length < 1:
+                    return fail_msg
+                # If expected is bool False, ensure 0 PIDs exist
+                elif isinstance(e_pids_length, bool) and \
+                        e_pids_length is False and a_pids_length != 0:
+                    return fail_msg
+                else:
+                    self.log.debug('PID check OK: {} {} {}: '
+                                   '{}'.format(e_sentry_name, e_proc_name,
+                                               e_pids_length, a_pids))
+        return None
+
+    def validate_list_of_identical_dicts(self, list_of_dicts):
+        """Check that all dicts within a list are identical."""
+        hashes = []
+        for _dict in list_of_dicts:
+            hashes.append(hash(frozenset(_dict.items())))
+
+        self.log.debug('Hashes: {}'.format(hashes))
+        if len(set(hashes)) == 1:
+            self.log.debug('Dicts within list are identical')
+        else:
+            return 'Dicts within list are not identical'
+
+        return None
+
+    def run_action(self, unit_sentry, action,
+                   _check_output=subprocess.check_output):
+        """Run the named action on a given unit sentry.
+
+        _check_output parameter is used for dependency injection.
+
+        @return action_id.
+        """
+        unit_id = unit_sentry.info["unit_name"]
+        command = ["juju", "action", "do", "--format=json", unit_id, action]
+        self.log.info("Running command: %s\n" % " ".join(command))
+        output = _check_output(command, universal_newlines=True)
+        data = json.loads(output)
+        action_id = data[u'Action queued with id']
+        return action_id
+
+    def wait_on_action(self, action_id, _check_output=subprocess.check_output):
+        """Wait for a given action, returning if it completed or not.
+
+        _check_output parameter is used for dependency injection.
+        """
+        command = ["juju", "action", "fetch", "--format=json", "--wait=0",
+                   action_id]
+        output = _check_output(command, universal_newlines=True)
+        data = json.loads(output)
+        return data.get(u"status") == "completed"

=== modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py'
--- tests/charmhelpers/contrib/openstack/amulet/deployment.py	2015-04-23 14:53:21 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/deployment.py	2015-08-27 15:02:41 +0000
@@ -44,7 +44,7 @@
            Determine if the local branch being tested is derived from its
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
-        base_charms = ['mysql', 'mongodb']
+        base_charms = ['mysql', 'mongodb', 'nrpe']
 
         if self.series in ['precise', 'trusty']:
             base_series = self.series
@@ -79,9 +79,9 @@
         services.append(this_service)
         use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
                       'ceph-osd', 'ceph-radosgw']
-        # Openstack subordinate charms do not expose an origin option as that
-        # is controlled by the principle
-        ignore = ['neutron-openvswitch']
+        # Most OpenStack subordinate charms do not expose an origin option
+        # as that is controlled by the principle.
+        ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe']
 
         if self.openstack:
             for svc in services:
@@ -110,7 +110,8 @@
         (self.precise_essex, self.precise_folsom, self.precise_grizzly,
          self.precise_havana, self.precise_icehouse,
          self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
-         self.trusty_kilo, self.vivid_kilo) = range(10)
+         self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
+         self.wily_liberty) = range(12)
 
         releases = {
             ('precise', None): self.precise_essex,
@@ -121,8 +122,10 @@
             ('trusty', None): self.trusty_icehouse,
             ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
             ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
+            ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
             ('utopic', None): self.utopic_juno,
-            ('vivid', None): self.vivid_kilo}
+            ('vivid', None): self.vivid_kilo,
+            ('wily', None): self.wily_liberty}
         return releases[(self.series, self.openstack)]
 
     def _get_openstack_release_string(self):
@@ -138,9 +141,43 @@
             ('trusty', 'icehouse'),
             ('utopic', 'juno'),
             ('vivid', 'kilo'),
+            ('wily', 'liberty'),
         ])
         if self.openstack:
             os_origin = self.openstack.split(':')[1]
             return os_origin.split('%s-' % self.series)[1].split('/')[0]
         else:
             return releases[self.series]
+
+    def get_ceph_expected_pools(self, radosgw=False):
+        """Return a list of expected ceph pools in a ceph + cinder + glance
+        test scenario, based on OpenStack release and whether ceph radosgw
+        is flagged as present or not."""
+
+        if self._get_openstack_release() >= self.trusty_kilo:
+            # Kilo or later
+            pools = [
+                'rbd',
+                'cinder',
+                'glance'
+            ]
+        else:
+            # Juno or earlier
+            pools = [
+                'data',
+                'metadata',
+                'rbd',
+                'cinder',
+                'glance'
+            ]
+
+        if radosgw:
+            pools.extend([
+                '.rgw.root',
+                '.rgw.control',
+                '.rgw',
+                '.rgw.gc',
+                '.users.uid'
+            ])
+
+        return pools

=== modified file 'tests/charmhelpers/contrib/openstack/amulet/utils.py'
--- tests/charmhelpers/contrib/openstack/amulet/utils.py	2015-02-10 18:50:39 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/utils.py	2015-08-27 15:02:41 +0000
@@ -14,16 +14,20 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import amulet
+import json
 import logging
 import os
+import six
 import time
 import urllib
 
+import cinderclient.v1.client as cinder_client
 import glanceclient.v1.client as glance_client
+import heatclient.v1.client as heat_client
 import keystoneclient.v2_0 as keystone_client
 import novaclient.v1_1.client as nova_client
-
-import six
+import swiftclient
 
 from charmhelpers.contrib.amulet.utils import (
     AmuletUtils
@@ -37,7 +41,7 @@
     """OpenStack amulet utilities.
 
        This class inherits from AmuletUtils and has additional support
-       that is specifically for use by OpenStack charms.
+       that is specifically for use by OpenStack charm tests.
        """
 
     def __init__(self, log_level=ERROR):
@@ -51,6 +55,8 @@
            Validate actual endpoint data vs expected endpoint data. The ports
            are used to find the matching endpoint.
            """
+        self.log.debug('Validating endpoint data...')
+        self.log.debug('actual: {}'.format(repr(endpoints)))
         found = False
         for ep in endpoints:
             self.log.debug('endpoint: {}'.format(repr(ep)))
@@ -77,6 +83,7 @@
            Validate a list of actual service catalog endpoints vs a list of
            expected service catalog endpoints.
            """
+        self.log.debug('Validating service catalog endpoint data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for k, v in six.iteritems(expected):
             if k in actual:
@@ -93,6 +100,7 @@
            Validate a list of actual tenant data vs list of expected tenant
            data.
            """
+        self.log.debug('Validating tenant data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -114,6 +122,7 @@
            Validate a list of actual role data vs a list of expected role
            data.
            """
+        self.log.debug('Validating role data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -134,6 +143,7 @@
            Validate a list of actual user data vs a list of expected user
            data.
            """
+        self.log.debug('Validating user data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -155,17 +165,30 @@
 
            Validate a list of actual flavors vs a list of expected flavors.
            """
+        self.log.debug('Validating flavor data...')
         self.log.debug('actual: {}'.format(repr(actual)))
         act = [a.name for a in actual]
         return self._validate_list_data(expected, act)
 
     def tenant_exists(self, keystone, tenant):
         """Return True if tenant exists."""
+        self.log.debug('Checking if tenant exists ({})...'.format(tenant))
         return tenant in [t.name for t in keystone.tenants.list()]
 
+    def authenticate_cinder_admin(self, keystone_sentry, username,
+                                  password, tenant):
+        """Authenticates admin user with cinder."""
+        # NOTE(beisner): cinder python client doesn't accept tokens.
+        service_ip = \
+            keystone_sentry.relation('shared-db',
+                                     'mysql:shared-db')['private-address']
+        ept = "http://{}:5000/v2.0".format(service_ip.strip().decode('utf-8'))
+        return cinder_client.Client(username, password, tenant, ept)
+
     def authenticate_keystone_admin(self, keystone_sentry, user, password,
                                     tenant):
         """Authenticates admin user with the keystone admin endpoint."""
+        self.log.debug('Authenticating keystone admin...')
         unit = keystone_sentry
         service_ip = unit.relation('shared-db',
                                    'mysql:shared-db')['private-address']
@@ -175,6 +198,7 @@
 
     def authenticate_keystone_user(self, keystone, user, password, tenant):
         """Authenticates a regular user with the keystone public endpoint."""
+        self.log.debug('Authenticating keystone user ({})...'.format(user))
         ep = keystone.service_catalog.url_for(service_type='identity',
                                               endpoint_type='publicURL')
         return keystone_client.Client(username=user, password=password,
@@ -182,19 +206,49 @@
 
     def authenticate_glance_admin(self, keystone):
         """Authenticates admin user with glance."""
+        self.log.debug('Authenticating glance admin...')
         ep = keystone.service_catalog.url_for(service_type='image',
                                               endpoint_type='adminURL')
         return glance_client.Client(ep, token=keystone.auth_token)
 
+    def authenticate_heat_admin(self, keystone):
+        """Authenticates the admin user with heat."""
+        self.log.debug('Authenticating heat admin...')
+        ep = keystone.service_catalog.url_for(service_type='orchestration',
+                                              endpoint_type='publicURL')
+        return heat_client.Client(endpoint=ep, token=keystone.auth_token)
+
     def authenticate_nova_user(self, keystone, user, password, tenant):
         """Authenticates a regular user with nova-api."""
+        self.log.debug('Authenticating nova user ({})...'.format(user))
         ep = keystone.service_catalog.url_for(service_type='identity',
                                               endpoint_type='publicURL')
         return nova_client.Client(username=user, api_key=password,
                                   project_id=tenant, auth_url=ep)
 
+    def authenticate_swift_user(self, keystone, user, password, tenant):
+        """Authenticates a regular user with swift api."""
+        self.log.debug('Authenticating swift user ({})...'.format(user))
+        ep = keystone.service_catalog.url_for(service_type='identity',
+                                              endpoint_type='publicURL')
+        return swiftclient.Connection(authurl=ep,
+                                      user=user,
+                                      key=password,
+                                      tenant_name=tenant,
+                                      auth_version='2.0')
+
     def create_cirros_image(self, glance, image_name):
-        """Download the latest cirros image and upload it to glance."""
+        """Download the latest cirros image and upload it to glance,
+        validate and return a resource pointer.
+
+        :param glance: pointer to authenticated glance connection
+        :param image_name: display name for new image
+        :returns: glance image pointer
+        """
+        self.log.debug('Creating glance cirros image '
+                       '({})...'.format(image_name))
+
+        # Download cirros image
         http_proxy = os.getenv('AMULET_HTTP_PROXY')
         self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
         if http_proxy:
@@ -203,57 +257,67 @@
         else:
             opener = urllib.FancyURLopener()
 
-        f = opener.open("http://download.cirros-cloud.net/version/released")
+        f = opener.open('http://download.cirros-cloud.net/version/released')
         version = f.read().strip()
-        cirros_img = "cirros-{}-x86_64-disk.img".format(version)
+        cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
         local_path = os.path.join('tests', cirros_img)
 
         if not os.path.exists(local_path):
-            cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net",
+            cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
                                                   version, cirros_img)
             opener.retrieve(cirros_url, local_path)
         f.close()
 
+        # Create glance image
         with open(local_path) as f:
             image = glance.images.create(name=image_name, is_public=True,
                                          disk_format='qcow2',
                                          container_format='bare', data=f)
-        count = 1
-        status = image.status
-        while status != 'active' and count < 10:
-            time.sleep(3)
-            image = glance.images.get(image.id)
-            status = image.status
-            self.log.debug('image status: {}'.format(status))
-            count += 1
-
-        if status != 'active':
-            self.log.error('image creation timed out')
-            return None
+
+        # Wait for image to reach active status
+        img_id = image.id
+        ret = self.resource_reaches_status(glance.images, img_id,
+                                           expected_stat='active',
+                                           msg='Image status wait')
+        if not ret:
+            msg = 'Glance image failed to reach expected state.'
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Re-validate new image
+        self.log.debug('Validating image attributes...')
+        val_img_name = glance.images.get(img_id).name
+        val_img_stat = glance.images.get(img_id).status
+        val_img_pub = glance.images.get(img_id).is_public
+        val_img_cfmt = glance.images.get(img_id).container_format
+        val_img_dfmt = glance.images.get(img_id).disk_format
+        msg_attr = ('Image attributes - name:{} public:{} id:{} stat:{} '
+                    'container fmt:{} disk fmt:{}'.format(
+                        val_img_name, val_img_pub, img_id,
+                        val_img_stat, val_img_cfmt, val_img_dfmt))
+
+        if val_img_name == image_name and val_img_stat == 'active' \
+                and val_img_pub is True and val_img_cfmt == 'bare' \
+                and val_img_dfmt == 'qcow2':
+            self.log.debug(msg_attr)
+        else:
+            msg = ('Volume validation failed, {}'.format(msg_attr))
+            amulet.raise_status(amulet.FAIL, msg=msg)
 
         return image
 
     def delete_image(self, glance, image):
         """Delete the specified image."""
-        num_before = len(list(glance.images.list()))
-        glance.images.delete(image)
-
-        count = 1
-        num_after = len(list(glance.images.list()))
-        while num_after != (num_before - 1) and count < 10:
-            time.sleep(3)
-            num_after = len(list(glance.images.list()))
-            self.log.debug('number of images: {}'.format(num_after))
-            count += 1
-
-        if num_after != (num_before - 1):
-            self.log.error('image deletion timed out')
-            return False
-
-        return True
+
+        # /!\ DEPRECATION WARNING
+        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+                      'delete_resource instead of delete_image.')
+        self.log.debug('Deleting glance image ({})...'.format(image))
+        return self.delete_resource(glance.images, image, msg='glance image')
 
     def create_instance(self, nova, image_name, instance_name, flavor):
         """Create the specified instance."""
+        self.log.debug('Creating instance '
+                       '({}|{}|{})'.format(instance_name, image_name, flavor))
         image = nova.images.find(name=image_name)
         flavor = nova.flavors.find(name=flavor)
         instance = nova.servers.create(name=instance_name, image=image,
@@ -276,19 +340,265 @@
 
     def delete_instance(self, nova, instance):
         """Delete the specified instance."""
-        num_before = len(list(nova.servers.list()))
-        nova.servers.delete(instance)
-
-        count = 1
-        num_after = len(list(nova.servers.list()))
-        while num_after != (num_before - 1) and count < 10:
-            time.sleep(3)
-            num_after = len(list(nova.servers.list()))
-            self.log.debug('number of instances: {}'.format(num_after))
-            count += 1
-
-        if num_after != (num_before - 1):
-            self.log.error('instance deletion timed out')
-            return False
-
-        return True
+
+        # /!\ DEPRECATION WARNING
+        self.log.warn('/!\\ DEPRECATION WARNING:  use '
+                      'delete_resource instead of delete_instance.')
+        self.log.debug('Deleting instance ({})...'.format(instance))
+        return self.delete_resource(nova.servers, instance,
+                                    msg='nova instance')
+
+    def create_or_get_keypair(self, nova, keypair_name="testkey"):
+        """Create a new keypair, or return pointer if it already exists."""
+        try:
+            _keypair = nova.keypairs.get(keypair_name)
+            self.log.debug('Keypair ({}) already exists, '
+                           'using it.'.format(keypair_name))
+            return _keypair
+        except:
+            self.log.debug('Keypair ({}) does not exist, '
+                           'creating it.'.format(keypair_name))
+
+        _keypair = nova.keypairs.create(name=keypair_name)
+        return _keypair
+
+    def create_cinder_volume(self, cinder, vol_name="demo-vol", vol_size=1,
+                             img_id=None, src_vol_id=None, snap_id=None):
+        """Create cinder volume, optionally from a glance image, OR
+        optionally as a clone of an existing volume, OR optionally
+        from a snapshot.  Wait for the new volume status to reach
+        the expected status, validate and return a resource pointer.
+
+        :param vol_name: cinder volume display name
+        :param vol_size: size in gigabytes
+        :param img_id: optional glance image id
+        :param src_vol_id: optional source volume id to clone
+        :param snap_id: optional snapshot id to use
+        :returns: cinder volume pointer
+        """
+        # Handle parameter input and avoid impossible combinations
+        if img_id and not src_vol_id and not snap_id:
+            # Create volume from image
+            self.log.debug('Creating cinder volume from glance image...')
+            bootable = 'true'
+        elif src_vol_id and not img_id and not snap_id:
+            # Clone an existing volume
+            self.log.debug('Cloning cinder volume...')
+            bootable = cinder.volumes.get(src_vol_id).bootable
+        elif snap_id and not src_vol_id and not img_id:
+            # Create volume from snapshot
+            self.log.debug('Creating cinder volume from snapshot...')
+            snap = cinder.volume_snapshots.find(id=snap_id)
+            vol_size = snap.size
+            snap_vol_id = cinder.volume_snapshots.get(snap_id).volume_id
+            bootable = cinder.volumes.get(snap_vol_id).bootable
+        elif not img_id and not src_vol_id and not snap_id:
+            # Create volume
+            self.log.debug('Creating cinder volume...')
+            bootable = 'false'
+        else:
+            # Impossible combination of parameters
+            msg = ('Invalid method use - name:{} size:{} img_id:{} '
+                   'src_vol_id:{} snap_id:{}'.format(vol_name, vol_size,
+                                                     img_id, src_vol_id,
+                                                     snap_id))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Create new volume
+        try:
+            vol_new = cinder.volumes.create(display_name=vol_name,
+                                            imageRef=img_id,
+                                            size=vol_size,
+                                            source_volid=src_vol_id,
+                                            snapshot_id=snap_id)
+            vol_id = vol_new.id
+        except Exception as e:
+            msg = 'Failed to create volume: {}'.format(e)
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Wait for volume to reach available status
+        ret = self.resource_reaches_status(cinder.volumes, vol_id,
+                                           expected_stat="available",
+                                           msg="Volume status wait")
+        if not ret:
+            msg = 'Cinder volume failed to reach expected state.'
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Re-validate new volume
+        self.log.debug('Validating volume attributes...')
+        val_vol_name = cinder.volumes.get(vol_id).display_name
+        val_vol_boot = cinder.volumes.get(vol_id).bootable
+        val_vol_stat = cinder.volumes.get(vol_id).status
+        val_vol_size = cinder.volumes.get(vol_id).size
+        msg_attr = ('Volume attributes - name:{} id:{} stat:{} boot:'
+                    '{} size:{}'.format(val_vol_name, vol_id,
+                                        val_vol_stat, val_vol_boot,
+                                        val_vol_size))
+
+        if val_vol_boot == bootable and val_vol_stat == 'available' \
+                and val_vol_name == vol_name and val_vol_size == vol_size:
+            self.log.debug(msg_attr)
+        else:
+            msg = ('Volume validation failed, {}'.format(msg_attr))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        return vol_new
+
+    def delete_resource(self, resource, resource_id,
+                        msg="resource", max_wait=120):
+        """Delete one openstack resource, such as one instance, keypair,
+        image, volume, stack, etc., and confirm deletion within max wait time.
+
+        :param resource: pointer to os resource type, ex:glance_client.images
+        :param resource_id: unique name or id for the openstack resource
+        :param msg: text to identify purpose in logging
+        :param max_wait: maximum wait time in seconds
+        :returns: True if successful, otherwise False
+        """
+        self.log.debug('Deleting OpenStack resource '
+                       '{} ({})'.format(resource_id, msg))
+        num_before = len(list(resource.list()))
+        resource.delete(resource_id)
+
+        tries = 0
+        num_after = len(list(resource.list()))
+        while num_after != (num_before - 1) and tries < (max_wait / 4):
+            self.log.debug('{} delete check: '
+                           '{} [{}:{}] {}'.format(msg, tries,
+                                                  num_before,
+                                                  num_after,
+                                                  resource_id))
+            time.sleep(4)
+            num_after = len(list(resource.list()))
+            tries += 1
+
+        self.log.debug('{}:  expected, actual count = {}, '
+                       '{}'.format(msg, num_before - 1, num_after))
+
+        if num_after == (num_before - 1):
+            return True
+        else:
+            self.log.error('{} delete timed out'.format(msg))
+            return False
+
+    def resource_reaches_status(self, resource, resource_id,
+                                expected_stat='available',
+                                msg='resource', max_wait=120):
+        """Wait for an openstack resources status to reach an
+           expected status within a specified time.  Useful to confirm that
+           nova instances, cinder vols, snapshots, glance images, heat stacks
+           and other resources eventually reach the expected status.
+
+        :param resource: pointer to os resource type, ex: heat_client.stacks
+        :param resource_id: unique id for the openstack resource
+        :param expected_stat: status to expect resource to reach
+        :param msg: text to identify purpose in logging
+        :param max_wait: maximum wait time in seconds
+        :returns: True if successful, False if status is not reached
+        """
+
+        tries = 0
+        resource_stat = resource.get(resource_id).status
+        while resource_stat != expected_stat and tries < (max_wait / 4):
+            self.log.debug('{} status check: '
+                           '{} [{}:{}] {}'.format(msg, tries,
+                                                  resource_stat,
+                                                  expected_stat,
+                                                  resource_id))
+            time.sleep(4)
+            resource_stat = resource.get(resource_id).status
+            tries += 1
+
+        self.log.debug('{}:  expected, actual status = {}, '
+                       '{}'.format(msg, resource_stat, expected_stat))
+
+        if resource_stat == expected_stat:
+            return True
+        else:
+            self.log.debug('{} never reached expected status: '
+                           '{}'.format(resource_id, expected_stat))
+            return False
+
+    def get_ceph_osd_id_cmd(self, index):
+        """Produce a shell command that will return a ceph-osd id."""
+        return ("`initctl list | grep 'ceph-osd ' | "
+                "awk 'NR=={} {{ print $2 }}' | "
+                "grep -o '[0-9]*'`".format(index + 1))
+
+    def get_ceph_pools(self, sentry_unit):
+        """Return a dict of ceph pools from a single ceph unit, with
+        pool name as keys, pool id as vals."""
+        pools = {}
+        cmd = 'sudo ceph osd lspools'
+        output, code = sentry_unit.run(cmd)
+        if code != 0:
+            msg = ('{} `{}` returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, code, output))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        # Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance,
+        for pool in str(output).split(','):
+            pool_id_name = pool.split(' ')
+            if len(pool_id_name) == 2:
+                pool_id = pool_id_name[0]
+                pool_name = pool_id_name[1]
+                pools[pool_name] = int(pool_id)
+
+        self.log.debug('Pools on {}: {}'.format(sentry_unit.info['unit_name'],
+                                                pools))
+        return pools
+
+    def get_ceph_df(self, sentry_unit):
+        """Return dict of ceph df json output, including ceph pool state.
+
+        :param sentry_unit: Pointer to amulet sentry instance (juju unit)
+        :returns: Dict of ceph df output
+        """
+        cmd = 'sudo ceph df --format=json'
+        output, code = sentry_unit.run(cmd)
+        if code != 0:
+            msg = ('{} `{}` returned {} '
+                   '{}'.format(sentry_unit.info['unit_name'],
+                               cmd, code, output))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+        return json.loads(output)
+
+    def get_ceph_pool_sample(self, sentry_unit, pool_id=0):
+        """Take a sample of attributes of a ceph pool, returning ceph
+        pool name, object count and disk space used for the specified
+        pool ID number.
+
+        :param sentry_unit: Pointer to amulet sentry instance (juju unit)
+        :param pool_id: Ceph pool ID
+        :returns: List of pool name, object count, kb disk space used
+        """
+        df = self.get_ceph_df(sentry_unit)
+        pool_name = df['pools'][pool_id]['name']
+        obj_count = df['pools'][pool_id]['stats']['objects']
+        kb_used = df['pools'][pool_id]['stats']['kb_used']
+        self.log.debug('Ceph {} pool (ID {}): {} objects, '
+                       '{} kb used'.format(pool_name, pool_id,
+                                           obj_count, kb_used))
+        return pool_name, obj_count, kb_used
+
+    def validate_ceph_pool_samples(self, samples, sample_type="resource pool"):
+        """Validate ceph pool samples taken over time, such as pool
+        object counts or pool kb used, before adding, after adding, and
+        after deleting items which affect those pool attributes.  The
+        2nd element is expected to be greater than the 1st; 3rd is expected