Merge lp:~billy-olsen/charms/trusty/nova-cloud-controller/public-endpoint-host into lp:~openstack-charmers-archive/charms/trusty/nova-cloud-controller/next

[Billy Olsen]

Provides a config option which allows the user to specify the public hostname used to advertise to keystone when creating endpoints.

Note: this branch includes the charm-helpers change found in merge proposal https://code.launchpad.net/~billy-olsen/charm-helpers/better-address-endpoint-overrides/+merge/261000 which will need to be re-synced when that proposal is merged.

This is to get feedback for the change itself.

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #5044 nova-cloud-controller-next for billy-olsen mp261006
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/5044/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #4724 nova-cloud-controller-next for billy-olsen mp261006
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/4724/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #4450 nova-cloud-controller-next for billy-olsen mp261006
    AMULET OK: passed

Build: http://10.245.162.77:8080/job/charm_amulet_test/4450/

[lp:~billy-olsen/charms/trusty/nova-cloud-controller/public-endpoint-host]

165. By Billy Olsen on 2015-06-04

Merge with /next

166. By Billy Olsen on 2015-06-04

c-h sync

167. By Billy Olsen on 2015-06-04

Fix unit tests broken with c-h sync

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #5068 nova-cloud-controller-next for billy-olsen mp261006
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/5068/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #4747 nova-cloud-controller-next for billy-olsen mp261006
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/4747/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #4475 nova-cloud-controller-next for billy-olsen mp261006
    AMULET OK: passed

Build: http://10.245.162.77:8080/job/charm_amulet_test/4475/

[Corey Bryant (corey.bryant)]

review: Approve

=== modified file 'config.yaml'
--- config.yaml	2015-05-18 19:07:48 +0000
+++ config.yaml	2015-06-04 23:33:37 +0000
@@ -230,6 +230,17 @@
       192.168.0.0/24)
       .
       This network will be used for public endpoints.
+  os-public-hostname:
+    type: string
+    default:
+    description: |
+      The hostname or address of the public endpoints provided by the
+      nova-cloud-controller in the keystone identity provider.
+      .
+      This value will be used for public endpoints. For example, an
+      os-public-hostname set to 'ncc.example.com' with ssl enabled will
+      create public endpoints such as 
+      https://ncc.example.com:8775/v2/$(tenant_id)s
   service-guard:
     type: boolean
     default: false

=== modified file 'hooks/charmhelpers/contrib/hahelpers/cluster.py'
--- hooks/charmhelpers/contrib/hahelpers/cluster.py	2015-03-16 14:17:04 +0000
+++ hooks/charmhelpers/contrib/hahelpers/cluster.py	2015-06-04 23:33:37 +0000
@@ -52,6 +52,8 @@
     bool_from_string,
 )
 
+DC_RESOURCE_NAME = 'DC'
+
 
 class HAIncompleteConfig(Exception):
     pass
@@ -95,6 +97,27 @@
     return False
 
 
+def is_crm_dc():
+    """
+    Determine leadership by querying the pacemaker Designated Controller
+    """
+    cmd = ['crm', 'status']
+    try:
+        status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+        if not isinstance(status, six.text_type):
+            status = six.text_type(status, "utf-8")
+    except subprocess.CalledProcessError:
+        return False
+    current_dc = ''
+    for line in status.split('\n'):
+        if line.startswith('Current DC'):
+            # Current DC: juju-lytrusty-machine-2 (168108163) - partition with quorum
+            current_dc = line.split(':')[1].split()[0]
+    if current_dc == get_unit_hostname():
+        return True
+    return False
+
+
 @retry_on_exception(5, base_delay=2, exc_type=CRMResourceNotFound)
 def is_crm_leader(resource, retry=False):
     """
@@ -104,6 +127,8 @@
     We allow this operation to be retried to avoid the possibility of getting a
     false negative. See LP #1396246 for more info.
     """
+    if resource == DC_RESOURCE_NAME:
+        return is_crm_dc()
     cmd = ['crm', 'resource', 'show', resource]
     try:
         status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)

=== modified file 'hooks/charmhelpers/contrib/openstack/ip.py'
--- hooks/charmhelpers/contrib/openstack/ip.py	2015-03-31 14:56:11 +0000
+++ hooks/charmhelpers/contrib/openstack/ip.py	2015-06-04 23:33:37 +0000
@@ -17,6 +17,7 @@
 from charmhelpers.core.hookenv import (
     config,
     unit_get,
+    service_name,
 )
 from charmhelpers.contrib.network.ip import (
     get_address_in_network,
@@ -26,8 +27,6 @@
 )
 from charmhelpers.contrib.hahelpers.cluster import is_clustered
 
-from functools import partial
-
 PUBLIC = 'public'
 INTERNAL = 'int'
 ADMIN = 'admin'
@@ -35,15 +34,18 @@
 ADDRESS_MAP = {
     PUBLIC: {
         'config': 'os-public-network',
-        'fallback': 'public-address'
+        'fallback': 'public-address',
+        'override': 'os-public-hostname',
     },
     INTERNAL: {
         'config': 'os-internal-network',
-        'fallback': 'private-address'
+        'fallback': 'private-address',
+        'override': 'os-internal-hostname',
     },
     ADMIN: {
         'config': 'os-admin-network',
-        'fallback': 'private-address'
+        'fallback': 'private-address',
+        'override': 'os-admin-hostname',
     }
 }
 
@@ -57,15 +59,50 @@
     :param endpoint_type: str endpoint type to resolve.
     :param returns: str base URL for services on the current service unit.
     """
-    scheme = 'http'
-    if 'https' in configs.complete_contexts():
-        scheme = 'https'
+    scheme = _get_scheme(configs)
+
     address = resolve_address(endpoint_type)
     if is_ipv6(address):
         address = "[{}]".format(address)
+
     return '%s://%s' % (scheme, address)
 
 
+def _get_scheme(configs):
+    """Returns the scheme to use for the url (either http or https)
+    depending upon whether https is in the configs value.
+
+    :param configs: OSTemplateRenderer config templating object to inspect
+                    for a complete https context.
+    :returns: either 'http' or 'https' depending on whether https is
+              configured within the configs context.
+    """
+    scheme = 'http'
+    if configs and 'https' in configs.complete_contexts():
+        scheme = 'https'
+    return scheme
+
+
+def _get_address_override(endpoint_type=PUBLIC):
+    """Returns any address overrides that the user has defined based on the
+    endpoint type.
+
+    Note: this function allows for the service name to be inserted into the
+    address if the user specifies {service_name}.somehost.org.
+
+    :param endpoint_type: the type of endpoint to retrieve the override
+                          value for.
+    :returns: any endpoint address or hostname that the user has overridden
+              or None if an override is not present.
+    """
+    override_key = ADDRESS_MAP[endpoint_type]['override']
+    addr_override = config(override_key)
+    if not addr_override:
+        return None
+    else:
+        return addr_override.format(service_name=service_name())
+
+
 def resolve_address(endpoint_type=PUBLIC):
     """Return unit address depending on net config.
 
@@ -77,7 +114,10 @@
 
     :param endpoint_type: Network endpoing type
     """
-    resolved_address = None
+    resolved_address = _get_address_override(endpoint_type)
+    if resolved_address:
+        return resolved_address
+
     vips = config('vip')
     if vips:
         vips = vips.split()
@@ -109,38 +149,3 @@
                          "clustered=%s)" % (net_type, clustered))
 
     return resolved_address
-
-
-def endpoint_url(configs, url_template, port, endpoint_type=PUBLIC,
-                 override=None):
-    """Returns the correct endpoint URL to advertise to Keystone.
-
-    This method provides the correct endpoint URL which should be advertised to
-    the keystone charm for endpoint creation. This method allows for the url to
-    be overridden to force a keystone endpoint to have specific URL for any of
-    the defined scopes (admin, internal, public).
-
-    :param configs: OSTemplateRenderer config templating object to inspect
-                    for a complete https context.
-    :param url_template: str format string for creating the url template. Only
-                         two values will be passed - the scheme+hostname
-                        returned by the canonical_url and the port.
-    :param endpoint_type: str endpoint type to resolve.
-    :param override: str the name of the config option which overrides the
-                     endpoint URL defined by the charm itself. None will
-                     disable any overrides (default).
-    """
-    if override:
-        # Return any user-defined overrides for the keystone endpoint URL.
-        user_value = config(override)
-        if user_value:
-            return user_value.strip()
-
-    return url_template % (canonical_url(configs, endpoint_type), port)
-
-
-public_endpoint = partial(endpoint_url, endpoint_type=PUBLIC)
-
-internal_endpoint = partial(endpoint_url, endpoint_type=INTERNAL)
-
-admin_endpoint = partial(endpoint_url, endpoint_type=ADMIN)

=== modified file 'hooks/charmhelpers/contrib/openstack/neutron.py'
--- hooks/charmhelpers/contrib/openstack/neutron.py	2015-04-13 08:51:41 +0000
+++ hooks/charmhelpers/contrib/openstack/neutron.py	2015-06-04 23:33:37 +0000
@@ -256,11 +256,14 @@
 def parse_mappings(mappings):
     parsed = {}
     if mappings:
-        mappings = mappings.split(' ')
+        mappings = mappings.split()
         for m in mappings:
             p = m.partition(':')
-            if p[1] == ':':
-                parsed[p[0].strip()] = p[2].strip()
+            key = p[0].strip()
+            if p[1]:
+                parsed[key] = p[2].strip()
+            else:
+                parsed[key] = ''
 
     return parsed
 
@@ -283,13 +286,13 @@
     Returns dict of the form {bridge:port}.
     """
     _mappings = parse_mappings(mappings)
-    if not _mappings:
+    if not _mappings or list(_mappings.values()) == ['']:
         if not mappings:
             return {}
 
         # For backwards-compatibility we need to support port-only provided in
         # config.
-        _mappings = {default_bridge: mappings.split(' ')[0]}
+        _mappings = {default_bridge: mappings.split()[0]}
 
     bridges = _mappings.keys()
     ports = _mappings.values()
@@ -309,6 +312,8 @@
 
     Mappings must be a space-delimited list of provider:start:end mappings.
 
+    The start:end range is optional and may be omitted.
+
     Returns dict of the form {provider: (start, end)}.
     """
     _mappings = parse_mappings(mappings)

=== modified file 'hooks/charmhelpers/contrib/openstack/utils.py'
--- hooks/charmhelpers/contrib/openstack/utils.py	2015-04-20 08:39:44 +0000
+++ hooks/charmhelpers/contrib/openstack/utils.py	2015-06-04 23:33:37 +0000
@@ -53,9 +53,13 @@
     get_ipv6_addr
 )
 
+from charmhelpers.contrib.python.packages import (
+    pip_create_virtualenv,
+    pip_install,
+)
+
 from charmhelpers.core.host import lsb_release, mounts, umount
 from charmhelpers.fetch import apt_install, apt_cache, install_remote
-from charmhelpers.contrib.python.packages import pip_install
 from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
 from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
 
@@ -497,7 +501,17 @@
 requirements_dir = None
 
 
-def git_clone_and_install(projects_yaml, core_project):
+def _git_yaml_load(projects_yaml):
+    """
+    Load the specified yaml into a dictionary.
+    """
+    if not projects_yaml:
+        return None
+
+    return yaml.load(projects_yaml)
+
+
+def git_clone_and_install(projects_yaml, core_project, depth=1):
     """
     Clone/install all specified OpenStack repositories.
 
@@ -510,23 +524,22 @@
              repository: 'git://git.openstack.org/openstack/requirements.git',
              branch: 'stable/icehouse'}
         directory: /mnt/openstack-git
-        http_proxy: http://squid.internal:3128
-        https_proxy: https://squid.internal:3128
+        http_proxy: squid-proxy-url
+        https_proxy: squid-proxy-url
 
         The directory, http_proxy, and https_proxy keys are optional.
     """
     global requirements_dir
     parent_dir = '/mnt/openstack-git'
-
-    if not projects_yaml:
-        return
-
-    projects = yaml.load(projects_yaml)
+    http_proxy = None
+
+    projects = _git_yaml_load(projects_yaml)
     _git_validate_projects_yaml(projects, core_project)
 
     old_environ = dict(os.environ)
 
     if 'http_proxy' in projects.keys():
+        http_proxy = projects['http_proxy']
         os.environ['http_proxy'] = projects['http_proxy']
     if 'https_proxy' in projects.keys():
         os.environ['https_proxy'] = projects['https_proxy']
@@ -534,15 +547,19 @@
     if 'directory' in projects.keys():
         parent_dir = projects['directory']
 
+    pip_create_virtualenv(os.path.join(parent_dir, 'venv'))
+
     for p in projects['repositories']:
         repo = p['repository']
         branch = p['branch']
         if p['name'] == 'requirements':
-            repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
+            repo_dir = _git_clone_and_install_single(repo, branch, depth,
+                                                     parent_dir, http_proxy,
                                                      update_requirements=False)
             requirements_dir = repo_dir
         else:
-            repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
+            repo_dir = _git_clone_and_install_single(repo, branch, depth,
+                                                     parent_dir, http_proxy,
                                                      update_requirements=True)
 
     os.environ = old_environ
@@ -574,7 +591,8 @@
         error_out('openstack-origin-git key \'{}\' is missing'.format(key))
 
 
-def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements):
+def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy,
+                                  update_requirements):
     """
     Clone and install a single git repository.
     """
@@ -587,7 +605,8 @@
 
     if not os.path.exists(dest_dir):
         juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
-        repo_dir = install_remote(repo, dest=parent_dir, branch=branch)
+        repo_dir = install_remote(repo, dest=parent_dir, branch=branch,
+                                  depth=depth)
     else:
         repo_dir = dest_dir
 
@@ -598,7 +617,12 @@
         _git_update_requirements(repo_dir, requirements_dir)
 
     juju_log('Installing git repo from dir: {}'.format(repo_dir))
-    pip_install(repo_dir)
+    if http_proxy:
+        pip_install(repo_dir, proxy=http_proxy,
+                    venv=os.path.join(parent_dir, 'venv'))
+    else:
+        pip_install(repo_dir,
+                    venv=os.path.join(parent_dir, 'venv'))
 
     return repo_dir
 
@@ -621,16 +645,27 @@
     os.chdir(orig_dir)
 
 
+def git_pip_venv_dir(projects_yaml):
+    """
+    Return the pip virtualenv path.
+    """
+    parent_dir = '/mnt/openstack-git'
+
+    projects = _git_yaml_load(projects_yaml)
+
+    if 'directory' in projects.keys():
+        parent_dir = projects['directory']
+
+    return os.path.join(parent_dir, 'venv')
+
+
 def git_src_dir(projects_yaml, project):
     """
     Return the directory where the specified project's source is located.
     """
     parent_dir = '/mnt/openstack-git'
 
-    if not projects_yaml:
-        return
-
-    projects = yaml.load(projects_yaml)
+    projects = _git_yaml_load(projects_yaml)
 
     if 'directory' in projects.keys():
         parent_dir = projects['directory']
@@ -640,3 +675,15 @@
             return os.path.join(parent_dir, os.path.basename(p['repository']))
 
     return None
+
+
+def git_yaml_value(projects_yaml, key):
+    """
+    Return the value in projects_yaml for the specified key.
+    """
+    projects = _git_yaml_load(projects_yaml)
+
+    if key in projects.keys():
+        return projects[key]
+
+    return None

=== modified file 'hooks/charmhelpers/contrib/peerstorage/__init__.py'
--- hooks/charmhelpers/contrib/peerstorage/__init__.py	2015-03-31 11:39:19 +0000
+++ hooks/charmhelpers/contrib/peerstorage/__init__.py	2015-06-04 23:33:37 +0000
@@ -73,6 +73,8 @@
     exc_list = exc_list if exc_list else []
     peerdb_settings = peer_retrieve('-', relation_name=relation_name)
     matched = {}
+    if peerdb_settings is None:
+        return matched
     for k, v in peerdb_settings.items():
         full_prefix = prefix + delimiter
         if k.startswith(full_prefix):

=== modified file 'hooks/charmhelpers/contrib/python/packages.py'
--- hooks/charmhelpers/contrib/python/packages.py	2015-03-13 13:01:00 +0000
+++ hooks/charmhelpers/contrib/python/packages.py	2015-06-04 23:33:37 +0000
@@ -17,8 +17,11 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
+import subprocess
+
 from charmhelpers.fetch import apt_install, apt_update
-from charmhelpers.core.hookenv import log
+from charmhelpers.core.hookenv import charm_dir, log
 
 try:
     from pip import main as pip_execute
@@ -51,11 +54,15 @@
     pip_execute(command)
 
 
-def pip_install(package, fatal=False, upgrade=False, **options):
+def pip_install(package, fatal=False, upgrade=False, venv=None, **options):
     """Install a python package"""
-    command = ["install"]
+    if venv:
+        venv_python = os.path.join(venv, 'bin/pip')
+        command = [venv_python, "install"]
+    else:
+        command = ["install"]
 
-    available_options = ('proxy', 'src', 'log', "index-url", )
+    available_options = ('proxy', 'src', 'log', 'index-url', )
     for option in parse_options(options, available_options):
         command.append(option)
 
@@ -69,7 +76,10 @@
 
     log("Installing {} package with options: {}".format(package,
                                                         command))
-    pip_execute(command)
+    if venv:
+        subprocess.check_call(command)
+    else:
+        pip_execute(command)
 
 
 def pip_uninstall(package, **options):
@@ -94,3 +104,16 @@
     """Returns the list of current python installed packages
     """
     return pip_execute(["list"])
+
+
+def pip_create_virtualenv(path=None):
+    """Create an isolated Python environment."""
+    apt_install('python-virtualenv')
+
+    if path:
+        venv_path = path
+    else:
+        venv_path = os.path.join(charm_dir(), 'venv')
+
+    if not os.path.exists(venv_path):
+        subprocess.check_call(['virtualenv', venv_path])

=== modified file 'hooks/charmhelpers/core/hookenv.py'
--- hooks/charmhelpers/core/hookenv.py	2015-04-13 08:51:41 +0000
+++ hooks/charmhelpers/core/hookenv.py	2015-06-04 23:33:37 +0000
@@ -21,12 +21,14 @@
 #  Charm Helpers Developers <[e-mail]>
 
 from __future__ import print_function
+from functools import wraps
 import os
 import json
 import yaml
 import subprocess
 import sys
 import errno
+import tempfile
 from subprocess import CalledProcessError
 
 import six
@@ -58,15 +60,17 @@
 
     will cache the result of unit_get + 'test' for future calls.
     """
+    @wraps(func)
     def wrapper(*args, **kwargs):
         global cache
         key = str((func, args, kwargs))
         try:
             return cache[key]
         except KeyError:
-            res = func(*args, **kwargs)
-            cache[key] = res
-            return res
+            pass  # Drop out of the exception handler scope.
+        res = func(*args, **kwargs)
+        cache[key] = res
+        return res
     return wrapper
 
 
@@ -178,7 +182,7 @@
 
 def remote_unit():
     """The remote unit for the current relation hook"""
-    return os.environ['JUJU_REMOTE_UNIT']
+    return os.environ.get('JUJU_REMOTE_UNIT', None)
 
 
 def service_name():
@@ -250,6 +254,12 @@
         except KeyError:
             return (self._prev_dict or {})[key]
 
+    def get(self, key, default=None):
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
     def keys(self):
         prev_keys = []
         if self._prev_dict is not None:
@@ -353,18 +363,49 @@
     """Set relation information for the current unit"""
     relation_settings = relation_settings if relation_settings else {}
     relation_cmd_line = ['relation-set']
+    accepts_file = "--file" in subprocess.check_output(
+        relation_cmd_line + ["--help"], universal_newlines=True)
     if relation_id is not None:
         relation_cmd_line.extend(('-r', relation_id))
-    for k, v in (list(relation_settings.items()) + list(kwargs.items())):
-        if v is None:
-            relation_cmd_line.append('{}='.format(k))
-        else:
-            relation_cmd_line.append('{}={}'.format(k, v))
-    subprocess.check_call(relation_cmd_line)
+    settings = relation_settings.copy()
+    settings.update(kwargs)
+    for key, value in settings.items():
+        # Force value to be a string: it always should, but some call
+        # sites pass in things like dicts or numbers.
+        if value is not None:
+            settings[key] = "{}".format(value)
+    if accepts_file:
+        # --file was introduced in Juju 1.23.2. Use it by default if
+        # available, since otherwise we'll break if the relation data is
+        # too big. Ideally we should tell relation-set to read the data from
+        # stdin, but that feature is broken in 1.23.2: Bug #1454678.
+        with tempfile.NamedTemporaryFile(delete=False) as settings_file:
+            settings_file.write(yaml.safe_dump(settings).encode("utf-8"))
+        subprocess.check_call(
+            relation_cmd_line + ["--file", settings_file.name])
+        os.remove(settings_file.name)
+    else:
+        for key, value in settings.items():
+            if value is None:
+                relation_cmd_line.append('{}='.format(key))
+            else:
+                relation_cmd_line.append('{}={}'.format(key, value))
+        subprocess.check_call(relation_cmd_line)
     # Flush cache of any relation-gets for local unit
     flush(local_unit())
 
 
+def relation_clear(r_id=None):
+    ''' Clears any relation data already set on relation r_id '''
+    settings = relation_get(rid=r_id,
+                            unit=local_unit())
+    for setting in settings:
+        if setting not in ['public-address', 'private-address']:
+            settings[setting] = None
+    relation_set(relation_id=r_id,
+                 **settings)
+
+
 @cached
 def relation_ids(reltype=None):
     """A list of relation_ids"""
@@ -509,6 +550,11 @@
         return None
 
 
+def unit_public_ip():
+    """Get this unit's public IP address"""
+    return unit_get('public-address')
+
+
 def unit_private_ip():
     """Get this unit's private IP address"""
     return unit_get('private-address')
@@ -605,3 +651,94 @@
 
     The results set by action_set are preserved."""
     subprocess.check_call(['action-fail', message])
+
+
+def status_set(workload_state, message):
+    """Set the workload state with a message
+
+    Use status-set to set the workload state with a message which is visible
+    to the user via juju status. If the status-set command is not found then
+    assume this is juju < 1.23 and juju-log the message unstead.
+
+    workload_state -- valid juju workload state.
+    message        -- status update message
+    """
+    valid_states = ['maintenance', 'blocked', 'waiting', 'active']
+    if workload_state not in valid_states:
+        raise ValueError(
+            '{!r} is not a valid workload state'.format(workload_state)
+        )
+    cmd = ['status-set', workload_state, message]
+    try:
+        ret = subprocess.call(cmd)
+        if ret == 0:
+            return
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
+    log_message = 'status-set failed: {} {}'.format(workload_state,
+                                                    message)
+    log(log_message, level='INFO')
+
+
+def status_get():
+    """Retrieve the previously set juju workload state
+
+    If the status-set command is not found then assume this is juju < 1.23 and
+    return 'unknown'
+    """
+    cmd = ['status-get']
+    try:
+        raw_status = subprocess.check_output(cmd, universal_newlines=True)
+        status = raw_status.rstrip()
+        return status
+    except OSError as e:
+        if e.errno == errno.ENOENT:
+            return 'unknown'
+        else:
+            raise
+
+
+def translate_exc(from_exc, to_exc):
+    def inner_translate_exc1(f):
+        def inner_translate_exc2(*args, **kwargs):
+            try:
+                return f(*args, **kwargs)
+            except from_exc:
+                raise to_exc
+
+        return inner_translate_exc2
+
+    return inner_translate_exc1
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def is_leader():
+    """Does the current unit hold the juju leadership
+
+    Uses juju to determine whether the current unit is the leader of its peers
+    """
+    cmd = ['is-leader', '--format=json']
+    return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def leader_get(attribute=None):
+    """Juju leader get value(s)"""
+    cmd = ['leader-get', '--format=json'] + [attribute or '-']
+    return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def leader_set(settings=None, **kwargs):
+    """Juju leader set value(s)"""
+    log("Juju leader-set '%s'" % (settings), level=DEBUG)
+    cmd = ['leader-set']
+    settings = settings or {}
+    settings.update(kwargs)
+    for k, v in settings.iteritems():
+        if v is None:
+            cmd.append('{}='.format(k))
+        else:
+            cmd.append('{}={}'.format(k, v))
+    subprocess.check_call(cmd)

=== modified file 'hooks/charmhelpers/core/host.py'
--- hooks/charmhelpers/core/host.py	2015-03-31 14:56:11 +0000
+++ hooks/charmhelpers/core/host.py	2015-06-04 23:33:37 +0000
@@ -90,7 +90,7 @@
             ['service', service_name, 'status'],
             stderr=subprocess.STDOUT).decode('UTF-8')
     except subprocess.CalledProcessError as e:
-        return 'unrecognized service' not in e.output
+        return b'unrecognized service' not in e.output
     else:
         return True
 

=== modified file 'hooks/charmhelpers/core/services/base.py'
--- hooks/charmhelpers/core/services/base.py	2015-03-31 14:56:11 +0000
+++ hooks/charmhelpers/core/services/base.py	2015-06-04 23:33:37 +0000
@@ -15,9 +15,9 @@
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import re
 import json
-from collections import Iterable
+from inspect import getargspec
+from collections import Iterable, OrderedDict
 
 from charmhelpers.core import host
 from charmhelpers.core import hookenv
@@ -119,7 +119,7 @@
         """
         self._ready_file = os.path.join(hookenv.charm_dir(), 'READY-SERVICES.json')
         self._ready = None
-        self.services = {}
+        self.services = OrderedDict()
         for service in services or []:
             service_name = service['service']
             self.services[service_name] = service
@@ -132,8 +132,8 @@
         if hook_name == 'stop':
             self.stop_services()
         else:
+            self.reconfigure_services()
             self.provide_data()
-            self.reconfigure_services()
         cfg = hookenv.config()
         if cfg.implicit_save:
             cfg.save()
@@ -145,15 +145,36 @@
         A provider must have a `name` attribute, which indicates which relation
         to set data on, and a `provide_data()` method, which returns a dict of
         data to set.
+
+        The `provide_data()` method can optionally accept two parameters:
+
+          * ``remote_service`` The name of the remote service that the data will
+            be provided to.  The `provide_data()` method will be called once
+            for each connected service (not unit).  This allows the method to
+            tailor its data to the given service.
+          * ``service_ready`` Whether or not the service definition had all of
+            its requirements met, and thus the ``data_ready`` callbacks run.
+
+        Note that the ``provided_data`` methods are now called **after** the
+        ``data_ready`` callbacks are run.  This gives the ``data_ready`` callbacks
+        a chance to generate any data necessary for the providing to the remote
+        services.
         """
-        hook_name = hookenv.hook_name()
-        for service in self.services.values():
+        for service_name, service in self.services.items():
+            service_ready = self.is_ready(service_name)
             for provider in service.get('provided_data', []):
-                if re.match(r'{}-relation-(joined|changed)'.format(provider.name), hook_name):
-                    data = provider.provide_data()
-                    _ready = provider._is_ready(data) if hasattr(provider, '_is_ready') else data
-                    if _ready:
-                        hookenv.relation_set(None, data)
+                for relid in hookenv.relation_ids(provider.name):
+                    units = hookenv.related_units(relid)
+                    if not units:
+                        continue
+                    remote_service = units[0].split('/')[0]
+                    argspec = getargspec(provider.provide_data)
+                    if len(argspec.args) > 1:
+                        data = provider.provide_data(remote_service, service_ready)
+                    else:
+                        data = provider.provide_data()
+                    if data:
+                        hookenv.relation_set(relid, data)
 
     def reconfigure_services(self, *service_names):
         """

=== modified file 'hooks/charmhelpers/fetch/__init__.py'
--- hooks/charmhelpers/fetch/__init__.py	2015-03-31 14:56:11 +0000
+++ hooks/charmhelpers/fetch/__init__.py	2015-06-04 23:33:37 +0000
@@ -158,7 +158,7 @@
 
 def apt_cache(in_memory=True):
     """Build and return an apt cache"""
-    import apt_pkg
+    from apt import apt_pkg
     apt_pkg.init()
     if in_memory:
         apt_pkg.config.set("Dir::Cache::pkgcache", "")

=== modified file 'hooks/charmhelpers/fetch/giturl.py'
--- hooks/charmhelpers/fetch/giturl.py	2015-03-13 13:01:00 +0000
+++ hooks/charmhelpers/fetch/giturl.py	2015-06-04 23:33:37 +0000
@@ -45,14 +45,16 @@
         else:
             return True
 
-    def clone(self, source, dest, branch):
+    def clone(self, source, dest, branch, depth=None):
         if not self.can_handle(source):
             raise UnhandledSource("Cannot handle {}".format(source))
 
-        repo = Repo.clone_from(source, dest)
-        repo.git.checkout(branch)
+        if depth:
+            Repo.clone_from(source, dest, branch=branch, depth=depth)
+        else:
+            Repo.clone_from(source, dest, branch=branch)
 
-    def install(self, source, branch="master", dest=None):
+    def install(self, source, branch="master", dest=None, depth=None):
         url_parts = self.parse_url(source)
         branch_name = url_parts.path.strip("/").split("/")[-1]
         if dest:
@@ -63,7 +65,7 @@
         if not os.path.exists(dest_dir):
             mkdir(dest_dir, perms=0o755)
         try:
-            self.clone(source, dest_dir, branch)
+            self.clone(source, dest_dir, branch, depth)
         except GitCommandError as e:
             raise UnhandledSource(e.message)
         except OSError as e:

=== modified file 'unit_tests/test_nova_cc_contexts.py'
--- unit_tests/test_nova_cc_contexts.py	2015-04-20 10:21:36 +0000
+++ unit_tests/test_nova_cc_contexts.py	2015-06-04 23:33:37 +0000
@@ -90,11 +90,13 @@
         self.assertEqual({'memcached_servers': "%s:11211" % (formated_ip, )},
                          instance_console())
 
+    @mock.patch('charmhelpers.contrib.openstack.neutron.os_release')
     @mock.patch.object(context, 'use_local_neutron_api')
     @mock.patch('charmhelpers.contrib.openstack.ip.config')
     @mock.patch('charmhelpers.contrib.openstack.ip.is_clustered')
     def test_neutron_context_single_vip(self, mock_is_clustered, mock_config,
-                                        mock_use_local_neutron_api):
+                                        mock_use_local_neutron_api,
+                                        _os_release):
         mock_use_local_neutron_api.return_value = True
         self.https.return_value = False
         mock_is_clustered.return_value = True
@@ -102,7 +104,7 @@
                   'os-internal-network': '10.0.0.1/24',
                   'os-admin-network': '10.0.1.0/24',
                   'os-public-network': '10.0.2.0/24'}
-        mock_config.side_effect = lambda key: config[key]
+        mock_config.side_effect = lambda key: config.get(key)
 
         mock_use_local_neutron_api.return_value = False
         ctxt = context.NeutronCCContext()()
@@ -114,18 +116,20 @@
         self.assertEqual(ctxt['nova_url'], 'http://10.0.0.1:8774/v2')
         self.assertEqual(ctxt['neutron_url'], 'http://10.0.0.1:9696')
 
+    @mock.patch('charmhelpers.contrib.openstack.neutron.os_release')
     @mock.patch.object(context, 'use_local_neutron_api')
     @mock.patch('charmhelpers.contrib.openstack.ip.config')
     @mock.patch('charmhelpers.contrib.openstack.ip.is_clustered')
     def test_neutron_context_multi_vip(self, mock_is_clustered, mock_config,
-                                       mock_use_local_neutron_api):
+                                       mock_use_local_neutron_api,
+                                       _os_release):
         self.https.return_value = False
         mock_is_clustered.return_value = True
         config = {'vip': '10.0.0.1 10.0.1.1 10.0.2.1',
                   'os-internal-network': '10.0.1.0/24',
                   'os-admin-network': '10.0.0.0/24',
                   'os-public-network': '10.0.2.0/24'}
-        mock_config.side_effect = lambda key: config[key]
+        mock_config.side_effect = lambda key: config.get(key)
 
         mock_use_local_neutron_api.return_value = False
         ctxt = context.NeutronCCContext()()

=== modified file 'unit_tests/test_nova_cc_hooks.py'
--- unit_tests/test_nova_cc_hooks.py	2015-05-22 14:32:59 +0000
+++ unit_tests/test_nova_cc_hooks.py	2015-06-04 23:33:37 +0000
@@ -26,13 +26,13 @@
     'api_port',
     'apt_update',
     'apt_install',
-    'canonical_url',
     'configure_installation_source',
     'charm_dir',
     'do_openstack_upgrade',
     'openstack_upgrade_available',
     'cmd_all_services',
     'config',
+    'determine_endpoints',
     'determine_packages',
     'determine_ports',
     'disable_services',
@@ -122,6 +122,8 @@
         hooks.config_changed()
         self.assertTrue(self.save_script_rc.called)
 
+    @patch('charmhelpers.contrib.openstack.ip.service_name',
+           lambda *args: 'nova-cloud-controller')
     @patch.object(hooks, 'cluster_joined')
     @patch.object(hooks, 'identity_joined')
     @patch.object(hooks, 'neutron_api_relation_joined')
@@ -198,9 +200,11 @@
         self.assertEquals(sorted(self.relation_set.call_args_list),
                           sorted(expected_relations))
 
+    @patch.object(hooks, 'canonical_url')
     @patch.object(utils, 'config')
     @patch.object(hooks, '_auth_config')
-    def test_compute_joined_neutron(self, auth_config, _util_config):
+    def test_compute_joined_neutron(self, auth_config, _util_config,
+                                    _canonical_url):
         _util_config.return_value = None
         self.is_relation_made.return_value = False
         self.network_manager.return_value = 'neutron'
@@ -208,7 +212,7 @@
         self.keystone_ca_cert_b64.return_value = 'foocert64'
         self.volume_service.return_value = 'cinder'
         self.unit_get.return_value = 'nova-cc-host1'
-        self.canonical_url.return_value = 'http://nova-cc-host1'
+        _canonical_url.return_value = 'http://nova-cc-host1'
         self.api_port.return_value = '9696'
         self.neutron_plugin.return_value = 'nvp'
         auth_config.return_value = FAKE_KS_AUTH_CFG
@@ -227,11 +231,12 @@
             quantum_plugin='nvp',
             network_manager='neutron', **FAKE_KS_AUTH_CFG)
 
+    @patch.object(hooks, 'canonical_url')
     @patch.object(utils, 'config')
     @patch.object(hooks, 'NeutronAPIContext')
     @patch.object(hooks, '_auth_config')
     def test_compute_joined_neutron_api_rel(self, auth_config, napi,
-                                            _util_config):
+                                            _util_config, _canonical_url):
         def mock_NeutronAPIContext():
             return {
                 'neutron_plugin': 'bob',
@@ -246,7 +251,7 @@
         self.keystone_ca_cert_b64.return_value = 'foocert64'
         self.volume_service.return_value = 'cinder'
         self.unit_get.return_value = 'nova-cc-host1'
-        self.canonical_url.return_value = 'http://nova-cc-host1'
+        _canonical_url.return_value = 'http://nova-cc-host1'
         self.api_port.return_value = '9696'
         self.neutron_plugin.return_value = 'nvp'
         auth_config.return_value = FAKE_KS_AUTH_CFG
@@ -264,13 +269,14 @@
             quantum_plugin='bob',
             network_manager='neutron', **FAKE_KS_AUTH_CFG)
 
+    @patch.object(hooks, 'canonical_url')
     @patch.object(hooks, '_auth_config')
-    def test_nova_vmware_joined(self, auth_config):
+    def test_nova_vmware_joined(self, auth_config, _canonical_url):
         auth_config.return_value = FAKE_KS_AUTH_CFG
         # quantum-security-groups, plugin
         self.neutron_plugin.return_value = 'nvp'
         self.network_manager.return_value = 'neutron'
-        self.canonical_url.return_value = 'http://nova-cc-host1'
+        _canonical_url.return_value = 'http://nova-cc-host1'
         self.api_port.return_value = '9696'
         hooks.nova_vmware_relation_joined()
         self.relation_set.assert_called_with(
@@ -288,6 +294,25 @@
                                              nova_hostname='nova.foohost.com')
         self.unit_get.assert_called_with('private-address')
 
+    @patch('charmhelpers.contrib.openstack.ip.service_name',
+           lambda *args: 'nova-cloud-controller')
+    @patch('charmhelpers.contrib.openstack.ip.unit_get')
+    @patch('charmhelpers.contrib.openstack.ip.is_clustered')
+    @patch('charmhelpers.contrib.openstack.ip.config')
+    def test_identity_joined(self, _ip_config, _is_clustered, _unit_get):
+        _is_clustered.return_value = False
+        _unit_get.return_value = '127.0.0.1'
+        _ip_config.side_effect = self.test_config.get
+
+        self.test_config.set('os-public-hostname', 'ncc.example.com')
+        hooks.identity_joined()
+
+        self.determine_endpoints.asssert_called_with(
+            public_url='http://ncc.example.com',
+            internal_url='http://127.0.0.1',
+            admin_url='http://127.0.0.1'
+        )
+
     def test_postgresql_nova_db_joined(self):
         self.is_relation_made.return_value = False
         hooks.pgsql_nova_db_joined()
@@ -452,9 +477,10 @@
                            call('/etc/neutron/neutron.conf')])
         cell_joined.assert_called_with(rid='nova-cell-api/0')
 
-    def test_nova_cell_relation_joined(self):
+    @patch.object(hooks, 'canonical_url')
+    def test_nova_cell_relation_joined(self, _canonical_url):
         self.uuid.uuid4.return_value = 'bob'
-        self.canonical_url.return_value = 'http://novaurl'
+        _canonical_url.return_value = 'http://novaurl'
         hooks.nova_cell_relation_joined(rid='rid',
                                         remote_restart=True)
         self.relation_set.assert_called_with(restart_trigger='bob',
@@ -473,19 +499,20 @@
         }
         self.assertEquals(hooks.get_cell_type(), 'parent')
 
+    @patch.object(hooks, 'canonical_url')
     @patch.object(os, 'rename')
     @patch.object(os.path, 'isfile')
     @patch.object(hooks, 'CONFIGS')
     @patch.object(hooks, 'get_cell_type')
     def test_neutron_api_relation_joined(self, get_cell_type, configs, isfile,
-                                         rename):
+                                         rename, _canonical_url):
         neutron_conf = '/etc/neutron/neutron.conf'
         nova_url = 'http://novaurl:8774/v2'
         isfile.return_value = True
         self.service_running.return_value = True
         _identity_joined = self.patch('identity_joined')
         self.relation_ids.return_value = ['relid']
-        self.canonical_url.return_value = 'http://novaurl'
+        _canonical_url.return_value = 'http://novaurl'
         get_cell_type.return_value = 'parent'
         self.uuid.uuid4.return_value = 'bob'
         with patch_open() as (_open, _file):
@@ -522,11 +549,12 @@
         self.assertTrue(_compute_joined.called)
         self.assertTrue(_quantum_joined.called)
 
+    @patch.object(hooks, 'canonical_url')
     @patch.object(utils, 'config')
-    def test_console_settings_vnc(self, _utils_config):
+    def test_console_settings_vnc(self, _utils_config, _canonical_url):
         _utils_config.return_value = 'vnc'
         _cc_host = "nova-cc-host1"
-        self.canonical_url.return_value = 'http://' + _cc_host
+        _canonical_url.return_value = 'http://' + _cc_host
         _con_sets = hooks.console_settings()
         console_settings = {
             'console_proxy_novnc_address': 'http://%s:6080/vnc_auto.html' %
@@ -542,11 +570,12 @@
         }
         self.assertEqual(_con_sets, console_settings)
 
+    @patch.object(hooks, 'canonical_url')
     @patch.object(utils, 'config')
-    def test_console_settings_xvpvnc(self, _utils_config):
+    def test_console_settings_xvpvnc(self, _utils_config, _canonical_url):
         _utils_config.return_value = 'xvpvnc'
         _cc_host = "nova-cc-host1"
-        self.canonical_url.return_value = 'http://' + _cc_host
+        _canonical_url.return_value = 'http://' + _cc_host
         _con_sets = hooks.console_settings()
         console_settings = {
             'console_access_protocol': 'xvpvnc',
@@ -558,11 +587,12 @@
         }
         self.assertEqual(_con_sets, console_settings)
 
+    @patch.object(hooks, 'canonical_url')
     @patch.object(utils, 'config')
-    def test_console_settings_novnc(self, _utils_config):
+    def test_console_settings_novnc(self, _utils_config, _canonical_url):
         _utils_config.return_value = 'novnc'
         _cc_host = "nova-cc-host1"
-        self.canonical_url.return_value = 'http://' + _cc_host
+        _canonical_url.return_value = 'http://' + _cc_host
         _con_sets = hooks.console_settings()
         console_settings = {
             'console_proxy_novnc_address': 'http://%s:6080/vnc_auto.html' %
@@ -574,11 +604,12 @@
         }
         self.assertEqual(_con_sets, console_settings)
 
+    @patch.object(hooks, 'canonical_url')
     @patch.object(utils, 'config')
-    def test_console_settings_spice(self, _utils_config):
+    def test_console_settings_spice(self, _utils_config, _canonical_url):
         _utils_config.return_value = 'spice'
         _cc_host = "nova-cc-host1"
-        self.canonical_url.return_value = 'http://' + _cc_host
+        _canonical_url.return_value = 'http://' + _cc_host
         _con_sets = hooks.console_settings()
         console_settings = {
             'console_proxy_spice_address': 'http://%s:6082/spice_auto.html' %
@@ -590,14 +621,16 @@
         }
         self.assertEqual(_con_sets, console_settings)
 
+    @patch.object(hooks, 'canonical_url')
     @patch.object(utils, 'config')
-    def test_console_settings_explicit_ip(self, _utils_config):
+    def test_console_settings_explicit_ip(self, _utils_config,
+                                          _canonical_url):
         _utils_config.return_value = 'spice'
         _cc_public_host = "public-host"
         _cc_private_host = "private-host"
         self.test_config.set('console-proxy-ip', _cc_public_host)
         _con_sets = hooks.console_settings()
-        self.canonical_url.return_value = 'http://' + _cc_private_host
+        _canonical_url.return_value = 'http://' + _cc_private_host
         console_settings = {
             'console_proxy_spice_address': 'http://%s:6082/spice_auto.html' %
                                            (_cc_public_host),