Merge lp:~le-charmers/charms/trusty/keystone/leadership-election into lp:~openstack-charmers-archive/charms/trusty/keystone/next

[Edward Hope-Morley]

[lp:~le-charmers/charms/trusty/keystone/leadership-election]

158. By Edward Hope-Morley on 2015-04-09

synced /next

159. By Edward Hope-Morley on 2015-04-09

make sync

160. By Liam Young on 2015-05-11

Merged trunk in + LE charmhelper sync

161. By Liam Young on 2015-06-04

Resync le charm helpers

=== modified file 'charm-helpers-tests.yaml'
--- charm-helpers-tests.yaml	2015-05-13 09:42:17 +0000
+++ charm-helpers-tests.yaml	2015-06-04 08:44:44 +0000
@@ -1,4 +1,4 @@
-branch: lp:charm-helpers
+branch: lp:charm-helpers 
 destination: tests/charmhelpers
 include:
     - contrib.amulet

=== modified file 'hooks/charmhelpers/contrib/hahelpers/cluster.py'
--- hooks/charmhelpers/contrib/hahelpers/cluster.py	2015-03-18 18:59:03 +0000
+++ hooks/charmhelpers/contrib/hahelpers/cluster.py	2015-06-04 08:44:44 +0000
@@ -44,6 +44,7 @@
     ERROR,
     WARNING,
     unit_get,
+    is_leader as juju_is_leader
 )
 from charmhelpers.core.decorators import (
     retry_on_exception,
@@ -52,6 +53,8 @@
     bool_from_string,
 )
 
+DC_RESOURCE_NAME = 'DC'
+
 
 class HAIncompleteConfig(Exception):
     pass
@@ -66,12 +69,21 @@
     Returns True if the charm executing this is the elected cluster leader.
 
     It relies on two mechanisms to determine leadership:
-        1. If the charm is part of a corosync cluster, call corosync to
+        1. If juju is sufficiently new and leadership election is supported,
+        the is_leader command will be used.
+        2. If the charm is part of a corosync cluster, call corosync to
         determine leadership.
-        2. If the charm is not part of a corosync cluster, the leader is
+        3. If the charm is not part of a corosync cluster, the leader is
         determined as being "the alive unit with the lowest unit numer". In
         other words, the oldest surviving unit.
     """
+    try:
+        return juju_is_leader()
+    except NotImplementedError:
+        log('Juju leadership election feature not enabled'
+            ', using fallback support',
+            level=WARNING)
+
     if is_clustered():
         if not is_crm_leader(resource):
             log('Deferring action to CRM leader.', level=INFO)
@@ -95,6 +107,27 @@
     return False
 
 
+def is_crm_dc():
+    """
+    Determine leadership by querying the pacemaker Designated Controller
+    """
+    cmd = ['crm', 'status']
+    try:
+        status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+        if not isinstance(status, six.text_type):
+            status = six.text_type(status, "utf-8")
+    except subprocess.CalledProcessError:
+        return False
+    current_dc = ''
+    for line in status.split('\n'):
+        if line.startswith('Current DC'):
+            # Current DC: juju-lytrusty-machine-2 (168108163) - partition with quorum
+            current_dc = line.split(':')[1].split()[0]
+    if current_dc == get_unit_hostname():
+        return True
+    return False
+
+
 @retry_on_exception(5, base_delay=2, exc_type=CRMResourceNotFound)
 def is_crm_leader(resource, retry=False):
     """
@@ -104,6 +137,8 @@
     We allow this operation to be retried to avoid the possibility of getting a
     false negative. See LP #1396246 for more info.
     """
+    if resource == DC_RESOURCE_NAME:
+        return is_crm_dc()
     cmd = ['crm', 'resource', 'show', resource]
     try:
         status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)

=== modified file 'hooks/charmhelpers/contrib/openstack/neutron.py'
--- hooks/charmhelpers/contrib/openstack/neutron.py	2015-04-16 19:55:16 +0000
+++ hooks/charmhelpers/contrib/openstack/neutron.py	2015-06-04 08:44:44 +0000
@@ -256,11 +256,14 @@
 def parse_mappings(mappings):
     parsed = {}
     if mappings:
-        mappings = mappings.split(' ')
+        mappings = mappings.split()
         for m in mappings:
             p = m.partition(':')
-            if p[1] == ':':
-                parsed[p[0].strip()] = p[2].strip()
+            key = p[0].strip()
+            if p[1]:
+                parsed[key] = p[2].strip()
+            else:
+                parsed[key] = ''
 
     return parsed
 
@@ -283,13 +286,13 @@
     Returns dict of the form {bridge:port}.
     """
     _mappings = parse_mappings(mappings)
-    if not _mappings:
+    if not _mappings or list(_mappings.values()) == ['']:
         if not mappings:
             return {}
 
         # For backwards-compatibility we need to support port-only provided in
         # config.
-        _mappings = {default_bridge: mappings.split(' ')[0]}
+        _mappings = {default_bridge: mappings.split()[0]}
 
     bridges = _mappings.keys()
     ports = _mappings.values()
@@ -309,6 +312,8 @@
 
     Mappings must be a space-delimited list of provider:start:end mappings.
 
+    The start:end range is optional and may be omitted.
+
     Returns dict of the form {provider: (start, end)}.
     """
     _mappings = parse_mappings(mappings)

=== added file 'hooks/charmhelpers/contrib/openstack/templates/section-zeromq'
--- hooks/charmhelpers/contrib/openstack/templates/section-zeromq	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/section-zeromq	2015-06-04 08:44:44 +0000
@@ -0,0 +1,14 @@
+{% if zmq_host -%}
+# ZeroMQ configuration (restart-nonce: {{ zmq_nonce }})
+rpc_backend = zmq
+rpc_zmq_host = {{ zmq_host }}
+{% if zmq_redis_address -%}
+rpc_zmq_matchmaker = redis
+matchmaker_heartbeat_freq = 15
+matchmaker_heartbeat_ttl = 30
+[matchmaker_redis]
+host = {{ zmq_redis_address }}
+{% else -%}
+rpc_zmq_matchmaker = ring
+{% endif -%}
+{% endif -%}

=== removed file 'hooks/charmhelpers/contrib/openstack/templates/section-zeromq'
--- hooks/charmhelpers/contrib/openstack/templates/section-zeromq	2015-04-09 02:17:36 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/section-zeromq	1970-01-01 00:00:00 +0000
@@ -1,14 +0,0 @@
-{% if zmq_host -%}
-# ZeroMQ configuration (restart-nonce: {{ zmq_nonce }})
-rpc_backend = zmq
-rpc_zmq_host = {{ zmq_host }}
-{% if zmq_redis_address -%}
-rpc_zmq_matchmaker = redis
-matchmaker_heartbeat_freq = 15
-matchmaker_heartbeat_ttl = 30
-[matchmaker_redis]
-host = {{ zmq_redis_address }}
-{% else -%}
-rpc_zmq_matchmaker = ring
-{% endif -%}
-{% endif -%}

=== modified file 'hooks/charmhelpers/contrib/openstack/utils.py'
--- hooks/charmhelpers/contrib/openstack/utils.py	2015-04-16 14:09:47 +0000
+++ hooks/charmhelpers/contrib/openstack/utils.py	2015-06-04 08:44:44 +0000
@@ -53,9 +53,13 @@
     get_ipv6_addr
 )
 
+from charmhelpers.contrib.python.packages import (
+    pip_create_virtualenv,
+    pip_install,
+)
+
 from charmhelpers.core.host import lsb_release, mounts, umount
 from charmhelpers.fetch import apt_install, apt_cache, install_remote
-from charmhelpers.contrib.python.packages import pip_install
 from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
 from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
 
@@ -497,7 +501,17 @@
 requirements_dir = None
 
 
-def git_clone_and_install(projects_yaml, core_project):
+def _git_yaml_load(projects_yaml):
+    """
+    Load the specified yaml into a dictionary.
+    """
+    if not projects_yaml:
+        return None
+
+    return yaml.load(projects_yaml)
+
+
+def git_clone_and_install(projects_yaml, core_project, depth=1):
     """
     Clone/install all specified OpenStack repositories.
 
@@ -510,23 +524,22 @@
              repository: 'git://git.openstack.org/openstack/requirements.git',
              branch: 'stable/icehouse'}
         directory: /mnt/openstack-git
-        http_proxy: http://squid.internal:3128
-        https_proxy: https://squid.internal:3128
+        http_proxy: squid-proxy-url
+        https_proxy: squid-proxy-url
 
         The directory, http_proxy, and https_proxy keys are optional.
     """
     global requirements_dir
     parent_dir = '/mnt/openstack-git'
-
-    if not projects_yaml:
-        return
-
-    projects = yaml.load(projects_yaml)
+    http_proxy = None
+
+    projects = _git_yaml_load(projects_yaml)
     _git_validate_projects_yaml(projects, core_project)
 
     old_environ = dict(os.environ)
 
     if 'http_proxy' in projects.keys():
+        http_proxy = projects['http_proxy']
         os.environ['http_proxy'] = projects['http_proxy']
     if 'https_proxy' in projects.keys():
         os.environ['https_proxy'] = projects['https_proxy']
@@ -534,15 +547,19 @@
     if 'directory' in projects.keys():
         parent_dir = projects['directory']
 
+    pip_create_virtualenv(os.path.join(parent_dir, 'venv'))
+
     for p in projects['repositories']:
         repo = p['repository']
         branch = p['branch']
         if p['name'] == 'requirements':
-            repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
+            repo_dir = _git_clone_and_install_single(repo, branch, depth,
+                                                     parent_dir, http_proxy,
                                                      update_requirements=False)
             requirements_dir = repo_dir
         else:
-            repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
+            repo_dir = _git_clone_and_install_single(repo, branch, depth,
+                                                     parent_dir, http_proxy,
                                                      update_requirements=True)
 
     os.environ = old_environ
@@ -574,7 +591,8 @@
         error_out('openstack-origin-git key \'{}\' is missing'.format(key))
 
 
-def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements):
+def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy,
+                                  update_requirements):
     """
     Clone and install a single git repository.
     """
@@ -587,7 +605,8 @@
 
     if not os.path.exists(dest_dir):
         juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
-        repo_dir = install_remote(repo, dest=parent_dir, branch=branch)
+        repo_dir = install_remote(repo, dest=parent_dir, branch=branch,
+                                  depth=depth)
     else:
         repo_dir = dest_dir
 
@@ -598,7 +617,12 @@
         _git_update_requirements(repo_dir, requirements_dir)
 
     juju_log('Installing git repo from dir: {}'.format(repo_dir))
-    pip_install(repo_dir)
+    if http_proxy:
+        pip_install(repo_dir, proxy=http_proxy,
+                    venv=os.path.join(parent_dir, 'venv'))
+    else:
+        pip_install(repo_dir,
+                    venv=os.path.join(parent_dir, 'venv'))
 
     return repo_dir
 
@@ -621,16 +645,27 @@
     os.chdir(orig_dir)
 
 
+def git_pip_venv_dir(projects_yaml):
+    """
+    Return the pip virtualenv path.
+    """
+    parent_dir = '/mnt/openstack-git'
+
+    projects = _git_yaml_load(projects_yaml)
+
+    if 'directory' in projects.keys():
+        parent_dir = projects['directory']
+
+    return os.path.join(parent_dir, 'venv')
+
+
 def git_src_dir(projects_yaml, project):
     """
     Return the directory where the specified project's source is located.
     """
     parent_dir = '/mnt/openstack-git'
 
-    if not projects_yaml:
-        return
-
-    projects = yaml.load(projects_yaml)
+    projects = _git_yaml_load(projects_yaml)
 
     if 'directory' in projects.keys():
         parent_dir = projects['directory']
@@ -640,3 +675,15 @@
             return os.path.join(parent_dir, os.path.basename(p['repository']))
 
     return None
+
+
+def git_yaml_value(projects_yaml, key):
+    """
+    Return the value in projects_yaml for the specified key.
+    """
+    projects = _git_yaml_load(projects_yaml)
+
+    if key in projects.keys():
+        return projects[key]
+
+    return None

=== modified file 'hooks/charmhelpers/contrib/peerstorage/__init__.py'
--- hooks/charmhelpers/contrib/peerstorage/__init__.py	2015-03-11 11:45:09 +0000
+++ hooks/charmhelpers/contrib/peerstorage/__init__.py	2015-06-04 08:44:44 +0000
@@ -14,14 +14,19 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import json
 import six
+
 from charmhelpers.core.hookenv import relation_id as current_relation_id
 from charmhelpers.core.hookenv import (
     is_relation_made,
     relation_ids,
-    relation_get,
+    relation_get as _relation_get,
     local_unit,
-    relation_set,
+    relation_set as _relation_set,
+    leader_get as _leader_get,
+    leader_set,
+    is_leader,
 )
 
 
@@ -54,6 +59,105 @@
 """
 
 
+def leader_get(attribute=None):
+    """Wrapper to ensure that settings are migrated from the peer relation.
+
+    This is to support upgrading an environment that does not support
+    Juju leadership election to one that does.
+
+    If a setting is not extant in the leader-get but is on the relation-get
+    peer rel, it is migrated and marked as such so that it is not re-migrated.
+    """
+    migration_key = '__leader_get_migrated_settings__'
+    if not is_leader():
+        return _leader_get(attribute=attribute)
+
+    settings_migrated = False
+    leader_settings = _leader_get(attribute=attribute)
+    previously_migrated = _leader_get(attribute=migration_key)
+
+    if previously_migrated:
+        migrated = set(json.loads(previously_migrated))
+    else:
+        migrated = set([])
+
+    try:
+        if migration_key in leader_settings:
+            del leader_settings[migration_key]
+    except TypeError:
+        pass
+
+    if attribute:
+        if attribute in migrated:
+            return leader_settings
+
+        # If attribute not present in leader db, check if this unit has set
+        # the attribute in the peer relation
+        if not leader_settings:
+            peer_setting = relation_get(attribute=attribute, unit=local_unit())
+            if peer_setting:
+                leader_set(settings={attribute: peer_setting})
+                leader_settings = peer_setting
+
+        if leader_settings:
+            settings_migrated = True
+            migrated.add(attribute)
+    else:
+        r_settings = relation_get(unit=local_unit())
+        if r_settings:
+            for key in set(r_settings.keys()).difference(migrated):
+                # Leader setting wins
+                if not leader_settings.get(key):
+                    leader_settings[key] = r_settings[key]
+
+                settings_migrated = True
+                migrated.add(key)
+
+            if settings_migrated:
+                leader_set(**leader_settings)
+
+    if migrated and settings_migrated:
+        migrated = json.dumps(list(migrated))
+        leader_set(settings={migration_key: migrated})
+
+    return leader_settings
+
+
+def relation_set(relation_id=None, relation_settings=None, **kwargs):
+    """Attempt to use leader-set if supported in the current version of Juju,
+    otherwise falls back on relation-set.
+
+    Note that we only attempt to use leader-set if the provided relation_id is
+    a peer relation id or no relation id is provided (in which case we assume
+    we are within the peer relation context).
+    """
+    try:
+        if relation_id in relation_ids('cluster'):
+            return leader_set(settings=relation_settings, **kwargs)
+        else:
+            raise NotImplementedError
+    except NotImplementedError:
+        return _relation_set(relation_id=relation_id,
+                             relation_settings=relation_settings, **kwargs)
+
+
+def relation_get(attribute=None, unit=None, rid=None):
+    """Attempt to use leader-get if supported in the current version of Juju,
+    otherwise falls back on relation-get.
+
+    Note that we only attempt to use leader-get if the provided rid is a peer
+    relation id or no relation id is provided (in which case we assume we are
+    within the peer relation context).
+    """
+    try:
+        if rid in relation_ids('cluster'):
+            return leader_get(attribute)
+        else:
+            raise NotImplementedError
+    except NotImplementedError:
+        return _relation_get(attribute=attribute, rid=rid, unit=unit)
+
+
 def peer_retrieve(key, relation_name='cluster'):
     """Retrieve a named key from peer relation `relation_name`."""
     cluster_rels = relation_ids(relation_name)
@@ -73,6 +177,8 @@
     exc_list = exc_list if exc_list else []
     peerdb_settings = peer_retrieve('-', relation_name=relation_name)
     matched = {}
+    if peerdb_settings is None:
+        return matched
     for k, v in peerdb_settings.items():
         full_prefix = prefix + delimiter
         if k.startswith(full_prefix):
@@ -96,12 +202,26 @@
                          'peer relation {}'.format(relation_name))
 
 
-def peer_echo(includes=None):
+def peer_echo(includes=None, force=False):
     """Echo filtered attributes back onto the same relation for storage.
 
     This is a requirement to use the peerstorage module - it needs to be called
     from the peer relation's changed hook.
+
+    If Juju leader support exists this will be a noop unless force is True.
     """
+    try:
+        is_leader()
+    except NotImplementedError:
+        pass
+    else:
+        if not force:
+            return  # NOOP if leader-election is supported
+
+    # Use original non-leader calls
+    relation_get = _relation_get
+    relation_set = _relation_set
+
     rdata = relation_get()
     echo_data = {}
     if includes is None:

=== modified file 'hooks/charmhelpers/contrib/python/packages.py'
--- hooks/charmhelpers/contrib/python/packages.py	2015-03-11 11:45:09 +0000
+++ hooks/charmhelpers/contrib/python/packages.py	2015-06-04 08:44:44 +0000
@@ -17,8 +17,11 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
+import subprocess
+
 from charmhelpers.fetch import apt_install, apt_update
-from charmhelpers.core.hookenv import log
+from charmhelpers.core.hookenv import charm_dir, log
 
 try:
     from pip import main as pip_execute
@@ -51,11 +54,15 @@
     pip_execute(command)
 
 
-def pip_install(package, fatal=False, upgrade=False, **options):
+def pip_install(package, fatal=False, upgrade=False, venv=None, **options):
     """Install a python package"""
-    command = ["install"]
+    if venv:
+        venv_python = os.path.join(venv, 'bin/pip')
+        command = [venv_python, "install"]
+    else:
+        command = ["install"]
 
-    available_options = ('proxy', 'src', 'log', "index-url", )
+    available_options = ('proxy', 'src', 'log', 'index-url', )
     for option in parse_options(options, available_options):
         command.append(option)
 
@@ -69,7 +76,10 @@
 
     log("Installing {} package with options: {}".format(package,
                                                         command))
-    pip_execute(command)
+    if venv:
+        subprocess.check_call(command)
+    else:
+        pip_execute(command)
 
 
 def pip_uninstall(package, **options):
@@ -94,3 +104,16 @@
     """Returns the list of current python installed packages
     """
     return pip_execute(["list"])
+
+
+def pip_create_virtualenv(path=None):
+    """Create an isolated Python environment."""
+    apt_install('python-virtualenv')
+
+    if path:
+        venv_path = path
+    else:
+        venv_path = os.path.join(charm_dir(), 'venv')
+
+    if not os.path.exists(venv_path):
+        subprocess.check_call(['virtualenv', venv_path])

=== modified file 'hooks/charmhelpers/contrib/unison/__init__.py'
--- hooks/charmhelpers/contrib/unison/__init__.py	2015-03-18 18:59:03 +0000
+++ hooks/charmhelpers/contrib/unison/__init__.py	2015-06-04 08:44:44 +0000
@@ -63,6 +63,7 @@
 from charmhelpers.core.host import (
     adduser,
     add_user_to_group,
+    pwgen,
 )
 
 from charmhelpers.core.hookenv import (
@@ -140,7 +141,7 @@
     ssh_dir = os.path.join(home_dir, '.ssh')
     auth_keys = os.path.join(ssh_dir, 'authorized_keys')
     log('Syncing authorized_keys @ %s.' % auth_keys)
-    with open(auth_keys, 'wb') as out:
+    with open(auth_keys, 'w') as out:
         for k in keys:
             out.write('%s\n' % k)
 
@@ -152,16 +153,16 @@
     khosts = []
     for host in hosts:
         cmd = ['ssh-keyscan', '-H', '-t', 'rsa', host]
-        remote_key = check_output(cmd).strip()
+        remote_key = check_output(cmd, universal_newlines=True).strip()
         khosts.append(remote_key)
     log('Syncing known_hosts @ %s.' % known_hosts)
-    with open(known_hosts, 'wb') as out:
+    with open(known_hosts, 'w') as out:
         for host in khosts:
             out.write('%s\n' % host)
 
 
 def ensure_user(user, group=None):
-    adduser(user)
+    adduser(user, pwgen())
     if group:
         add_user_to_group(user, group)
 

=== modified file 'hooks/charmhelpers/core/hookenv.py'
--- hooks/charmhelpers/core/hookenv.py	2015-04-15 15:21:50 +0000
+++ hooks/charmhelpers/core/hookenv.py	2015-06-04 08:44:44 +0000
@@ -21,12 +21,14 @@
 #  Charm Helpers Developers <[e-mail]>
 
 from __future__ import print_function
+from functools import wraps
 import os
 import json
 import yaml
 import subprocess
 import sys
 import errno
+import tempfile
 from subprocess import CalledProcessError
 
 import six
@@ -58,15 +60,17 @@
 
     will cache the result of unit_get + 'test' for future calls.
     """
+    @wraps(func)
     def wrapper(*args, **kwargs):
         global cache
         key = str((func, args, kwargs))
         try:
             return cache[key]
         except KeyError:
-            res = func(*args, **kwargs)
-            cache[key] = res
-            return res
+            pass  # Drop out of the exception handler scope.
+        res = func(*args, **kwargs)
+        cache[key] = res
+        return res
     return wrapper
 
 
@@ -178,7 +182,7 @@
 
 def remote_unit():
     """The remote unit for the current relation hook"""
-    return os.environ['JUJU_REMOTE_UNIT']
+    return os.environ.get('JUJU_REMOTE_UNIT', None)
 
 
 def service_name():
@@ -250,6 +254,12 @@
         except KeyError:
             return (self._prev_dict or {})[key]
 
+    def get(self, key, default=None):
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
     def keys(self):
         prev_keys = []
         if self._prev_dict is not None:
@@ -353,18 +363,49 @@
     """Set relation information for the current unit"""
     relation_settings = relation_settings if relation_settings else {}
     relation_cmd_line = ['relation-set']
+    accepts_file = "--file" in subprocess.check_output(
+        relation_cmd_line + ["--help"], universal_newlines=True)
     if relation_id is not None:
         relation_cmd_line.extend(('-r', relation_id))
-    for k, v in (list(relation_settings.items()) + list(kwargs.items())):
-        if v is None:
-            relation_cmd_line.append('{}='.format(k))
-        else:
-            relation_cmd_line.append('{}={}'.format(k, v))
-    subprocess.check_call(relation_cmd_line)
+    settings = relation_settings.copy()
+    settings.update(kwargs)
+    for key, value in settings.items():
+        # Force value to be a string: it always should, but some call
+        # sites pass in things like dicts or numbers.
+        if value is not None:
+            settings[key] = "{}".format(value)
+    if accepts_file:
+        # --file was introduced in Juju 1.23.2. Use it by default if
+        # available, since otherwise we'll break if the relation data is
+        # too big. Ideally we should tell relation-set to read the data from
+        # stdin, but that feature is broken in 1.23.2: Bug #1454678.
+        with tempfile.NamedTemporaryFile(delete=False) as settings_file:
+            settings_file.write(yaml.safe_dump(settings).encode("utf-8"))
+        subprocess.check_call(
+            relation_cmd_line + ["--file", settings_file.name])
+        os.remove(settings_file.name)
+    else:
+        for key, value in settings.items():
+            if value is None:
+                relation_cmd_line.append('{}='.format(key))
+            else:
+                relation_cmd_line.append('{}={}'.format(key, value))
+        subprocess.check_call(relation_cmd_line)
     # Flush cache of any relation-gets for local unit
     flush(local_unit())
 
 
+def relation_clear(r_id=None):
+    ''' Clears any relation data already set on relation r_id '''
+    settings = relation_get(rid=r_id,
+                            unit=local_unit())
+    for setting in settings:
+        if setting not in ['public-address', 'private-address']:
+            settings[setting] = None
+    relation_set(relation_id=r_id,
+                 **settings)
+
+
 @cached
 def relation_ids(reltype=None):
     """A list of relation_ids"""
@@ -509,6 +550,11 @@
         return None
 
 
+def unit_public_ip():
+    """Get this unit's public IP address"""
+    return unit_get('public-address')
+
+
 def unit_private_ip():
     """Get this unit's private IP address"""
     return unit_get('private-address')
@@ -605,3 +651,94 @@
 
     The results set by action_set are preserved."""
     subprocess.check_call(['action-fail', message])
+
+
+def status_set(workload_state, message):
+    """Set the workload state with a message
+
+    Use status-set to set the workload state with a message which is visible
+    to the user via juju status. If the status-set command is not found then
+    assume this is juju < 1.23 and juju-log the message unstead.
+
+    workload_state -- valid juju workload state.
+    message        -- status update message
+    """
+    valid_states = ['maintenance', 'blocked', 'waiting', 'active']
+    if workload_state not in valid_states:
+        raise ValueError(
+            '{!r} is not a valid workload state'.format(workload_state)
+        )
+    cmd = ['status-set', workload_state, message]
+    try:
+        ret = subprocess.call(cmd)
+        if ret == 0:
+            return
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
+    log_message = 'status-set failed: {} {}'.format(workload_state,
+                                                    message)
+    log(log_message, level='INFO')
+
+
+def status_get():
+    """Retrieve the previously set juju workload state
+
+    If the status-set command is not found then assume this is juju < 1.23 and
+    return 'unknown'
+    """
+    cmd = ['status-get']
+    try:
+        raw_status = subprocess.check_output(cmd, universal_newlines=True)
+        status = raw_status.rstrip()
+        return status
+    except OSError as e:
+        if e.errno == errno.ENOENT:
+            return 'unknown'
+        else:
+            raise
+
+
+def translate_exc(from_exc, to_exc):
+    def inner_translate_exc1(f):
+        def inner_translate_exc2(*args, **kwargs):
+            try:
+                return f(*args, **kwargs)
+            except from_exc:
+                raise to_exc
+
+        return inner_translate_exc2
+
+    return inner_translate_exc1
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def is_leader():
+    """Does the current unit hold the juju leadership
+
+    Uses juju to determine whether the current unit is the leader of its peers
+    """
+    cmd = ['is-leader', '--format=json']
+    return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def leader_get(attribute=None):
+    """Juju leader get value(s)"""
+    cmd = ['leader-get', '--format=json'] + [attribute or '-']
+    return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def leader_set(settings=None, **kwargs):
+    """Juju leader set value(s)"""
+    log("Juju leader-set '%s'" % (settings), level=DEBUG)
+    cmd = ['leader-set']
+    settings = settings or {}
+    settings.update(kwargs)
+    for k, v in settings.iteritems():
+        if v is None:
+            cmd.append('{}='.format(k))
+        else:
+            cmd.append('{}={}'.format(k, v))
+    subprocess.check_call(cmd)

=== modified file 'hooks/charmhelpers/core/host.py'
--- hooks/charmhelpers/core/host.py	2015-03-30 11:43:06 +0000
+++ hooks/charmhelpers/core/host.py	2015-06-04 08:44:44 +0000
@@ -90,7 +90,7 @@
             ['service', service_name, 'status'],
             stderr=subprocess.STDOUT).decode('UTF-8')
     except subprocess.CalledProcessError as e:
-        return 'unrecognized service' not in e.output
+        return b'unrecognized service' not in e.output
     else:
         return True
 

=== modified file 'hooks/charmhelpers/core/services/base.py'
--- hooks/charmhelpers/core/services/base.py	2015-03-11 11:45:09 +0000
+++ hooks/charmhelpers/core/services/base.py	2015-06-04 08:44:44 +0000
@@ -15,9 +15,9 @@
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import re
 import json
-from collections import Iterable
+from inspect import getargspec
+from collections import Iterable, OrderedDict
 
 from charmhelpers.core import host
 from charmhelpers.core import hookenv
@@ -119,7 +119,7 @@
         """
         self._ready_file = os.path.join(hookenv.charm_dir(), 'READY-SERVICES.json')
         self._ready = None
-        self.services = {}
+        self.services = OrderedDict()
         for service in services or []:
             service_name = service['service']
             self.services[service_name] = service
@@ -132,8 +132,8 @@
         if hook_name == 'stop':
             self.stop_services()
         else:
+            self.reconfigure_services()
             self.provide_data()
-            self.reconfigure_services()
         cfg = hookenv.config()
         if cfg.implicit_save:
             cfg.save()
@@ -145,15 +145,36 @@
         A provider must have a `name` attribute, which indicates which relation
         to set data on, and a `provide_data()` method, which returns a dict of
         data to set.
+
+        The `provide_data()` method can optionally accept two parameters:
+
+          * ``remote_service`` The name of the remote service that the data will
+            be provided to.  The `provide_data()` method will be called once
+            for each connected service (not unit).  This allows the method to
+            tailor its data to the given service.
+          * ``service_ready`` Whether or not the service definition had all of
+            its requirements met, and thus the ``data_ready`` callbacks run.
+
+        Note that the ``provided_data`` methods are now called **after** the
+        ``data_ready`` callbacks are run.  This gives the ``data_ready`` callbacks
+        a chance to generate any data necessary for the providing to the remote
+        services.
         """
-        hook_name = hookenv.hook_name()
-        for service in self.services.values():
+        for service_name, service in self.services.items():
+            service_ready = self.is_ready(service_name)
             for provider in service.get('provided_data', []):
-                if re.match(r'{}-relation-(joined|changed)'.format(provider.name), hook_name):
-                    data = provider.provide_data()
-                    _ready = provider._is_ready(data) if hasattr(provider, '_is_ready') else data
-                    if _ready:
-                        hookenv.relation_set(None, data)
+                for relid in hookenv.relation_ids(provider.name):
+                    units = hookenv.related_units(relid)
+                    if not units:
+                        continue
+                    remote_service = units[0].split('/')[0]
+                    argspec = getargspec(provider.provide_data)
+                    if len(argspec.args) > 1:
+                        data = provider.provide_data(remote_service, service_ready)
+                    else:
+                        data = provider.provide_data()
+                    if data:
+                        hookenv.relation_set(relid, data)
 
     def reconfigure_services(self, *service_names):
         """

=== modified file 'hooks/charmhelpers/fetch/__init__.py'
--- hooks/charmhelpers/fetch/__init__.py	2015-03-11 11:45:09 +0000
+++ hooks/charmhelpers/fetch/__init__.py	2015-06-04 08:44:44 +0000
@@ -158,7 +158,7 @@
 
 def apt_cache(in_memory=True):
     """Build and return an apt cache"""
-    import apt_pkg
+    from apt import apt_pkg
     apt_pkg.init()
     if in_memory:
         apt_pkg.config.set("Dir::Cache::pkgcache", "")

=== modified file 'hooks/charmhelpers/fetch/giturl.py'
--- hooks/charmhelpers/fetch/giturl.py	2015-03-11 11:45:09 +0000
+++ hooks/charmhelpers/fetch/giturl.py	2015-06-04 08:44:44 +0000
@@ -45,14 +45,16 @@
         else:
             return True
 
-    def clone(self, source, dest, branch):
+    def clone(self, source, dest, branch, depth=None):
         if not self.can_handle(source):
             raise UnhandledSource("Cannot handle {}".format(source))
 
-        repo = Repo.clone_from(source, dest)
-        repo.git.checkout(branch)
+        if depth:
+            Repo.clone_from(source, dest, branch=branch, depth=depth)
+        else:
+            Repo.clone_from(source, dest, branch=branch)
 
-    def install(self, source, branch="master", dest=None):
+    def install(self, source, branch="master", dest=None, depth=None):
         url_parts = self.parse_url(source)
         branch_name = url_parts.path.strip("/").split("/")[-1]
         if dest:
@@ -63,7 +65,7 @@
         if not os.path.exists(dest_dir):
             mkdir(dest_dir, perms=0o755)
         try:
-            self.clone(source, dest_dir, branch)
+            self.clone(source, dest_dir, branch, depth)
         except GitCommandError as e:
             raise UnhandledSource(e.message)
         except OSError as e:

=== modified file 'hooks/keystone_hooks.py'
--- hooks/keystone_hooks.py	2015-04-01 14:39:21 +0000
+++ hooks/keystone_hooks.py	2015-06-04 08:44:44 +0000
@@ -2,7 +2,6 @@
 import hashlib
 import json
 import os
-import stat
 import sys
 
 from subprocess import check_call
@@ -68,18 +67,18 @@
     setup_ipv6,
     send_notifications,
     check_peer_actions,
-    CA_CERT_PATH,
-    ensure_permissions,
     get_ssl_sync_request_units,
     is_ssl_cert_master,
     is_db_ready,
     clear_ssl_synced_units,
     is_db_initialised,
+    update_certs_if_available,
     is_pki_enabled,
     ensure_ssl_dir,
     ensure_pki_dir_permissions,
     force_ssl_sync,
     filter_null,
+    ensure_ssl_dirs,
 )
 
 from charmhelpers.contrib.hahelpers.cluster import (
@@ -149,13 +148,7 @@
 
     check_call(['chmod', '-R', 'g+wrx', '/var/lib/keystone/'])
 
-    # Ensure unison can write to certs dir.
-    # FIXME: need to a better way around this e.g. move cert to it's own dir
-    # and give that unison permissions.
-    path = os.path.dirname(CA_CERT_PATH)
-    perms = int(oct(stat.S_IMODE(os.stat(path).st_mode) |
-                    (stat.S_IWGRP | stat.S_IXGRP)), base=8)
-    ensure_permissions(path, group='keystone', perms=perms)
+    ensure_ssl_dirs()
 
     save_script_rc()
     configure_https()
@@ -423,6 +416,7 @@
 @hooks.hook('cluster-relation-changed',
             'cluster-relation-departed')
 @restart_on_change(restart_map(), stopstart=True)
+@update_certs_if_available
 def cluster_changed():
     unison.ssh_authorized_peers(user=SSH_USER,
                                 group='juju_keystone',
@@ -430,9 +424,9 @@
                                 ensure_local_user=True)
     # NOTE(jamespage) re-echo passwords for peer storage
     echo_whitelist = ['_passwd', 'identity-service:', 'ssl-cert-master',
-                      'db-initialised']
+                      'db-initialised', 'ssl-cert-available-updates']
     log("Peer echo whitelist: %s" % (echo_whitelist), level=DEBUG)
-    peer_echo(includes=echo_whitelist)
+    peer_echo(includes=echo_whitelist, force=True)
 
     check_peer_actions()
 
@@ -466,6 +460,14 @@
         CONFIGS.write_all()
 
 
[e-mail]('leader-settings-changed')
+def leader_settings_changed():
+    log('Firing identity_changed hook for all related services.')
+    for rid in relation_ids('identity-service'):
+            for unit in related_units(rid):
+                identity_changed(relation_id=rid, remote_unit=unit)
+
+
 @hooks.hook('ha-relation-joined')
 def ha_joined(relation_id=None):
     cluster_config = get_hacluster_config()
@@ -575,6 +577,8 @@
                                 peer_interface='cluster',
                                 ensure_local_user=True)
 
+    ensure_ssl_dirs()
+
     CONFIGS.write_all()
     update_nrpe_config()
 

=== modified file 'hooks/keystone_ssl.py'
--- hooks/keystone_ssl.py	2015-02-18 17:20:23 +0000
+++ hooks/keystone_ssl.py	2015-06-04 08:44:44 +0000
@@ -5,12 +5,10 @@
 import subprocess
 import tarfile
 import tempfile
-import time
 
 from charmhelpers.core.hookenv import (
     log,
     DEBUG,
-    WARNING,
 )
 
 CA_EXPIRY = '365'
@@ -312,31 +310,9 @@
         if os.path.isfile(crtpath):
             log('Found existing certificate for %s.' % common_name,
                 level=DEBUG)
-            max_retries = 3
-            while True:
-                mtime = os.path.getmtime(crtpath)
-
-                crt = open(crtpath, 'r').read()
-                try:
-                    key = open(keypath, 'r').read()
-                except:
-                    msg = ('Could not load ssl private key for %s from %s' %
-                           (common_name, keypath))
-                    raise Exception(msg)
-
-                # Ensure we are not reading a file that is being written to
-                if mtime != os.path.getmtime(crtpath):
-                    max_retries -= 1
-                    if max_retries == 0:
-                        msg = ("crt contents changed during read - retry "
-                               "failed")
-                        raise Exception(msg)
-
-                    log("crt contents changed during read - re-reading",
-                        level=WARNING)
-                    time.sleep(1)
-                else:
-                    return crt, key
+            crt = open(crtpath, 'r').read()
+            key = open(keypath, 'r').read()
+            return crt, key
 
         crt, key = self._create_certificate(common_name, common_name)
         return open(crt, 'r').read(), open(key, 'r').read()

=== modified file 'hooks/keystone_utils.py'
--- hooks/keystone_utils.py	2015-05-08 11:43:00 +0000
+++ hooks/keystone_utils.py	2015-06-04 08:44:44 +0000
@@ -8,6 +8,7 @@
 import re
 import shutil
 import subprocess
+import tarfile
 import threading
 import time
 import urlparse
@@ -71,7 +72,6 @@
     DEBUG,
     INFO,
     WARNING,
-    ERROR,
 )
 
 from charmhelpers.fetch import (
@@ -160,6 +160,8 @@
 
 APACHE_SSL_DIR = '/etc/apache2/ssl/keystone'
 SYNC_FLAGS_DIR = '/var/lib/keystone/juju_sync_flags/'
+SYNC_DIR = '/var/lib/keystone/juju_sync/'
+SSL_SYNC_ARCHIVE = os.path.join(SYNC_DIR, 'juju-ssl-sync.tar')
 SSL_DIR = '/var/lib/keystone/juju_ssl/'
 PKI_CERTS_DIR = os.path.join(SSL_DIR, 'pki')
 SSL_CA_NAME = 'Ubuntu Cloud'
@@ -382,26 +384,20 @@
             return
 
 
-def set_db_initialised():
-    for rid in relation_ids('cluster'):
-        relation_set(relation_settings={'db-initialised': 'True'},
-                     relation_id=rid)
-
-
 def is_db_initialised():
-    for rid in relation_ids('cluster'):
-        units = related_units(rid) + [local_unit()]
-        for unit in units:
-            db_initialised = relation_get(attribute='db-initialised',
-                                          unit=unit, rid=rid)
-            if db_initialised:
-                log("Database is initialised", level=DEBUG)
-                return True
+    if relation_ids('cluster'):
+        inited = peer_retrieve('db-initialised')
+        if inited and bool_from_string(inited):
+            log("Database is initialised", level=DEBUG)
+            return True
 
     log("Database is NOT initialised", level=DEBUG)
     return False
 
 
+# NOTE(jamespage): Retry deals with sync issues during one-shot HA deploys.
+#                  mysql might be restarting or suchlike.
+@retry_on_exception(5, base_delay=3, exc_type=subprocess.CalledProcessError)
 def migrate_database():
     """Runs keystone-manage to initialize a new database or migrate existing"""
     log('Migrating the keystone database.', level=INFO)
@@ -413,7 +409,7 @@
     subprocess.check_output(cmd)
     service_start('keystone')
     time.sleep(10)
-    set_db_initialised()
+    peer_store('db-initialised', 'True')
 
 # OLD
 
@@ -768,6 +764,16 @@
     return passwd
 
 
+def ensure_ssl_dirs():
+    """Ensure unison has access to these dirs."""
+    for path in [SYNC_FLAGS_DIR, SYNC_DIR]:
+        if not os.path.isdir(path):
+            mkdir(path, SSH_USER, 'juju_keystone', 0o775)
+        else:
+            ensure_permissions(path, user=SSH_USER, group='keystone',
+                               perms=0o755)
+
+
 def ensure_permissions(path, user=None, group=None, perms=None, recurse=False,
                        maxdepth=50):
     """Set chownand chmod for path
@@ -864,7 +870,7 @@
                                  service.strip(), action))
         log("Creating action %s" % (flagfile), level=DEBUG)
         write_file(flagfile, content='', owner=SSH_USER, group='keystone',
-                   perms=0o644)
+                   perms=0o744)
 
 
 def create_peer_actions(actions):
@@ -873,7 +879,7 @@
         flagfile = os.path.join(SYNC_FLAGS_DIR, action)
         log("Creating action %s" % (flagfile), level=DEBUG)
         write_file(flagfile, content='', owner=SSH_USER, group='keystone',
-                   perms=0o644)
+                   perms=0o744)
 
 
 @retry_on_exception(3, base_delay=2, exc_type=subprocess.CalledProcessError)
@@ -1011,6 +1017,22 @@
     return True
 
 
+def stage_paths_for_sync(paths):
+    shutil.rmtree(SYNC_DIR)
+    ensure_ssl_dirs()
+    with tarfile.open(SSL_SYNC_ARCHIVE, 'w') as fd:
+        for path in paths:
+            if os.path.exists(path):
+                log("Adding path '%s' sync tarball" % (path), level=DEBUG)
+                fd.add(path)
+            else:
+                log("Path '%s' does not exist - not adding to sync "
+                    "tarball" % (path), level=INFO)
+
+    ensure_permissions(SYNC_DIR, user=SSH_USER, group='keystone',
+                       perms=0o755, recurse=True)
+
+
 def is_pki_enabled():
     enable_pki = config('enable-pki')
     if enable_pki and bool_from_string(enable_pki):
@@ -1025,6 +1047,33 @@
                        perms=0o755, recurse=True)
 
 
+def update_certs_if_available(f):
+    def _inner_update_certs_if_available(*args, **kwargs):
+        path = None
+        for rid in relation_ids('cluster'):
+            path = relation_get(attribute='ssl-cert-available-updates',
+                                rid=rid, unit=local_unit())
+
+        if path and os.path.exists(path):
+            log("Updating certs from '%s'" % (path), level=DEBUG)
+            with tarfile.open(path) as fd:
+                files = ["/%s" % m.name for m in fd.getmembers()]
+                fd.extractall(path='/')
+
+            for syncfile in files:
+                ensure_permissions(syncfile, user='keystone', group='keystone',
+                                   perms=0o744, recurse=True)
+
+            # Mark as complete
+            os.rename(path, "%s.complete" % (path))
+        else:
+            log("No cert updates available", level=DEBUG)
+
+        return f(*args, **kwargs)
+
+    return _inner_update_certs_if_available
+
+
 def synchronize_ca(fatal=False):
     """Broadcast service credentials to peers.
 
@@ -1038,7 +1087,7 @@
 
     Returns a dictionary of settings to be set on the cluster relation.
     """
-    paths_to_sync = [SYNC_FLAGS_DIR]
+    paths_to_sync = []
     peer_service_actions = {'restart': []}
     peer_actions = []
 
@@ -1068,9 +1117,6 @@
         paths_to_sync.append(PKI_CERTS_DIR)
         peer_actions.append('ensure-pki-permissions')
 
-    # Ensure unique
-    paths_to_sync = list(set(paths_to_sync))
-
     if not paths_to_sync:
         log("Nothing to sync - skipping", level=DEBUG)
         return {}
@@ -1083,50 +1129,27 @@
 
     create_peer_actions(peer_actions)
 
-    cluster_rel_settings = {}
-
-    retries = 3
-    while True:
-        hash1 = hashlib.sha256()
-        for path in paths_to_sync:
-            update_hash_from_path(hash1, path)
-
-        try:
-            synced_units = unison_sync(paths_to_sync)
-            if synced_units:
-                # Format here needs to match that used when peers request sync
-                synced_units = [u.replace('/', '-') for u in synced_units]
-                cluster_rel_settings['ssl-synced-units'] = \
-                    json.dumps(synced_units)
-        except Exception as exc:
-            if fatal:
-                raise
-            else:
-                log("Sync failed but fatal=False - %s" % (exc), level=INFO)
-                return {}
-
-        hash2 = hashlib.sha256()
-        for path in paths_to_sync:
-            update_hash_from_path(hash2, path)
-
-        # Detect whether someone else has synced to this unit while we did our
-        # transfer.
-        if hash1.hexdigest() != hash2.hexdigest():
-            retries -= 1
-            if retries > 0:
-                log("SSL dir contents changed during sync - retrying unison "
-                    "sync %s more times" % (retries), level=WARNING)
-            else:
-                log("SSL dir contents changed during sync - retries failed",
-                    level=ERROR)
-                return {}
-        else:
-            break
-
-    hash = hash1.hexdigest()
-    log("Sending restart-services-trigger=%s to all peers" % (hash),
+    paths_to_sync = list(set(paths_to_sync))
+    stage_paths_for_sync(paths_to_sync)
+
+    hash1 = hashlib.sha256()
+    for path in paths_to_sync:
+        update_hash_from_path(hash1, path)
+
+    cluster_rel_settings = {'ssl-cert-available-updates': SSL_SYNC_ARCHIVE,
+                            'sync-hash': hash1.hexdigest()}
+
+    synced_units = unison_sync([SSL_SYNC_ARCHIVE, SYNC_FLAGS_DIR])
+    if synced_units:
+        # Format here needs to match that used when peers request sync
+        synced_units = [u.replace('/', '-') for u in synced_units]
+        cluster_rel_settings['ssl-synced-units'] = \
+            json.dumps(synced_units)
+
+    trigger = str(uuid.uuid4())
+    log("Sending restart-services-trigger=%s to all peers" % (trigger),
         level=DEBUG)
-    cluster_rel_settings['restart-services-trigger'] = hash
+    cluster_rel_settings['restart-services-trigger'] = trigger
 
     log("Sync complete", level=DEBUG)
     return cluster_rel_settings

=== added symlink 'hooks/leader-settings-changed'
=== target is u'keystone_hooks.py'
=== modified file 'unit_tests/test_keystone_hooks.py'
--- unit_tests/test_keystone_hooks.py	2015-04-17 12:10:54 +0000
+++ unit_tests/test_keystone_hooks.py	2015-06-04 08:44:44 +0000
@@ -59,6 +59,9 @@
     'add_service_to_keystone',
     'synchronize_ca_if_changed',
     'update_nrpe_config',
+    'ensure_ssl_dirs',
+    'is_db_initialised',
+    'is_db_ready',
     # other
     'check_call',
     'execd_preinstall',
@@ -237,18 +240,15 @@
         configs.write = MagicMock()
         hooks.pgsql_db_changed()
 
-    @patch.object(hooks, 'is_db_initialised')
-    @patch.object(hooks, 'is_db_ready')
     @patch('keystone_utils.log')
     @patch('keystone_utils.ensure_ssl_cert_master')
     @patch.object(hooks, 'CONFIGS')
     @patch.object(hooks, 'identity_changed')
     def test_db_changed_allowed(self, identity_changed, configs,
                                 mock_ensure_ssl_cert_master,
-                                mock_log, mock_is_db_ready,
-                                mock_is_db_initialised):
-        mock_is_db_initialised.return_value = True
-        mock_is_db_ready.return_value = True
+                                mock_log):
+        self.is_db_initialised.return_value = True
+        self.is_db_ready.return_value = True
         mock_ensure_ssl_cert_master.return_value = False
         self.relation_ids.return_value = ['identity-service:0']
         self.related_units.return_value = ['unit/0']
@@ -262,15 +262,13 @@
             relation_id='identity-service:0',
             remote_unit='unit/0')
 
-    @patch.object(hooks, 'is_db_ready')
     @patch('keystone_utils.log')
     @patch('keystone_utils.ensure_ssl_cert_master')
     @patch.object(hooks, 'CONFIGS')
     @patch.object(hooks, 'identity_changed')
     def test_db_changed_not_allowed(self, identity_changed, configs,
-                                    mock_ensure_ssl_cert_master, mock_log,
-                                    mock_is_db_ready):
-        mock_is_db_ready.return_value = False
+                                    mock_ensure_ssl_cert_master, mock_log):
+        self.is_db_ready.return_value = False
         mock_ensure_ssl_cert_master.return_value = False
         self.relation_ids.return_value = ['identity-service:0']
         self.related_units.return_value = ['unit/0']
@@ -284,15 +282,12 @@
 
     @patch('keystone_utils.log')
     @patch('keystone_utils.ensure_ssl_cert_master')
-    @patch.object(hooks, 'is_db_initialised')
-    @patch.object(hooks, 'is_db_ready')
     @patch.object(hooks, 'CONFIGS')
     @patch.object(hooks, 'identity_changed')
     def test_postgresql_db_changed(self, identity_changed, configs,
-                                   mock_is_db_ready, mock_is_db_initialised,
                                    mock_ensure_ssl_cert_master, mock_log):
-        mock_is_db_initialised.return_value = True
-        mock_is_db_ready.return_value = True
+        self.is_db_initialised.return_value = True
+        self.is_db_ready.return_value = True
         mock_ensure_ssl_cert_master.return_value = False
         self.relation_ids.return_value = ['identity-service:0']
         self.related_units.return_value = ['unit/0']
@@ -309,15 +304,13 @@
     @patch.object(hooks, 'git_install_requested')
     @patch('keystone_utils.log')
     @patch('keystone_utils.ensure_ssl_cert_master')
+    @patch('keystone_utils.ensure_ssl_dirs')
     @patch.object(hooks, 'ensure_pki_dir_permissions')
     @patch.object(hooks, 'ensure_ssl_dir')
     @patch.object(hooks, 'is_pki_enabled')
     @patch.object(hooks, 'is_ssl_cert_master')
     @patch.object(hooks, 'send_ssl_sync_request')
-    @patch.object(hooks, 'is_db_initialised')
-    @patch.object(hooks, 'is_db_ready')
     @patch.object(hooks, 'peer_units')
-    @patch.object(hooks, 'ensure_permissions')
     @patch.object(hooks, 'admin_relation_changed')
     @patch.object(hooks, 'cluster_joined')
     @patch.object(unison, 'ensure_user')
@@ -331,22 +324,20 @@
                                               ensure_user,
                                               cluster_joined,
                                               admin_relation_changed,
-                                              ensure_permissions,
                                               mock_peer_units,
-                                              mock_is_db_ready,
-                                              mock_is_db_initialised,
                                               mock_send_ssl_sync_request,
                                               mock_is_ssl_cert_master,
                                               mock_is_pki_enabled,
                                               mock_ensure_ssl_dir,
                                               mock_ensure_pki_dir_permissions,
+                                              mock_ensure_ssl_dirs,
                                               mock_ensure_ssl_cert_master,
                                               mock_log, git_requested):
         git_requested.return_value = False
         mock_is_pki_enabled.return_value = True
         mock_is_ssl_cert_master.return_value = True
-        mock_is_db_initialised.return_value = True
-        mock_is_db_ready.return_value = True
+        self.is_db_initialised.return_value = True
+        self.is_db_ready.return_value = True
         self.openstack_upgrade_available.return_value = False
         self.is_elected_leader.return_value = True
         # avoid having to mock syncer
@@ -374,13 +365,13 @@
     @patch.object(hooks, 'git_install_requested')
     @patch('keystone_utils.log')
     @patch('keystone_utils.ensure_ssl_cert_master')
+    @patch('keystone_utils.ensure_ssl_dirs')
     @patch.object(hooks, 'update_all_identity_relation_units')
     @patch.object(hooks, 'ensure_pki_dir_permissions')
     @patch.object(hooks, 'ensure_ssl_dir')
     @patch.object(hooks, 'is_pki_enabled')
     @patch.object(hooks, 'peer_units')
     @patch.object(hooks, 'is_ssl_cert_master')
-    @patch.object(hooks, 'ensure_permissions')
     @patch.object(hooks, 'cluster_joined')
     @patch.object(unison, 'ensure_user')
     @patch.object(unison, 'get_homedir')
@@ -391,13 +382,13 @@
                                                   identity_changed,
                                                   configs, get_homedir,
                                                   ensure_user, cluster_joined,
-                                                  ensure_permissions,
                                                   mock_is_ssl_cert_master,
                                                   mock_peer_units,
                                                   mock_is_pki_enabled,
                                                   mock_ensure_ssl_dir,
                                                   mock_ensure_pki_permissions,
                                                   mock_update_all_id_rel_units,
+                                                  ensure_ssl_dirs,
                                                   mock_ensure_ssl_cert_master,
                                                   mock_log, git_requested):
         git_requested.return_value = False
@@ -423,15 +414,13 @@
     @patch.object(hooks, 'git_install_requested')
     @patch('keystone_utils.log')
     @patch('keystone_utils.ensure_ssl_cert_master')
+    @patch('keystone_utils.ensure_ssl_dirs')
     @patch.object(hooks, 'ensure_pki_dir_permissions')
     @patch.object(hooks, 'ensure_ssl_dir')
     @patch.object(hooks, 'is_pki_enabled')
     @patch.object(hooks, 'is_ssl_cert_master')
     @patch.object(hooks, 'send_ssl_sync_request')
-    @patch.object(hooks, 'is_db_initialised')
-    @patch.object(hooks, 'is_db_ready')
     @patch.object(hooks, 'peer_units')
-    @patch.object(hooks, 'ensure_permissions')
     @patch.object(hooks, 'admin_relation_changed')
     @patch.object(hooks, 'cluster_joined')
     @patch.object(unison, 'ensure_user')
@@ -444,22 +433,20 @@
                                                    configs, get_homedir,
                                                    ensure_user, cluster_joined,
                                                    admin_relation_changed,
-                                                   ensure_permissions,
                                                    mock_peer_units,
-                                                   mock_is_db_ready,
-                                                   mock_is_db_initialised,
                                                    mock_send_ssl_sync_request,
                                                    mock_is_ssl_cert_master,
                                                    mock_is_pki_enabled,
                                                    mock_ensure_ssl_dir,
                                                    mock_ensure_pki_permissions,
+                                                   mock_ensure_ssl_dirs,
                                                    mock_ensure_ssl_cert_master,
                                                    mock_log, git_requested):
         git_requested.return_value = False
         mock_is_pki_enabled.return_value = True
         mock_is_ssl_cert_master.return_value = True
-        mock_is_db_ready.return_value = True
-        mock_is_db_initialised.return_value = True
+        self.is_db_ready.return_value = True
+        self.is_db_initialised.return_value = True
         self.openstack_upgrade_available.return_value = True
         self.is_elected_leader.return_value = True
         # avoid having to mock syncer
@@ -496,7 +483,6 @@
     @patch.object(hooks, 'is_db_initialised')
     @patch.object(hooks, 'is_db_ready')
     @patch.object(hooks, 'peer_units')
-    @patch.object(hooks, 'ensure_permissions')
     @patch.object(hooks, 'admin_relation_changed')
     @patch.object(hooks, 'cluster_joined')
     @patch.object(unison, 'ensure_user')
@@ -508,7 +494,7 @@
                                         identity_changed,
                                         configs, get_homedir, ensure_user,
                                         cluster_joined, admin_relation_changed,
-                                        ensure_permissions, mock_peer_units,
+                                        mock_peer_units,
                                         mock_is_db_ready,
                                         mock_is_db_initialised,
                                         mock_send_ssl_sync_request,
@@ -546,18 +532,15 @@
         self.assertFalse(self.openstack_upgrade_available.called)
         self.assertFalse(self.do_openstack_upgrade.called)
 
-    @patch.object(hooks, 'is_db_initialised')
-    @patch.object(hooks, 'is_db_ready')
     @patch('keystone_utils.log')
     @patch('keystone_utils.ensure_ssl_cert_master')
     @patch.object(hooks, 'hashlib')
     @patch.object(hooks, 'send_notifications')
     def test_identity_changed_leader(self, mock_send_notifications,
                                      mock_hashlib, mock_ensure_ssl_cert_master,
-                                     mock_log, mock_is_db_ready,
-                                     mock_is_db_initialised):
-        mock_is_db_initialised.return_value = True
-        mock_is_db_ready.return_value = True
+                                     mock_log):
+        self.is_db_initialised.return_value = True
+        self.is_db_ready.return_value = True
         mock_ensure_ssl_cert_master.return_value = False
         hooks.identity_changed(
             relation_id='identity-service:0',
@@ -597,6 +580,7 @@
     @patch.object(hooks, 'get_ssl_sync_request_units')
     @patch.object(hooks, 'is_ssl_cert_master')
     @patch.object(hooks, 'peer_units')
+    @patch('keystone_utils.relation_ids')
     @patch('keystone_utils.config')
     @patch('keystone_utils.log')
     @patch('keystone_utils.ensure_ssl_cert_master')
@@ -607,7 +591,8 @@
     def test_cluster_changed(self, configs, ssh_authorized_peers,
                              check_peer_actions, mock_synchronize_ca,
                              mock_ensure_ssl_cert_master,
-                             mock_log, mock_config, mock_peer_units,
+                             mock_log, mock_config, mock_relation_ids,
+                             mock_peer_units,
                              mock_is_ssl_cert_master,
                              mock_get_ssl_sync_request_units,
                              mock_update_all_identity_relation_units):
@@ -618,6 +603,7 @@
         mock_is_ssl_cert_master.return_value = False
         mock_peer_units.return_value = ['unit/0']
         mock_ensure_ssl_cert_master.return_value = False
+        mock_relation_ids.return_value = []
         self.is_elected_leader.return_value = False
 
         def fake_rel_get(attribute=None, *args, **kwargs):
@@ -632,8 +618,8 @@
 
         hooks.cluster_changed()
         whitelist = ['_passwd', 'identity-service:', 'ssl-cert-master',
-                     'db-initialised']
-        self.peer_echo.assert_called_with(includes=whitelist)
+                     'db-initialised', 'ssl-cert-available-updates']
+        self.peer_echo.assert_called_with(force=True, includes=whitelist)
         ssh_authorized_peers.assert_called_with(
             user=self.ssh_user, group='juju_keystone',
             peer_interface='cluster', ensure_local_user=True)
@@ -733,18 +719,14 @@
 
     @patch('keystone_utils.log')
     @patch('keystone_utils.ensure_ssl_cert_master')
-    @patch.object(hooks, 'is_db_ready')
-    @patch.object(hooks, 'is_db_initialised')
     @patch.object(hooks, 'identity_changed')
     @patch.object(hooks, 'CONFIGS')
     def test_ha_relation_changed_clustered_leader(self, configs,
                                                   identity_changed,
-                                                  mock_is_db_initialised,
-                                                  mock_is_db_ready,
                                                   mock_ensure_ssl_cert_master,
                                                   mock_log):
-        mock_is_db_initialised.return_value = True
-        mock_is_db_ready.return_value = True
+        self.is_db_initialised.return_value = True
+        self.is_db_ready.return_value = True
         mock_ensure_ssl_cert_master.return_value = False
         self.relation_get.return_value = True
         self.is_elected_leader.return_value = True
@@ -807,8 +789,8 @@
                                   mock_is_elected_leader,
                                   mock_relation_ids,
                                   mock_log,
+                                  mock_is_db_initialised,
                                   mock_is_db_ready,
-                                  mock_is_db_initialised,
                                   git_requested):
         mock_is_db_initialised.return_value = True
         mock_is_db_ready.return_value = True