Merge lp:~hopem/charms/trusty/keystone/fix-pki-token-support into lp:~openstack-charmers-archive/charms/trusty/keystone/next

[Edward Hope-Morley]

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #2133 keystone-next for hopem mp249881
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/2133/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #1922 keystone-next for hopem mp249881
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/1922/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #2041 keystone-next for hopem mp249881
    AMULET OK: passed

Build: http://10.245.162.77:8080/job/charm_amulet_test/2041/

[lp:~hopem/charms/trusty/keystone/fix-pki-token-support]

123. By Edward Hope-Morley on 2015-02-24

synced /next

124. By Edward Hope-Morley on 2015-02-24

synced charm-helpers;

125. By Edward Hope-Morley on 2015-02-24

don't push race-prone data on cluster relation to avoid spinning

126. By Edward Hope-Morley on 2015-02-24

remove pkiz

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #2206 keystone-next for hopem mp249881
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/2206/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #1995 keystone-next for hopem mp249881
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/1995/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #2152 keystone-next for hopem mp249881
    AMULET FAIL: amulet-test failed

AMULET Results (max last 2 lines):
  ERROR subprocess encountered error code 1
  make: *** [test] Error 1

Full amulet test output: http://paste.ubuntu.com/10396276/
Build: http://10.245.162.77:8080/job/charm_amulet_test/2152/

[lp:~hopem/charms/trusty/keystone/fix-pki-token-support]

127. By Edward Hope-Morley on 2015-03-10

synced /next

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #2560 keystone-next for hopem mp249881
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/2560/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #2350 keystone-next for hopem mp249881
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/2350/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #2429 keystone-next for hopem mp249881
    AMULET FAIL: juju-pull-logs failed

AMULET Results (max last 2 lines):
  WARNING cannot delete security group "juju-osci-sv07-0". Used by another environment?
  juju-test INFO : Results: 3 passed, 0 failed, 0 errored

Full amulet test output: http://paste.ubuntu.com/10575545/
Build: http://10.245.162.77:8080/job/charm_amulet_test/2429/

[lp:~hopem/charms/trusty/keystone/fix-pki-token-support]

128. By Edward Hope-Morley on 2015-03-11

cleanuo

[Liam Young (gnuoy)]

Approve

review: Approve

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #2604 keystone-next for hopem mp249881
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/2604/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #2394 keystone-next for hopem mp249881
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/2394/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #2432 keystone-next for hopem mp249881
    AMULET OK: passed

Build: http://10.245.162.77:8080/job/charm_amulet_test/2432/

=== modified file 'hooks/keystone_context.py'
--- hooks/keystone_context.py	2015-02-16 11:38:34 +0000
+++ hooks/keystone_context.py	2015-03-11 14:46:50 +0000
@@ -18,6 +18,7 @@
 
 from charmhelpers.core.hookenv import (
     log,
+    DEBUG,
     INFO,
 )
 
@@ -173,9 +174,8 @@
 
     def __call__(self):
         from keystone_utils import (
-            api_port, set_admin_token,
-            endpoint_url, resolve_address,
-            PUBLIC, ADMIN
+            api_port, set_admin_token, endpoint_url, resolve_address,
+            PUBLIC, ADMIN, PKI_CERTS_DIR, SSH_USER, ensure_permissions,
         )
         ctxt = {}
         ctxt['token'] = set_admin_token(config('admin-token'))
@@ -205,6 +205,31 @@
         enable_pki = config('enable-pki')
         if enable_pki and bool_from_string(enable_pki):
             ctxt['signing'] = True
+            ctxt['token_provider'] = 'pki'
+
+        if 'token_provider' in ctxt:
+            log("Configuring PKI token cert paths", level=DEBUG)
+            certs = os.path.join(PKI_CERTS_DIR, 'certs')
+            privates = os.path.join(PKI_CERTS_DIR, 'privates')
+            for path in [PKI_CERTS_DIR, certs, privates]:
+                perms = 0o755
+                if not os.path.isdir(path):
+                    mkdir(path=path, owner=SSH_USER, group='keystone',
+                          perms=perms)
+                else:
+                    # Ensure accessible by ssh user and group (for sync).
+                    ensure_permissions(path, user=SSH_USER,
+                                       group='keystone', perms=perms)
+
+            signing_paths = {'certfile': os.path.join(certs,
+                                                      'signing_cert.pem'),
+                             'keyfile': os.path.join(privates,
+                                                     'signing_key.pem'),
+                             'ca_certs': os.path.join(certs, 'ca.pem'),
+                             'ca_key': os.path.join(certs, 'ca_key.pem')}
+
+            for key, val in signing_paths.iteritems():
+                ctxt[key] = val
 
         # Base endpoint URL's which are used in keystone responses
         # to unauthenticated requests to redirect clients to the

=== modified file 'hooks/keystone_hooks.py'
--- hooks/keystone_hooks.py	2015-03-10 12:02:11 +0000
+++ hooks/keystone_hooks.py	2015-03-11 14:46:50 +0000
@@ -72,6 +72,10 @@
     is_db_ready,
     clear_ssl_synced_units,
     is_db_initialised,
+    is_pki_enabled,
+    ensure_ssl_dir,
+    ensure_pki_dir_permissions,
+    force_ssl_sync,
     filter_null,
 )
 
@@ -115,7 +119,7 @@
 
 @hooks.hook('config-changed')
 @restart_on_change(restart_map())
-@synchronize_ca_if_changed()
+@synchronize_ca_if_changed(fatal=True)
 def config_changed():
     if config('prefer-ipv6'):
         setup_ipv6()
@@ -131,6 +135,9 @@
     if openstack_upgrade_available('keystone'):
         do_openstack_upgrade(configs=CONFIGS)
 
+    # Ensure ssl dir exists and is unison-accessible
+    ensure_ssl_dir()
+
     check_call(['chmod', '-R', 'g+wrx', '/var/lib/keystone/'])
 
     # Ensure unison can write to certs dir.
@@ -143,12 +150,16 @@
 
     save_script_rc()
     configure_https()
+
     update_nrpe_config()
     CONFIGS.write_all()
 
+    if is_pki_enabled():
+        initialise_pki()
+
     # Update relations since SSL may have been configured. If we have peer
     # units we can rely on the sync to do this in cluster relation.
-    if is_elected_leader(CLUSTER_RES) and not peer_units():
+    if not peer_units():
         update_all_identity_relation_units()
 
     for rid in relation_ids('identity-admin'):
@@ -161,6 +172,22 @@
         ha_joined(relation_id=r_id)
 
 
+@synchronize_ca_if_changed(fatal=True)
+def initialise_pki():
+    """Create certs and keys required for PKI token signing.
+
+    NOTE: keystone.conf [signing] section must be up-to-date prior to
+          executing this.
+    """
+    if is_ssl_cert_master():
+        log("Ensuring PKI token certs created", level=DEBUG)
+        cmd = ['keystone-manage', 'pki_setup', '--keystone-user', 'keystone',
+               '--keystone-group', 'keystone']
+        check_call(cmd)
+
+    ensure_pki_dir_permissions()
+
+
 @hooks.hook('shared-db-relation-joined')
 def db_joined():
     if is_relation_made('pgsql-db'):
@@ -301,6 +328,7 @@
             peerdb_settings = filter_null(peerdb_settings)
             if 'service_password' in peerdb_settings:
                 relation_set(relation_id=rel_id, **peerdb_settings)
+
         log('Deferring identity_changed() to service leader.')
 
     if notifications:
@@ -319,12 +347,20 @@
     """
     unit = local_unit().replace('/', '-')
     count = 0
-    if bool_from_string(config('use-https')):
+
+    use_https = config('use-https')
+    if use_https and bool_from_string(use_https):
         count += 1
 
-    if bool_from_string(config('https-service-endpoints')):
+    https_service_endpoints = config('https-service-endpoints')
+    if (https_service_endpoints and
+            bool_from_string(https_service_endpoints)):
         count += 2
 
+    enable_pki = config('enable-pki')
+    if enable_pki and bool_from_string(enable_pki):
+        count += 3
+
     key = 'ssl-sync-required-%s' % (unit)
     settings = {key: count}
 
@@ -391,23 +427,32 @@
 
     check_peer_actions()
 
-    if is_elected_leader(CLUSTER_RES) or is_ssl_cert_master():
-        units = get_ssl_sync_request_units()
-        synced_units = relation_get(attribute='ssl-synced-units',
-                                    unit=local_unit())
-        if synced_units:
-            synced_units = json.loads(synced_units)
-            diff = set(units).symmetric_difference(set(synced_units))
-
-        if units and (not synced_units or diff):
-            log("New peers joined and need syncing - %s" %
-                (', '.join(units)), level=DEBUG)
-            update_all_identity_relation_units_force_sync()
-        else:
-            update_all_identity_relation_units()
-
-        for rid in relation_ids('identity-admin'):
-            admin_relation_changed(rid)
+    if is_pki_enabled():
+        initialise_pki()
+
+    # Figure out if we need to mandate a sync
+    units = get_ssl_sync_request_units()
+    synced_units = relation_get(attribute='ssl-synced-units',
+                                unit=local_unit())
+    diff = None
+    if synced_units:
+        synced_units = json.loads(synced_units)
+        diff = set(units).symmetric_difference(set(synced_units))
+
+    if units and (not synced_units or diff):
+        log("New peers joined and need syncing - %s" %
+            (', '.join(units)), level=DEBUG)
+        update_all_identity_relation_units_force_sync()
+    else:
+        update_all_identity_relation_units()
+
+    for rid in relation_ids('identity-admin'):
+        admin_relation_changed(rid)
+
+    if not is_elected_leader(CLUSTER_RES) and is_ssl_cert_master():
+        # Force and sync and trigger a sync master re-election since we are not
+        # leader anymore.
+        force_ssl_sync()
     else:
         CONFIGS.write_all()
 

=== modified file 'hooks/keystone_ssl.py'
--- hooks/keystone_ssl.py	2015-01-13 11:04:56 +0000
+++ hooks/keystone_ssl.py	2015-03-11 14:46:50 +0000
@@ -113,15 +113,16 @@
 
 
 def init_ca(ca_dir, common_name, org_name=ORG_NAME, org_unit_name=ORG_UNIT):
-    print 'Ensuring certificate authority exists at %s.' % ca_dir
+    log('Ensuring certificate authority exists at %s.' % ca_dir, level=DEBUG)
     if not os.path.exists(ca_dir):
-        print 'Initializing new certificate authority at %s' % ca_dir
+        log('Initializing new certificate authority at %s' % ca_dir,
+            level=DEBUG)
         os.mkdir(ca_dir)
 
     for i in ['certs', 'crl', 'newcerts', 'private']:
         d = os.path.join(ca_dir, i)
         if not os.path.exists(d):
-            print 'Creating %s.' % d
+            log('Creating %s.' % d, level=DEBUG)
             os.mkdir(d)
     os.chmod(os.path.join(ca_dir, 'private'), 0o710)
 
@@ -132,9 +133,11 @@
     if not os.path.isfile(os.path.join(ca_dir, 'index.txt')):
         with open(os.path.join(ca_dir, 'index.txt'), 'wb') as out:
             out.write('')
-    if not os.path.isfile(os.path.join(ca_dir, 'ca.cnf')):
-        print 'Creating new CA config in %s' % ca_dir
-        with open(os.path.join(ca_dir, 'ca.cnf'), 'wb') as out:
+
+    conf = os.path.join(ca_dir, 'ca.cnf')
+    if not os.path.isfile(conf):
+        log('Creating new CA config in %s' % ca_dir, level=DEBUG)
+        with open(conf, 'wb') as out:
             out.write(CA_CONFIG % locals())
 
 
@@ -144,40 +147,42 @@
     key = os.path.join(ca_dir, 'private', 'cacert.key')
     for f in [crt, key]:
         if not os.path.isfile(f):
-            print 'Missing %s, will re-initialize cert+key.' % f
+            log('Missing %s, will re-initialize cert+key.' % f, level=DEBUG)
             init = True
         else:
-            print 'Found %s.' % f
+            log('Found %s.' % f, level=DEBUG)
+
     if init:
-        cmd = ['openssl', 'req', '-config', os.path.join(ca_dir, 'ca.cnf'),
+        conf = os.path.join(ca_dir, 'ca.cnf')
+        cmd = ['openssl', 'req', '-config', conf,
                '-x509', '-nodes', '-newkey', 'rsa', '-days', '21360',
                '-keyout', key, '-out', crt, '-outform', 'PEM']
         subprocess.check_call(cmd)
+
     return crt, key
 
 
 def intermediate_ca_csr_key(ca_dir):
-    print 'Creating new intermediate CSR.'
+    log('Creating new intermediate CSR.', level=DEBUG)
     key = os.path.join(ca_dir, 'private', 'cacert.key')
     csr = os.path.join(ca_dir, 'cacert.csr')
-    cmd = ['openssl', 'req', '-config', os.path.join(ca_dir, 'ca.cnf'),
-           '-sha1', '-newkey', 'rsa', '-nodes', '-keyout', key, '-out',
-           csr, '-outform',
-           'PEM']
+    conf = os.path.join(ca_dir, 'ca.cnf')
+    cmd = ['openssl', 'req', '-config', conf, '-sha1', '-newkey', 'rsa',
+           '-nodes', '-keyout', key, '-out', csr, '-outform', 'PEM']
     subprocess.check_call(cmd)
     return csr, key
 
 
 def sign_int_csr(ca_dir, csr, common_name):
-    print 'Signing certificate request %s.' % csr
-    crt = os.path.join(ca_dir, 'certs',
-                       '%s.crt' % os.path.basename(csr).split('.')[0])
+    log('Signing certificate request %s.' % csr, level=DEBUG)
+    crt_name = os.path.basename(csr).split('.')[0]
+    crt = os.path.join(ca_dir, 'certs', '%s.crt' % crt_name)
     subj = '/O=%s/OU=%s/CN=%s' % (ORG_NAME, ORG_UNIT, common_name)
-    cmd = ['openssl', 'ca', '-batch', '-config',
-           os.path.join(ca_dir, 'ca.cnf'),
-           '-extensions', 'ca_extensions', '-days', CA_EXPIRY, '-notext',
-           '-in', csr, '-out', crt, '-subj', subj, '-batch']
-    print ' '.join(cmd)
+    conf = os.path.join(ca_dir, 'ca.cnf')
+    cmd = ['openssl', 'ca', '-batch', '-config', conf, '-extensions',
+           'ca_extensions', '-days', CA_EXPIRY, '-notext', '-in', csr, '-out',
+           crt, '-subj', subj, '-batch']
+    log("Executing: %s" % ' '.join(cmd), level=DEBUG)
     subprocess.check_call(cmd)
     return crt
 
@@ -187,19 +192,20 @@
     return root_ca_crt_key(ca_dir)
 
 
-def init_intermediate_ca(ca_dir, common_name, root_ca_dir,
-                         org_name=ORG_NAME, org_unit_name=ORG_UNIT):
+def init_intermediate_ca(ca_dir, common_name, root_ca_dir, org_name=ORG_NAME,
+                         org_unit_name=ORG_UNIT):
     init_ca(ca_dir, common_name)
     if not os.path.isfile(os.path.join(ca_dir, 'cacert.pem')):
         csr, key = intermediate_ca_csr_key(ca_dir)
         crt = sign_int_csr(root_ca_dir, csr, common_name)
         shutil.copy(crt, os.path.join(ca_dir, 'cacert.pem'))
     else:
-        print 'Intermediate CA certificate already exists.'
+        log('Intermediate CA certificate already exists.', level=DEBUG)
 
-    if not os.path.isfile(os.path.join(ca_dir, 'signing.cnf')):
-        print 'Creating new signing config in %s' % ca_dir
-        with open(os.path.join(ca_dir, 'signing.cnf'), 'wb') as out:
+    conf = os.path.join(ca_dir, 'signing.cnf')
+    if not os.path.isfile(conf):
+        log('Creating new signing config in %s' % ca_dir, level=DEBUG)
+        with open(conf, 'wb') as out:
             out.write(SIGNING_CONFIG % locals())
 
 
@@ -212,7 +218,7 @@
            key, '-out', csr, '-subj', subj]
     subprocess.check_call(cmd)
     crt = sign_int_csr(ca_dir, csr, common_name)
-    print 'Signed new CSR, crt @ %s' % crt
+    log('Signed new CSR, crt @ %s' % crt, level=DEBUG)
     return
 
 
@@ -221,13 +227,14 @@
     if os.path.isfile(bundle_file):
         current = open(bundle_file, 'r').read().strip()
         if new_bundle == current:
-            print 'CA Bundle @ %s is up to date.' % bundle_file
+            log('CA Bundle @ %s is up to date.' % bundle_file, level=DEBUG)
             return
-        else:
-            print 'Updating CA bundle @ %s.' % bundle_file
+
+        log('Updating CA bundle @ %s.' % bundle_file, level=DEBUG)
 
     with open(bundle_file, 'wb') as out:
         out.write(new_bundle)
+
     subprocess.check_call(['update-ca-certificates'])
 
 
@@ -250,15 +257,19 @@
 class JujuCA(object):
 
     def __init__(self, name, ca_dir, root_ca_dir, user, group):
-        root_crt, root_key = init_root_ca(root_ca_dir,
-                                          '%s Certificate Authority' % name)
-        init_intermediate_ca(ca_dir,
-                             '%s Intermediate Certificate Authority' % name,
-                             root_ca_dir)
+        # Root CA
+        cn = '%s Certificate Authority' % name
+        root_crt, root_key = init_root_ca(root_ca_dir, cn)
+        # Intermediate CA
+        cn = '%s Intermediate Certificate Authority' % name
+        init_intermediate_ca(ca_dir, cn, root_ca_dir)
+
+        # Create dirs
         cmd = ['chown', '-R', '%s.%s' % (user, group), ca_dir]
         subprocess.check_call(cmd)
         cmd = ['chown', '-R', '%s.%s' % (user, group), root_ca_dir]
         subprocess.check_call(cmd)
+
         self.ca_dir = ca_dir
         self.root_ca_dir = root_ca_dir
         self.user = user
@@ -268,8 +279,8 @@
     def _sign_csr(self, csr, service, common_name):
         subj = '/O=%s/OU=%s/CN=%s' % (ORG_NAME, ORG_UNIT, common_name)
         crt = os.path.join(self.ca_dir, 'certs', '%s.crt' % common_name)
-        cmd = ['openssl', 'ca', '-config',
-               os.path.join(self.ca_dir, 'signing.cnf'), '-extensions',
+        conf = os.path.join(self.ca_dir, 'signing.cnf')
+        cmd = ['openssl', 'ca', '-config', conf, '-extensions',
                'req_extensions', '-days', '365', '-notext', '-in', csr,
                '-out', crt, '-batch', '-subj', subj]
         subprocess.check_call(cmd)
@@ -288,10 +299,16 @@
         log('Signed new CSR, crt @ %s' % crt, level=DEBUG)
         return crt, key
 
+    def get_key_path(self, cn):
+        return os.path.join(self.ca_dir, 'certs', '%s.key' % cn)
+
+    def get_cert_path(self, cn):
+        return os.path.join(self.ca_dir, 'certs', '%s.crt' % cn)
+
     def get_cert_and_key(self, common_name):
         log('Getting certificate and key for %s.' % common_name, level=DEBUG)
-        keypath = os.path.join(self.ca_dir, 'certs', '%s.key' % common_name)
-        crtpath = os.path.join(self.ca_dir, 'certs', '%s.crt' % common_name)
+        keypath = self.get_key_path(common_name)
+        crtpath = self.get_cert_path(common_name)
         if os.path.isfile(crtpath):
             log('Found existing certificate for %s.' % common_name,
                 level=DEBUG)
@@ -324,8 +341,24 @@
         crt, key = self._create_certificate(common_name, common_name)
         return open(crt, 'r').read(), open(key, 'r').read()
 
+    @property
+    def ca_cert_path(self):
+        return os.path.join(self.ca_dir, 'cacert.pem')
+
+    @property
+    def ca_key_path(self):
+        return os.path.join(self.ca_dir, 'private',  'cacert.key')
+
+    @property
+    def root_ca_cert_path(self):
+        return os.path.join(self.root_ca_dir, 'cacert.pem')
+
+    @property
+    def root_ca_key_path(self):
+        return os.path.join(self.root_ca_dir, 'private',  'cacert.key')
+
     def get_ca_bundle(self):
-        int_cert = open(os.path.join(self.ca_dir, 'cacert.pem')).read()
-        root_cert = open(os.path.join(self.root_ca_dir, 'cacert.pem')).read()
+        int_cert = open(self.ca_cert_path).read()
+        root_cert = open(self.root_ca_cert_path).read()
         # NOTE: ordering of certs in bundle matters!
         return int_cert + root_cert

=== modified file 'hooks/keystone_utils.py'
--- hooks/keystone_utils.py	2015-03-11 11:57:29 +0000
+++ hooks/keystone_utils.py	2015-03-11 14:46:50 +0000
@@ -137,10 +137,13 @@
 APACHE_SSL_DIR = '/etc/apache2/ssl/keystone'
 SYNC_FLAGS_DIR = '/var/lib/keystone/juju_sync_flags/'
 SSL_DIR = '/var/lib/keystone/juju_ssl/'
+PKI_CERTS_DIR = os.path.join(SSL_DIR, 'pki')
 SSL_CA_NAME = 'Ubuntu Cloud'
 CLUSTER_RES = 'grp_ks_vips'
 SSH_USER = 'juju_keystone'
+CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
 SSL_SYNC_SEMAPHORE = threading.Semaphore()
+SSL_DIRS = [SSL_DIR, APACHE_SSL_DIR, CA_CERT_PATH]
 
 BASE_RESOURCE_MAP = OrderedDict([
     (KEYSTONE_CONF, {
@@ -172,8 +175,6 @@
     }),
 ])
 
-CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
-
 valid_services = {
     "nova": {
         "type": "compute",
@@ -706,7 +707,8 @@
     return passwd
 
 
-def ensure_permissions(path, user=None, group=None, perms=None):
+def ensure_permissions(path, user=None, group=None, perms=None, recurse=False,
+                       maxdepth=50):
     """Set chownand chmod for path
 
     Note that -1 for uid or gid result in no change.
@@ -726,6 +728,16 @@
     if perms:
         os.chmod(path, perms)
 
+    if recurse:
+        if not maxdepth:
+            log("Max recursion depth reached - skipping further recursion")
+            return
+
+        paths = glob.glob("%s/*" % (path))
+        for path in paths:
+            ensure_permissions(path, user=user, group=group, perms=perms,
+                               recurse=recurse, maxdepth=maxdepth - 1)
+
 
 def check_peer_actions():
     """Honour service action requests from sync master.
@@ -766,6 +778,9 @@
                 elif action == 'update-ca-certificates':
                     log("Running %s" % (action), level=DEBUG)
                     subprocess.check_call(['update-ca-certificates'])
+                elif action == 'ensure-pki-permissions':
+                    log("Running %s" % (action), level=DEBUG)
+                    ensure_pki_dir_permissions()
                 else:
                     log("Unknown action flag=%s" % (flag), level=WARNING)
 
@@ -863,8 +878,12 @@
 
 
 def is_ssl_enabled():
-    if (bool_from_string(config('use-https')) or
-            bool_from_string(config('https-service-endpoints'))):
+    use_https = config('use-https')
+    https_service_endpoints = config('https-service-endpoints')
+    if ((use_https and bool_from_string(use_https)) or
+            (https_service_endpoints and
+                bool_from_string(https_service_endpoints)) or
+            is_pki_enabled()):
         log("SSL/HTTPS is enabled", level=DEBUG)
         return True
 
@@ -931,6 +950,20 @@
     return True
 
 
+def is_pki_enabled():
+    enable_pki = config('enable-pki')
+    if enable_pki and bool_from_string(enable_pki):
+        return True
+
+    return False
+
+
+def ensure_pki_dir_permissions():
+    # Ensure accessible by unison user and group (for sync).
+    ensure_permissions(PKI_CERTS_DIR, user=SSH_USER, group='keystone',
+                       perms=0o755, recurse=True)
+
+
 def synchronize_ca(fatal=False):
     """Broadcast service credentials to peers.
 
@@ -945,12 +978,18 @@
     Returns a dictionary of settings to be set on the cluster relation.
     """
     paths_to_sync = [SYNC_FLAGS_DIR]
+    peer_service_actions = []
+    peer_actions = []
 
     if bool_from_string(config('https-service-endpoints')):
         log("Syncing all endpoint certs since https-service-endpoints=True",
             level=DEBUG)
         paths_to_sync.append(SSL_DIR)
         paths_to_sync.append(CA_CERT_PATH)
+        # We need to restart peer apache services to ensure they have picked up
+        # new ssl keys.
+        peer_service_actions.append(('restart', ('apache2')))
+        peer_actions.append('update-ca-certificates')
 
     if bool_from_string(config('use-https')):
         log("Syncing keystone-endpoint certs since use-https=True",
@@ -958,6 +997,15 @@
         paths_to_sync.append(SSL_DIR)
         paths_to_sync.append(APACHE_SSL_DIR)
         paths_to_sync.append(CA_CERT_PATH)
+        # We need to restart peer apache services to ensure they have picked up
+        # new ssl keys.
+        peer_service_actions.append(('restart', ('apache2')))
+        peer_actions.append('update-ca-certificates')
+
+    if is_pki_enabled():
+        log("Syncing token certs", level=DEBUG)
+        paths_to_sync.append(PKI_CERTS_DIR)
+        peer_actions.append('ensure-pki-permissions')
 
     # Ensure unique
     paths_to_sync = list(set(paths_to_sync))
@@ -969,10 +1017,11 @@
     if not os.path.isdir(SYNC_FLAGS_DIR):
         mkdir(SYNC_FLAGS_DIR, SSH_USER, 'keystone', 0o775)
 
-    # We need to restart peer apache services to ensure they have picked up
-    # new ssl keys.
-    create_peer_service_actions('restart', ['apache2'])
-    create_peer_actions(['update-ca-certificates'])
+    for action, services in set(peer_service_actions):
+        create_peer_service_actions(action, services)
+
+    for action in set(peer_actions):
+        create_peer_actions(action)
 
     cluster_rel_settings = {}
 
@@ -989,11 +1038,11 @@
                 synced_units = [u.replace('/', '-') for u in synced_units]
                 cluster_rel_settings['ssl-synced-units'] = \
                     json.dumps(synced_units)
-        except:
+        except Exception as exc:
             if fatal:
                 raise
             else:
-                log("Sync failed but fatal=False", level=INFO)
+                log("Sync failed but fatal=False - %s" % (exc), level=INFO)
                 return {}
 
         hash2 = hashlib.sha256()
@@ -1069,21 +1118,19 @@
                     return f(*args, **kwargs)
 
                 if not ensure_ssl_cert_master():
-                    log("Not leader - ignoring sync", level=DEBUG)
+                    log("Not ssl-cert-master - ignoring sync", level=DEBUG)
                     return f(*args, **kwargs)
 
                 peer_settings = {}
                 if not force:
-                    ssl_dirs = [SSL_DIR, APACHE_SSL_DIR, CA_CERT_PATH]
-
                     hash1 = hashlib.sha256()
-                    for path in ssl_dirs:
+                    for path in SSL_DIRS:
                         update_hash_from_path(hash1, path)
 
                     ret = f(*args, **kwargs)
 
                     hash2 = hashlib.sha256()
-                    for path in ssl_dirs:
+                    for path in SSL_DIRS:
                         update_hash_from_path(hash2, path)
 
                     if hash1.hexdigest() != hash2.hexdigest():
@@ -1118,15 +1165,33 @@
     return inner_synchronize_ca_if_changed1
 
 
+@synchronize_ca_if_changed(force=True, fatal=True)
+def force_ssl_sync():
+    """Force SSL sync to all peers.
+
+    This is useful if we need to relinquish ssl-cert-master status while
+    making sure that the new master has up-to-date certs.
+    """
+    return
+
+
+def ensure_ssl_dir():
+    """Ensure juju ssl dir exists and is unsion read/writable."""
+    perms = 0o755
+    if not os.path.isdir(SSL_DIR):
+        mkdir(SSL_DIR, SSH_USER, 'keystone', perms)
+    else:
+        ensure_permissions(SSL_DIR, user=SSH_USER, group='keystone',
+                           perms=perms)
+
+
 def get_ca(user='keystone', group='keystone'):
     """Initialize a new CA object if one hasn't already been loaded.
 
     This will create a new CA or load an existing one.
     """
     if not ssl.CA_SINGLETON:
-        if not os.path.isdir(SSL_DIR):
-            os.mkdir(SSL_DIR)
-
+        ensure_ssl_dir()
         d_name = '_'.join(SSL_CA_NAME.lower().split(' '))
         ca = ssl.JujuCA(name=SSL_CA_NAME, user=user, group=group,
                         ca_dir=os.path.join(SSL_DIR,
@@ -1134,12 +1199,6 @@
                         root_ca_dir=os.path.join(SSL_DIR,
                                                  '%s_root_ca' % d_name))
 
-        # SSL_DIR is synchronized via all peers over unison+ssh, need
-        # to ensure permissions.
-        subprocess.check_output(['chown', '-R', '%s.%s' % (user, group),
-                                 '%s' % SSL_DIR])
-        subprocess.check_output(['chmod', '-R', 'g+rwx', '%s' % SSL_DIR])
-
         # Ensure a master is elected. This should cover the following cases:
         # * single unit == 'oldest' unit is elected as master
         # * multi unit + not clustered == 'oldest' unit is elcted as master
@@ -1184,9 +1243,13 @@
             # Some backend services advertise no endpoint but require a
             # hook execution to update auth strategy.
             relation_data = {}
+            rel_only_data = {}
             # Check if clustered and use vip + haproxy ports if so
-            relation_data["auth_host"] = resolve_address(ADMIN)
-            relation_data["service_host"] = resolve_address(PUBLIC)
+            # NOTE(hopem): don't put these on peer relation because racey
+            #              leader election causes cluster relation to spin)
+            rel_only_data["auth_host"] = resolve_address(ADMIN)
+            rel_only_data["service_host"] = resolve_address(PUBLIC)
+
             relation_data["auth_protocol"] = protocol
             relation_data["service_protocol"] = protocol
             relation_data["auth_port"] = config('admin-port')
@@ -1209,8 +1272,8 @@
                 log("Creating requested role: %s" % role)
                 create_role(role)
 
-            peer_store_and_set(relation_id=relation_id,
-                               **relation_data)
+            relation_set(relation_id=relation_id, **rel_only_data)
+            peer_store_and_set(relation_id=relation_id, **relation_data)
             return
         else:
             ensure_valid_service(settings['service'])
@@ -1314,13 +1377,16 @@
     # service credentials
     service_tenant = config('service-tenant')
 
+    # NOTE(hopem): don't put these on peer relation because racey
+    #              leader election causes cluster relation to spin)
+    rel_only_data = {"auth_host": resolve_address(ADMIN),
+                     "service_host": resolve_address(PUBLIC)}
+
     # NOTE(dosaboy): we use __null__ to represent settings that are to be
     # routed to relations via the cluster relation and set to None.
     relation_data = {
         "admin_token": token,
-        "service_host": resolve_address(PUBLIC),
         "service_port": config("service-port"),
-        "auth_host": resolve_address(ADMIN),
         "auth_port": config("admin-port"),
         "service_username": service_username,
         "service_password": service_password,
@@ -1353,6 +1419,7 @@
         relation_data['ca_cert'] = b64encode(ca_bundle)
         relation_data['https_keystone'] = 'True'
 
+    relation_set(relation_id=relation_id, **rel_only_data)
     # NOTE(dosaboy): '__null__' settings are for peer relation only so that
     # settings can flushed so we filter them out for non-peer relation.
     filtered = filter_null(relation_data)

=== modified file 'templates/icehouse/keystone.conf'
--- templates/icehouse/keystone.conf	2014-10-07 12:29:11 +0000
+++ templates/icehouse/keystone.conf	2015-03-11 14:46:50 +0000
@@ -43,7 +43,15 @@
 
 [token]
 driver = keystone.token.backends.sql.Token
-provider = keystone.token.providers.uuid.Provider
+{% if token_provider == 'pki' -%}
+provider = keystone.token.providers.pki.Provider
+{% elif token_provider == 'pkiz' -%}
+provider = keystone.token.providers.pkiz.Provider
+{% else -%}
+provider = keystone.token.providers.uuid.Provider 
+{% endif %}
+
+{% include "parts/section-signing" %}
 
 [cache]
 
@@ -58,8 +66,6 @@
 
 [oauth1]
 
-[signing]
-
 [auth]
 methods = external,password,token,oauth1
 password = keystone.auth.plugins.password.Password

=== added directory 'templates/parts'
=== added file 'templates/parts/section-signing'
--- templates/parts/section-signing	1970-01-01 00:00:00 +0000
+++ templates/parts/section-signing	2015-03-11 14:46:50 +0000
@@ -0,0 +1,13 @@
+[signing]
+{% if certfile -%}
+certfile = {{ certfile }}
+{% endif -%}
+{% if keyfile -%}
+keyfile = {{ keyfile }}
+{% endif -%}
+{% if ca_certs -%}
+ca_certs = {{ ca_certs }}
+{% endif -%}
+{% if ca_key -%}
+ca_key = {{ ca_key }}
+{% endif -%} 

=== modified file 'unit_tests/test_keystone_hooks.py'
--- unit_tests/test_keystone_hooks.py	2015-03-10 12:02:11 +0000
+++ unit_tests/test_keystone_hooks.py	2015-03-11 14:46:50 +0000
@@ -273,6 +273,10 @@
 
     @patch('keystone_utils.log')
     @patch('keystone_utils.ensure_ssl_cert_master')
+    @patch.object(hooks, 'ensure_pki_dir_permissions')
+    @patch.object(hooks, 'ensure_ssl_dir')
+    @patch.object(hooks, 'is_pki_enabled')
+    @patch.object(hooks, 'is_ssl_cert_master')
     @patch.object(hooks, 'send_ssl_sync_request')
     @patch.object(hooks, 'is_db_initialised')
     @patch.object(hooks, 'is_db_ready')
@@ -285,13 +289,25 @@
     @patch.object(hooks, 'CONFIGS')
     @patch.object(hooks, 'identity_changed')
     @patch.object(hooks, 'configure_https')
-    def test_config_changed_no_openstack_upgrade_leader(
-            self, configure_https, identity_changed,
-            configs, get_homedir, ensure_user, cluster_joined,
-            admin_relation_changed, ensure_permissions, mock_peer_units,
-            mock_is_db_ready, mock_is_db_initialised,
-            mock_send_ssl_sync_request,
-            mock_ensure_ssl_cert_master, mock_log):
+    def test_config_changed_no_upgrade_leader(self, configure_https,
+                                              identity_changed,
+                                              configs, get_homedir,
+                                              ensure_user,
+                                              cluster_joined,
+                                              admin_relation_changed,
+                                              ensure_permissions,
+                                              mock_peer_units,
+                                              mock_is_db_ready,
+                                              mock_is_db_initialised,
+                                              mock_send_ssl_sync_request,
+                                              mock_is_ssl_cert_master,
+                                              mock_is_pki_enabled,
+                                              mock_ensure_ssl_dir,
+                                              mock_ensure_pki_dir_permissions,
+                                              mock_ensure_ssl_cert_master,
+                                              mock_log):
+        mock_is_pki_enabled.return_value = True
+        mock_is_ssl_cert_master.return_value = True
         mock_is_db_initialised.return_value = True
         mock_is_db_ready.return_value = True
         self.openstack_upgrade_available.return_value = False
@@ -320,6 +336,12 @@
 
     @patch('keystone_utils.log')
     @patch('keystone_utils.ensure_ssl_cert_master')
+    @patch.object(hooks, 'update_all_identity_relation_units')
+    @patch.object(hooks, 'ensure_pki_dir_permissions')
+    @patch.object(hooks, 'ensure_ssl_dir')
+    @patch.object(hooks, 'is_pki_enabled')
+    @patch.object(hooks, 'peer_units')
+    @patch.object(hooks, 'is_ssl_cert_master')
     @patch.object(hooks, 'ensure_permissions')
     @patch.object(hooks, 'cluster_joined')
     @patch.object(unison, 'ensure_user')
@@ -327,11 +349,22 @@
     @patch.object(hooks, 'CONFIGS')
     @patch.object(hooks, 'identity_changed')
     @patch.object(hooks, 'configure_https')
-    def test_config_changed_no_openstack_upgrade_not_leader(
-            self, configure_https, identity_changed,
-            configs, get_homedir, ensure_user, cluster_joined,
-            ensure_permissions, mock_ensure_ssl_cert_master,
-            mock_log):
+    def test_config_changed_no_upgrade_not_leader(self, configure_https,
+                                                  identity_changed,
+                                                  configs, get_homedir,
+                                                  ensure_user, cluster_joined,
+                                                  ensure_permissions,
+                                                  mock_is_ssl_cert_master,
+                                                  mock_peer_units,
+                                                  mock_is_pki_enabled,
+                                                  mock_ensure_ssl_dir,
+                                                  mock_ensure_pki_permissions,
+                                                  mock_update_all_id_rel_units,
+                                                  mock_ensure_ssl_cert_master,
+                                                  mock_log):
+        mock_is_pki_enabled.return_value = True
+        mock_is_ssl_cert_master.return_value = True
+        mock_peer_units.return_value = []
         self.openstack_upgrade_available.return_value = False
         self.is_elected_leader.return_value = False
         mock_ensure_ssl_cert_master.return_value = False
@@ -350,6 +383,10 @@
 
     @patch('keystone_utils.log')
     @patch('keystone_utils.ensure_ssl_cert_master')
+    @patch.object(hooks, 'ensure_pki_dir_permissions')
+    @patch.object(hooks, 'ensure_ssl_dir')
+    @patch.object(hooks, 'is_pki_enabled')
+    @patch.object(hooks, 'is_ssl_cert_master')
     @patch.object(hooks, 'send_ssl_sync_request')
     @patch.object(hooks, 'is_db_initialised')
     @patch.object(hooks, 'is_db_ready')
@@ -372,8 +409,14 @@
                                                    mock_is_db_ready,
                                                    mock_is_db_initialised,
                                                    mock_send_ssl_sync_request,
+                                                   mock_is_ssl_cert_master,
+                                                   mock_is_pki_enabled,
+                                                   mock_ensure_ssl_dir,
+                                                   mock_ensure_pki_permissions,
                                                    mock_ensure_ssl_cert_master,
                                                    mock_log):
+        mock_is_pki_enabled.return_value = True
+        mock_is_ssl_cert_master.return_value = True
         mock_is_db_ready.return_value = True
         mock_is_db_initialised.return_value = True
         self.openstack_upgrade_available.return_value = True
@@ -449,8 +492,11 @@
             user=self.ssh_user, group='juju_keystone',
             peer_interface='cluster', ensure_local_user=True)
 
+    @patch.object(hooks, 'update_all_identity_relation_units')
+    @patch.object(hooks, 'get_ssl_sync_request_units')
     @patch.object(hooks, 'is_ssl_cert_master')
     @patch.object(hooks, 'peer_units')
+    @patch('keystone_utils.config')
     @patch('keystone_utils.log')
     @patch('keystone_utils.ensure_ssl_cert_master')
     @patch('keystone_utils.synchronize_ca')
@@ -460,12 +506,29 @@
     def test_cluster_changed(self, configs, ssh_authorized_peers,
                              check_peer_actions, mock_synchronize_ca,
                              mock_ensure_ssl_cert_master,
-                             mock_log, mock_peer_units,
-                             mock_is_ssl_cert_master):
+                             mock_log, mock_config, mock_peer_units,
+                             mock_is_ssl_cert_master,
+                             mock_get_ssl_sync_request_units,
+                             mock_update_all_identity_relation_units):
+
+        relation_settings = {'foo_passwd': '123',
+                             'identity-service:16_foo': 'bar'}
+
         mock_is_ssl_cert_master.return_value = False
         mock_peer_units.return_value = ['unit/0']
         mock_ensure_ssl_cert_master.return_value = False
         self.is_elected_leader.return_value = False
+
+        def fake_rel_get(attribute=None, *args, **kwargs):
+            if not attribute:
+                return relation_settings
+
+            return relation_settings.get(attribute)
+
+        self.relation_get.side_effect = fake_rel_get
+
+        mock_config.return_value = None
+
         hooks.cluster_changed()
         whitelist = ['_passwd', 'identity-service:', 'ssl-cert-master',
                      'db-initialised']

=== modified file 'unit_tests/test_keystone_utils.py'
--- unit_tests/test_keystone_utils.py	2015-03-10 12:02:11 +0000
+++ unit_tests/test_keystone_utils.py	2015-03-11 14:46:50 +0000
@@ -179,18 +179,19 @@
         self.assertTrue(self.https.called)
         self.assertTrue(self.create_role.called)
 
-        relation_data = {'auth_host': '10.10.10.10',
-                         'service_host': '10.10.10.10',
-                         'auth_protocol': 'https',
+        rel_only_data = {'auth_host': '10.10.10.10',
+                         'service_host': '10.10.10.10'}
+        relation_data = {'auth_protocol': 'https',
                          'service_protocol': 'https',
                          'auth_port': 80,
                          'service_port': 81,
                          'https_keystone': 'True',
                          'ca_cert': 'certificate',
                          'region': 'RegionOne'}
-        self.peer_store_and_set.assert_called_with(
-            relation_id=relation_id,
-            **relation_data)
+        self.relation_set.assert_called_with(relation_id=relation_id,
+                                             **rel_only_data)
+        self.peer_store_and_set.assert_called_with(relation_id=relation_id,
+                                                   **relation_data)
 
     @patch.object(utils, 'ensure_valid_service')
     @patch.object(utils, 'add_endpoint')
@@ -236,14 +237,15 @@
         self.grant_role.assert_called_with('keystone', 'admin', 'tenant')
         self.create_role.assert_called_with('role1', 'keystone', 'tenant')
 
+        rel_only_data = {'auth_host': '10.0.0.3',
+                         'service_host': '10.0.0.3'}
         relation_data = {'admin_token': 'token', 'service_port': 81,
                          'auth_port': 80, 'service_username': 'keystone',
                          'service_password': 'password',
                          'service_tenant': 'tenant',
                          'https_keystone': '__null__',
                          'ssl_cert': '__null__', 'ssl_key': '__null__',
-                         'ca_cert': '__null__', 'auth_host': '10.0.0.3',
-                         'service_host': '10.0.0.3',
+                         'ca_cert': '__null__',
                          'auth_protocol': 'http', 'service_protocol': 'http',
                          'service_tenant_id': 'tenant_id'}
 
@@ -254,9 +256,11 @@
             else:
                 filtered[k] = v
 
-        call1 = call(relation_id=relation_id, **filtered)
-        call2 = call(relation_id='cluster/0', **relation_data)
-        self.relation_set.assert_has_calls([call1, call2])
+        call1 = call(relation_id=relation_id, **rel_only_data)
+        call2 = call(relation_id=relation_id, **filtered)
+        call3 = call(relation_id='cluster/0', **relation_data)
+        self.assertTrue(self.relation_set.called)
+        self.relation_set.assert_has_calls([call1, call2, call3])
 
     @patch.object(utils, 'ensure_valid_service')
     @patch.object(utils, 'add_endpoint')