Merge lp:~gandelman-a/charms/precise/keystone/cluster_leader into lp:~openstack-charmers/charms/precise/keystone/ha-support

[Adam Gandelman]

Adds better support for service leaders.

* The service leader is determined depending on how keystone is currently clustered. If there are multiple units, but no hacluster subordinate, the oldest service unit is elected leader (lowest unit number). If hacluster exists and the service is clustered, the CRM is consulted and the node hosting the resources is designated the leader.

* Only the leader may initialize or touch the database (create users, endpoints, etc)

* The leader is responsible for synchronizing a list of service credentials to all peers. The list is stored on disk and resolves the issue of the passwd dump files in /var/lib/keystone/ being out-of-sync among peers.

We can use the same approach in the rabbitmq-server charm if it works out here.

[lp:~gandelman-a/charms/precise/keystone/cluster_leader]

51. By Adam Gandelman on 2013-01-31

Little cleanup.

[James Page (james-page)]

Tested OK and generally looks sound

+1 merged.

review: Approve

=== modified file 'hooks/keystone-hooks'
--- hooks/keystone-hooks	2013-01-21 15:05:12 +0000
+++ hooks/keystone-hooks	2013-01-31 21:34:20 +0000
@@ -73,12 +73,14 @@
                         driver='keystone.token.backends.sql.Token')
     update_config_block('ec2',
                         driver='keystone.contrib.ec2.backends.sql.Ec2')
+
     execute("service keystone stop", echo=True)
     execute("keystone-manage db_sync")
     execute("service keystone start", echo=True)
     time.sleep(5)
     ensure_initial_admin(config)
 
+
 def db_joined():
     relation_data = { "database": config["database"],
                       "username": config["database-user"],
@@ -96,7 +98,14 @@
                              relation_data["password"],
                              relation_data["private-address"],
                              config["database"]))
+
     execute("service keystone stop", echo=True)
+
+    if not eligible_leader():
+        juju_log('Deferring DB initialization to service leader.')
+        execute("service keystone start")
+        return
+
     execute("keystone-manage db_sync", echo=True)
     execute("service keystone start")
     time.sleep(5)
@@ -137,8 +146,8 @@
                                  adminurl=adminurl,
                                  internalurl=internalurl)
 
-    if is_clustered() and not is_leader():
-        # Only respond if service unit is the leader
+    if not eligible_leader():
+        juju_log('Deferring identity_changed() to service leader.')
         return
 
     settings = relation_get_dict(relation_id=relation_id,
@@ -217,16 +226,7 @@
     token = get_admin_token()
     juju_log("Creating service credentials for '%s'" % service_username)
 
-    # TODO: This needs to be changed as it won't work with ha keystone
-    stored_passwd = '/var/lib/keystone/%s.passwd' % service_username
-    if os.path.isfile(stored_passwd):
-        juju_log("Loading stored service passwd from %s" % stored_passwd)
-        service_password = open(stored_passwd, 'r').readline().strip('\n')
-    else:
-        juju_log("Generating a new service password for %s" % service_username)
-        service_password = execute('pwgen -c 32 1', die=True)[0].strip()
-        open(stored_passwd, 'w+').writelines("%s\n" % service_password)
-
+    service_password = get_service_password(service_username)
     create_user(service_username, service_password, config['service-tenant'])
     grant_role(service_username, config['admin-role'], config['service-tenant'])
 
@@ -254,6 +254,7 @@
         relation_data["service_port"] = SERVICE_PORTS['keystone_service']
 
     relation_set(relation_data)
+    synchronize_service_credentials()
 
 def config_changed():
 
@@ -268,11 +269,9 @@
 
     set_admin_token(config['admin-token'])
 
-    if is_clustered() and is_leader():
+    if eligible_leader():
         juju_log('Cluster leader - ensuring endpoint configuration is up to date')
         ensure_initial_admin(config)
-    elif not is_clustered():
-        ensure_initial_admin(config)
 
     update_config_block('logger_root', level=config['log-level'],
                         file='/etc/keystone/logging.conf')
@@ -286,11 +285,9 @@
 
 def upgrade_charm():
     cluster_changed()
-    if is_clustered() and is_leader():
+    if eligible_leader():
         juju_log('Cluster leader - ensuring endpoint configuration is up to date')
         ensure_initial_admin(config)
-    elif not is_clustered():
-        ensure_initial_admin(config)
 
 
 SERVICE_PORTS = {
@@ -310,6 +307,8 @@
     configure_haproxy(cluster_hosts,
                       SERVICE_PORTS)
 
+    synchronize_service_credentials()
+
 
 def ha_relation_changed():
     relation_data = relation_get_dict()
@@ -387,7 +386,7 @@
 
 # keystone-hooks gets called by symlink corresponding to the requested relation
 # hook.
-arg0 = sys.argv[0].split("/").pop()
-if arg0 not in hooks.keys():
-    error_out("Unsupported hook: %s" % arg0)
-hooks[arg0]()
+hook = os.path.basename(sys.argv[0])
+if hook not in hooks.keys():
+    error_out("Unsupported hook: %s" % hook)
+hooks[hook]()

=== modified file 'hooks/utils.py'
--- hooks/utils.py	2013-01-22 17:49:36 +0000
+++ hooks/utils.py	2013-01-31 21:34:20 +0000
@@ -11,10 +11,10 @@
 keystone_conf = "/etc/keystone/keystone.conf"
 stored_passwd = "/var/lib/keystone/keystone.passwd"
 stored_token = "/var/lib/keystone/keystone.token"
-
+SERVICE_PASSWD_PATH = '/var/lib/keystone/services.passwd'
 
 def execute(cmd, die=False, echo=False):
-    """ Executes a command 
+    """ Executes a command
 
     if die=True, script will exit(1) if command does not return 0
     if echo=True, output of command will be printed to stdout
@@ -398,6 +398,31 @@
     manager.api.users.update_password(user=user_id, password=password)
     juju_log("Successfully updated password for user '%s'" % username)
 
+def load_stored_passwords(path=SERVICE_PASSWD_PATH):
+    creds = {}
+    if not os.path.isfile(path):
+        return creds
+
+    stored_passwd = open(path, 'r')
+    for l in stored_passwd.readlines():
+        user, passwd = l.strip().split(':')
+        creds[user] = passwd
+    return creds
+
+def save_stored_passwords(path=SERVICE_PASSWD_PATH, **creds):
+    with open(path, 'wb') as stored_passwd:
+        [stored_passwd.write('%s:%s\n' % (u, p)) for u, p in creds.iteritems()]
+
+def get_service_password(service_username):
+    creds = load_stored_passwords()
+    if service_username in creds:
+        return creds[service_username]
+
+    passwd = subprocess.check_output(['pwgen', '-c', '32', '1']).strip()
+    creds[service_username] = passwd
+    save_stored_passwords(**creds)
+
+    return passwd
 
 def configure_pki_tokens(config):
     '''Configure PKI token signing, if enabled.'''
@@ -482,3 +507,53 @@
         return True
     else:
         return False
+
+
+def peer_units():
+    peers = []
+    for r_id in (relation_ids('cluster') or []):
+        for unit in (relation_list(r_id) or []):
+            peers.append(unit)
+    return peers
+
+def oldest_peer(peers):
+    local_unit_no = os.getenv('JUJU_UNIT_NAME').split('/')[1]
+    for peer in peers:
+        remote_unit_no = peer.split('/')[1]
+        if remote_unit_no < local_unit_no:
+            return False
+    return True
+
+
+def eligible_leader():
+    if is_clustered():
+        if not is_leader():
+            juju_log('Deferring action to CRM leader.')
+            return False
+    else:
+        peers = peer_units()
+        if peers and not oldest_peer(peers):
+            juju_log('Deferring action to oldest service unit.')
+            return False
+    return True
+
+
+def synchronize_service_credentials():
+    '''
+    Broadcast service credentials to peers or consume those that have been
+    broadcasted by peer, depending on hook context.
+    '''
+    if os.path.basename(sys.argv[0]) == 'cluster-relation-changed':
+        r_data = relation_get_dict()
+        if 'service_credentials' in r_data:
+            juju_log('Saving service passwords from peer.')
+            save_stored_passwords(**json.loads(r_data['service_credentials']))
+        return
+
+    creds = load_stored_passwords()
+    if not creds:
+        return
+    juju_log('Synchronizing service passwords to all peers.')
+    creds = json.dumps(creds)
+    for r_id in (relation_ids('cluster') or []):
+        relation_set_2(rid=r_id, service_credentials=creds)

=== modified file 'revision'
--- revision	2013-01-22 17:49:36 +0000
+++ revision	2013-01-31 21:34:20 +0000
@@ -1,1 +1,1 @@
-195
+196