Merge lp:~thedac/charms/trusty/rabbitmq-server/backport-cluster-race-fixes into lp:charms/trusty/rabbitmq-server

[David Ames]

Backport cluster race conditions fix from next to the stable charm fixing LP Bug#1486177

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #8949 rabbitmq-server for thedac mp270712
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/8949/

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #9718 rabbitmq-server for thedac mp270712
    LINT FAIL: lint-test failed
    LINT FAIL: charm-proof failed

LINT Results (max last 2 lines):
make: *** [lint] Error 100
ERROR:root:Make target returned non-zero.

Full lint test output: http://paste.ubuntu.com/12329593/
Build: http://10.245.162.77:8080/job/charm_lint_check/9718/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #6348 rabbitmq-server for thedac mp270712
    AMULET FAIL: amulet-test failed

AMULET Results (max last 2 lines):
make: *** [functional_test] Error 1
ERROR:root:Make target returned non-zero.

Full amulet test output: http://paste.ubuntu.com/12329701/
Build: http://10.245.162.77:8080/job/charm_amulet_test/6348/

[Liam Young (gnuoy)]

Approve, (amulet fixes to follow).

review: Approve

=== modified file 'hooks/rabbit_utils.py'
--- hooks/rabbit_utils.py	2015-08-10 16:38:33 +0000
+++ hooks/rabbit_utils.py	2015-09-10 16:44:56 +0000
@@ -299,7 +299,12 @@
             cmd = [RABBITMQ_CTL, 'stop_app']
             subprocess.check_call(cmd)
             cmd = [RABBITMQ_CTL, cluster_cmd, 'rabbit@%s' % node]
-            subprocess.check_call(cmd)
+            try:
+                subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+            except subprocess.CalledProcessError as e:
+                if not e.returncode == 2 or \
+                        "{ok,already_member}" not in e.output:
+                    raise e
             cmd = [RABBITMQ_CTL, 'start_app']
             subprocess.check_call(cmd)
             log('Host clustered with %s.' % node)

=== modified file 'hooks/rabbitmq_server_relations.py'
--- hooks/rabbitmq_server_relations.py	2015-08-10 16:38:33 +0000
+++ hooks/rabbitmq_server_relations.py	2015-09-10 16:44:56 +0000
@@ -81,6 +81,7 @@
     peer_store,
     peer_store_and_set,
     peer_retrieve_by_prefix,
+    leader_get,
 )
 
 from charmhelpers.contrib.network.ip import get_address_in_network
@@ -304,31 +305,35 @@
         return
 
     if is_elected_leader('res_rabbitmq_vip'):
+        log('Leader peer_storing cookie', level=INFO)
         cookie = open(rabbit.COOKIE_PATH, 'r').read().strip()
         peer_store('cookie', cookie)
 
 
 @hooks.hook('cluster-relation-changed')
 def cluster_changed():
+    # Future travelers beware ordering is significant
+    rdata = relation_get()
+    # sync passwords
+    blacklist = ['hostname', 'private-address', 'public-address']
+    whitelist = [a for a in rdata.keys() if a not in blacklist]
+    peer_echo(includes=whitelist)
+
     cookie = peer_retrieve('cookie')
     if not cookie:
         log('cluster_joined: cookie not yet set.', level=INFO)
         return
 
-    rdata = relation_get()
     if config('prefer-ipv6') and rdata.get('hostname'):
-        private_address = rdata['private-address']
-        hostname = rdata['hostname']
+        private_address = rdata.get('private-address')
+        hostname = rdata.get('hostname')
         if hostname:
             rabbit.update_hosts_file({private_address: hostname})
 
-    # sync passwords
-    blacklist = ['hostname', 'private-address', 'public-address']
-    whitelist = [a for a in rdata.keys() if a not in blacklist]
-    peer_echo(includes=whitelist)
-
     if not is_sufficient_peers():
         # Stop rabbit until leader has finished configuring
+        log('Not enough peers, stopping until leader is configured',
+            level=INFO)
         service_stop('rabbitmq-server')
         return
 
@@ -358,9 +363,12 @@
             amqp_changed(relation_id=rid, remote_unit=unit)
 
 
-def update_cookie():
+def update_cookie(leaders_cookie=None):
     # sync cookie
-    cookie = peer_retrieve('cookie')
+    if leaders_cookie:
+        cookie = leaders_cookie
+    else:
+        cookie = peer_retrieve('cookie')
     cookie_local = None
     with open(rabbit.COOKIE_PATH, 'r') as f:
         cookie_local = f.read().strip()
@@ -763,6 +771,18 @@
 
 @hooks.hook('leader-settings-changed')
 def leader_settings_changed():
+    # Get cookie from leader, update cookie locally and
+    # force cluster-relation-changed hooks to run on peers
+    cookie = leader_get(attribute='cookie')
+    if cookie:
+        update_cookie(leaders_cookie=cookie)
+        # Force cluster-relation-changed hooks to run on peers
+        # This will precipitate peer clustering
+        # Without this a chicken and egg scenario prevails when
+        # using LE and peerstorage
+        for rid in relation_ids('cluster'):
+            relation_set(relation_id=rid, relation_settings={'cookie': cookie})
+
     # If leader has changed and access credentials, ripple these
     # out from all units
     for rid in relation_ids('amqp'):