Merge ~cjwatson/launchpad:charm-rabbitmq-ha into launchpad:master

[Colin Watson]

charm: Handle multiple RabbitMQ broker URLs

Now that we can tell Launchpad to use multiple RabbitMQ broker URLs with round-robin between them, we should make use of that rather than just picking one of them. This allows for higher availability.

[Ioana Lasc (ilasc)]

review: Approve

diff --git a/charm/layer/launchpad-base/reactive/launchpad-base.py b/charm/layer/launchpad-base/reactive/launchpad-base.py
index 8e0d8d1..a3473f7 100644
--- a/charm/layer/launchpad-base/reactive/launchpad-base.py
+++ b/charm/layer/launchpad-base/reactive/launchpad-base.py
@@ -35,6 +35,19 @@ def prepare_rabbitmq(rabbitmq):
     rabbitmq.request_access(config["rabbitmq_user"], config["domain"])
 
 
+def get_rabbitmq_uris(rabbitmq):
+    for conversation in rabbitmq.conversations():
+        for relation_id in conversation.relation_ids:
+            for unit in hookenv.related_units(relation_id):
+                hostname = hookenv.relation_get(
+                    "private-address", unit, relation_id
+                )
+                vhost = rabbitmq.vhost()
+                username = rabbitmq.username()
+                password = rabbitmq.password()
+                yield f"amqp://{username}:{password}@{hostname}/{vhost}"
+
+
 @when("ols.configured", "db.master.available", "rabbitmq.available")
 @when_not("launchpad.base.configured")
 def configure(db, rabbitmq):
@@ -54,14 +67,7 @@ def configure(db, rabbitmq):
     # specific to the appserver.  We need to teach Launchpad to be able to
     # log in as one role and then switch to another.
     config["db_user"] = parse_dsn(db_primary)["user"]
-    # XXX cjwatson 2022-09-29: How do we implement HA?  Looks like we'd need
-    # code changes to let us configure multiple broker URLs.  (At the moment
-    # we rely on round-robin DNS on production, but that probably doesn't
-    # provide very good HA either.)
-    config["rabbitmq_host"] = rabbitmq.private_address()
-    config["rabbitmq_username"] = rabbitmq.username()
-    config["rabbitmq_password"] = rabbitmq.password()
-    config["rabbitmq_vhost"] = rabbitmq.vhost()
+    config["rabbitmq_broker_urls"] = sorted(get_rabbitmq_uris(rabbitmq))
     configure_lazr(
         config,
         "launchpad-base-lazr.conf",
@@ -100,3 +106,9 @@ def build_label_changed():
 def config_changed():
     remove_state("launchpad.base.configured")
     remove_state("service.configured")
+
+
+@hook("{requires:rabbitmq}-relation-changed")
+def rabbitmq_relation_changed(rabbitmq):
+    remove_state("launchpad.base.configured")
+    remove_state("service.configured")
diff --git a/charm/layer/launchpad-base/templates/launchpad-base-lazr.conf b/charm/layer/launchpad-base/templates/launchpad-base-lazr.conf
index 4ea639e..9e597f5 100644
--- a/charm/layer/launchpad-base/templates/launchpad-base-lazr.conf
+++ b/charm/layer/launchpad-base/templates/launchpad-base-lazr.conf
@@ -82,11 +82,6 @@ authentication_endpoint: http://{{ domain_xmlrpc_private }}:{{ port_xmlrpc }}/au
 buglist_batch_size: {{ default_batch_size }}
 bugmail_error_from_address: noreply@bugs.{{ domain }}
 
-[rabbitmq]
-host: {{ rabbitmq_host }}
-userid: {{ rabbitmq_username }}
-virtual_host: {{ rabbitmq_vhost }}
-
 {%- if statsd_environment %}
 [statsd]
 host: 127.0.0.1
diff --git a/charm/layer/launchpad-base/templates/launchpad-base-secrets-lazr.conf b/charm/layer/launchpad-base/templates/launchpad-base-secrets-lazr.conf
index 1d5ff79..5b45030 100644
--- a/charm/layer/launchpad-base/templates/launchpad-base-secrets-lazr.conf
+++ b/charm/layer/launchpad-base/templates/launchpad-base-secrets-lazr.conf
@@ -14,5 +14,5 @@
 {{- opt("shared_secret", mailman_shared_secret) }}
 
 [rabbitmq]
-password: {{ rabbitmq_password }}
+broker_urls: {{ rabbitmq_broker_urls|join(" ") }}