Merge ~vultaire/charm-elasticsearch:lp1835410 into charm-elasticsearch:master

[Paul Goins]

Made initial peer connection code more forgiving

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[Jeremy Lounder (jldev)]

review: Approve

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change successfully merged at revision 3cb5d16d4a183b190a39f000c0c25e81b9d2921d

diff --git a/bin/wait_for_peer.py b/bin/wait_for_peer.py
new file mode 100755
index 0000000..bffd549
--- /dev/null
+++ b/bin/wait_for_peer.py
@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+
+import logging
+import os
+import socket
+import sys
+import time
+import traceback
+
+logging.basicConfig(
+    stream=sys.stderr,
+    level=logging.DEBUG,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+)
+
+import requests
+
+sys.path.append(os.path.abspath("charm-helpers"))
+from charmhelpers.core.host import service_restart
+
+
+CONNECTION_TIMEOUT = 5
+POLL_DELAY = 5
+SCRIPT_TIMEOUT = 600
+
+
+def main():
+    start_time = time.time()
+    while True:
+        node_count = get_node_count()
+        if node_count > 1:
+            logging.info("Multiple ES nodes detected; success.")
+            return 0
+        if time.time() - start_time > SCRIPT_TIMEOUT:
+            logging.error("Timed out while waiting for peers")
+            return 1
+        logging.info("Node count is not greater than 1, will restart and retry")
+        logging.info("Restarting elasticsearch...")
+        restart_elasticsearch()
+        logging.info("Waiting for elasticsearch port to open...")
+        wait_for_es_port()
+        logging.info("Waiting {} seconds...".format(POLL_DELAY))
+        time.sleep(POLL_DELAY)
+
+
+def get_node_count():
+    # NOTE: 503 errors have been intermittently sighted here, so we'll retry
+    # in case of errors.
+    cluster_health_url = "http://localhost:9200/_cluster/health"
+    max_attempts = 5
+    for i in range(max_attempts):
+        try:
+            return requests.get(cluster_health_url, timeout=CONNECTION_TIMEOUT).json()[
+                "number_of_nodes"
+            ]
+        except Exception:
+            if i < max_attempts - 1:
+                logging.info(
+                    "Error occurred while polling {}; retrying".format(
+                        cluster_health_url
+                    )
+                )
+                time.sleep(POLL_DELAY)
+            else:
+                logging.warning(traceback.format_exc())
+    else:
+        logging.warning(
+            "Error occurred while polling {}; giving up".format(cluster_health_url)
+        )
+        return 0
+
+
+def restart_elasticsearch():
+    service_restart("elasticsearch")
+
+
+def wait_for_es_port(timeout=60):
+    start_time = time.time()
+    while time.time() - start_time < timeout:
+        try:
+            _test_socket_connection("localhost", 9200)
+        except Exception:
+            time.sleep(POLL_DELAY)
+        else:
+            return
+    else:
+        # Last try; allow exceptions to bubble up
+        _test_socket_connection("localhost", 9200)
+
+
+def _test_socket_connection(host, port):
+    s = socket.create_connection((host, port), timeout=CONNECTION_TIMEOUT)
+    s.close()
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/tasks/peer-relations.yml b/tasks/peer-relations.yml
index a9b5bc7..d50dd2e 100644
--- a/tasks/peer-relations.yml
+++ b/tasks/peer-relations.yml
@@ -19,42 +19,7 @@
     - peer-relation-changed
   wait_for: port=9200
 
-- name: Record current cluster health
-  tags:
-    - peer-relation-joined
-    - peer-relation-changed
-  uri: url=http://localhost:9200/_cluster/health return_content=yes
-  register: cluster_health
-
-- name: Restart if not part of cluster
-  tags:
-    - peer-relation-joined
-    - peer-relation-changed
-  service: name=elasticsearch state=restarted
-  when: cluster_health.json.number_of_nodes == 1
-
-- name: Wait until the local service is available after restart
-  tags:
-    - peer-relation-joined
-    - peer-relation-changed
-  wait_for: port=9200
-  when: cluster_health.json.number_of_nodes == 1
-
-- name: Pause to ensure that after restart unit has time to join.
-  tags:
-    - peer-relation-changed
-  pause: seconds=30
-  when: cluster_health.json.number_of_nodes == 1
-
-- name: Record cluster health after restart
-  tags:
-    - peer-relation-changed
-  uri: url=http://localhost:9200/_cluster/health return_content=yes
-  register: cluster_health_after_restart
-  when: cluster_health.json.number_of_nodes == 1
-
-- name: Fail if unit is still not part of cluster
+- name: Wait for the cluster to increase in size
   tags:
     - peer-relation-changed
-  fail: msg="Unit failed to join cluster after peer-relation-changed"
-  when: cluster_health.json.number_of_nodes == 1 and cluster_health_after_restart.json.number_of_nodes == 1
+  command: bin/wait_for_peer.py