Merge lp:~bjornt/landscape-charm/leader-destroyed-integration-tests into lp:~landscape/landscape-charm/trunk

[Björn Tillenius]

Add integration tests for when the landscape-server leader dies.

I added tests to ensuring that all services are up and running,
including package-upload which runs only on the leader.

Add integration tests for when the landscape-server leader dies.

I added tests to ensuring that all services are up and running,
including package-upload which runs only on the leader.

[🤖 Landscape Builder (landscape-builder)]

Command: make ci-test
Result: Success
Revno: 321
Branch: lp:~bjornt/landscape-charm/leader-destroyed-integration-tests
Jenkins: https://ci.lscape.net/job/latch-test/1361/

review: Approve (test results)

[Adam Collard (adam-collard)]

Running the tests locally causes my machine to die in a pit of swappiness and despair.

Code looks good though, and I'm happy to +1 on the assumption that they pass for you and the other reviewer.

review: Approve

[Free Ekanayaka (free.ekanayaka)]

+1 works as advertised

review: Approve

[lp:~bjornt/landscape-charm/leader-destroyed-integration-tests]

322. By Björn Tillenius on 2015-06-11

Typo.

=== modified file 'tests/basic/test_ha.py'
--- tests/basic/test_ha.py	2015-06-09 09:52:44 +0000
+++ tests/basic/test_ha.py	2015-06-11 15:59:45 +0000
@@ -15,7 +15,7 @@
     def setUp(self):
         super(LandscapeHATest, self).setUp()
         self.leader = self.layer.leader
-        self.non_leader = self.layer.non_leader
+        [self.non_leader] = self.layer.non_leaders
 
     def test_app_leader_down(self):
         """

=== added file 'tests/basic/test_leader.py'
--- tests/basic/test_leader.py	1970-01-01 00:00:00 +0000
+++ tests/basic/test_leader.py	2015-06-11 15:59:45 +0000
@@ -0,0 +1,47 @@
+from helpers import IntegrationTest
+from layers import LandscapeLeaderDestroyedLayer
+
+
+class LandscapeLeaderDestroyedTest(IntegrationTest):
+    """Test what happens when the landscape-server leader gets destroyed.
+
+    The tests in here are intended to test that the newly elected leader
+    takes over correctly.
+    """
+
+    layer = LandscapeLeaderDestroyedLayer
+
+    def test_app(self):
+        """
+        Verify that the APP service is up after the leader unit is destroyed.
+        """
+        self.environment.check_service("appserver")
+
+    def test_msg(self):
+        """
+        Verify that the MSG service is up after the leader unit is destroyed.
+        """
+        self.environment.check_service("msgserver")
+
+    def test_ping(self):
+        """
+        Verify that the PING service is up after the leader unit is destroyed.
+        """
+        self.environment.check_service("pingserver")
+
+    def test_api(self):
+        """
+        Verify that the API service is up after the leader unit is destroyed.
+        """
+        self.environment.check_service("api")
+
+    def test_package_upload_new_leader(self):
+        """
+        Verify that the package upload service is now running on the new
+        leader.
+
+        package-upload is running only on the leader unit, and if a
+        leader unit gets destroyed, the package-upload service is
+        transferred to the new leader.
+        """
+        self.environment.check_service("package-upload")

=== modified file 'tests/helpers.py'
--- tests/helpers.py	2015-06-11 09:56:29 +0000
+++ tests/helpers.py	2015-06-11 15:59:45 +0000
@@ -10,6 +10,7 @@
 import tempfile
 import shutil
 import json
+import time
 
 from os import getenv
 from time import sleep
@@ -268,12 +269,7 @@
         """Start the given Landscape service on the given unit."""
         self._deployment.configure(
             "landscape-server", {"ssl-cert": cert, "ssl-key": key})
-        # Wait for initial landscape-server hooks to fire
-        self._deployment.sentry.wait()
-        # Wait for haproxy hooks to fire
-        self._deployment.sentry.wait()
-        # Wait for landscape-server hooks triggered by the haproxy ones to fire
-        self._deployment.sentry.wait()
+        self._wait_for_deployment_change_hooks()
 
     def set_unit_count(self, service, new_count):
         """Change the service to have the given number of units.
@@ -296,10 +292,23 @@
         while current_count > new_count:
             self._deployment.destroy_unit(existing_units.pop())
             current_count -= 1
-        # Wait for all the hooks to finish firing.
-        self._deployment.sentry.wait()
-        self._deployment.sentry.wait()
-        self._deployment.sentry.wait()
+        self._wait_for_deployment_change_hooks()
+
+    def destroy_landscape_leader(self):
+        """Destroy the landscape-server leader
+
+        This method will wait at most 60 seconds for a new leader to
+        elected before returning.
+        """
+        leader, _ = self.get_unit_ids("landscape-server")
+        self._deployment.destroy_unit("landscape-server/{}".format(leader))
+        self._wait_for_deployment_change_hooks()
+        for _ in range(60):
+            leader, _ = self.get_unit_ids("landscape-server")
+            if leader is not None:
+                break
+            time.sleep(1)
+        assert leader is not None, "No new leader was elected."
 
     def get_unit_ids(self, service):
         """Return the numerical id parts for the units of the given service.
@@ -320,12 +329,20 @@
             result, code = unit.run("is-leader --format=json")
             if json.loads(result):
                 assert leader is None, "Multiple leaders found."
-                leader = unit_number
+                leader = int(unit_number)
             else:
-                non_leaders.append(unit_number)
-
-        return int(leader), sorted(
-            int(unit_number) for unit_number in non_leaders)
+                non_leaders.append(int(unit_number))
+
+        return leader, sorted(non_leaders)
+
+    def _wait_for_deployment_change_hooks(self):
+        """Wait for hooks to finish firing after a change in the deployment."""
+        # Wait for initial landscape-server hooks to fire
+        self._deployment.sentry.wait()
+        # Wait for haproxy hooks to fire
+        self._deployment.sentry.wait()
+        # Wait for landscape-server hooks triggered by the haproxy ones to fire
+        self._deployment.sentry.wait()
 
     def _run(self, command, unit):
         """Run a command on the given unit.

=== modified file 'tests/layers.py'
--- tests/layers.py	2015-06-11 07:49:02 +0000
+++ tests/layers.py	2015-06-11 15:59:45 +0000
@@ -66,5 +66,20 @@
     @classmethod
     def setUp(cls):
         cls.environment.set_unit_count("landscape-server", 2)
-        cls.leader, [cls.non_leader] = cls.environment.get_unit_ids(
+        cls.leader, cls.non_leaders = cls.environment.get_unit_ids(
+            "landscape-server")
+
+
+class LandscapeLeaderDestroyedLayer(TwoLandscapeUnitsLayer):
+    """Layer for tests meant to run when the leader has been destroyed.
+
+    After setting up this layer, a new leader will have been elected.
+
+    Note that this layer is destructive and reduces the deployment to 1 unit.
+    """
+
+    @classmethod
+    def setUp(cls):
+        cls.environment.destroy_landscape_leader()
+        cls.leader, cls.non_leaders = cls.environment.get_unit_ids(
             "landscape-server")