Merge ~adam-collard/maas:busy-wait-rack-controller-on-secret into maas:master

[Adam Collard]

LP: #1903318 - Sleep for up to 5 minutes waiting for the secret

When there's no secret on disk, block for up to 5 minutes waiting for
it before falling back on starting no services.

[Lee Trager (ltrager)]

LGTM!

review: Approve

[MAAS Lander (maas-lander)]

LANDING
-b busy-wait-rack-controller-on-secret lp:~adam-collard/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED BUILD
LOG: http://maas-ci.internal:8080/job/maas/job/branch-tester/8640/consoleText

[MAAS Lander (maas-lander)]

UNIT TESTS
-b busy-wait-rack-controller-on-secret lp:~adam-collard/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci.internal:8080/job/maas/job/branch-tester/8641/console
COMMIT: 761d02c51b30e2fc8f15dd1c1ce5f43626fbb875

review: Needs Fixing

[Lee Trager (ltrager)]

See email

review: Needs Information

[Lee Trager (ltrager)]

Merged in [1] to fix lint.

[1] https://code.launchpad.net/~ltrager/maas/+git/maas/+merge/393526

diff --git a/src/provisioningserver/plugin.py b/src/provisioningserver/plugin.py
index 9bdda73..a38e279 100644
--- a/src/provisioningserver/plugin.py
+++ b/src/provisioningserver/plugin.py
@@ -8,6 +8,7 @@ __all__ = ["Options", "ProvisioningServiceMaker"]
 from errno import ENOPROTOOPT
 import socket
 from socket import error as socket_error
+from time import sleep
 
 from twisted.application.service import IServiceMaker
 from twisted.internet import reactor
@@ -23,6 +24,7 @@ from provisioningserver.monkey import (
 )
 from provisioningserver.prometheus.utils import clean_prometheus_dir
 from provisioningserver.security import get_shared_secret_from_filesystem
+from provisioningserver.utils.twisted import retries
 from provisioningserver.utils.debug import (
     register_sigusr1_toggle_cprofile,
     register_sigusr2_thread_dump_handler,
@@ -230,7 +232,7 @@ class ProvisioningServiceMaker:
         # Get something going with the logs.
         logger.configure(verbosity, logger.LoggingMode.TWISTD)
 
-    def makeService(self, options, clock=reactor):
+    def makeService(self, options, clock=reactor, sleep=sleep):
         """Construct the MAAS Cluster service."""
         register_sigusr1_toggle_cprofile("rackd")
         register_sigusr2_thread_dump_handler()
@@ -252,7 +254,13 @@ class ProvisioningServiceMaker:
 
         from provisioningserver import services
 
-        if get_shared_secret_from_filesystem():
+        secret = None
+        for elapsed, remaining, wait in retries(timeout=5 * 60, clock=clock):
+            secret = get_shared_secret_from_filesystem()
+            if secret is not None:
+                break
+            sleep(wait)
+        if secret is not None:
             # only setup services if the shared secret is configured
             for service in self._makeServices(
                 tftp_root, tftp_port, clock=clock
diff --git a/src/provisioningserver/tests/test_plugin.py b/src/provisioningserver/tests/test_plugin.py
index db186e2..63e3c64 100644
--- a/src/provisioningserver/tests/test_plugin.py
+++ b/src/provisioningserver/tests/test_plugin.py
@@ -6,6 +6,7 @@
 __all__ = []
 
 import os
+from itertools import count
 from pathlib import Path
 from subprocess import Popen
 
@@ -23,6 +24,7 @@ from testtools.matchers import (
 )
 from twisted.application.internet import StreamServerEndpointService
 from twisted.application.service import MultiService
+from twisted.internet.task import Clock
 
 from maastesting.fixtures import TempDirectory
 from maastesting.matchers import MockCalledOnceWith
@@ -107,10 +109,43 @@ class TestProvisioningServiceMaker(MAASTestCase):
         self.mock_get_shared_secret.return_value = None
         service_maker = ProvisioningServiceMaker("foo", "bar")
         self.patch(service_maker, "_loadSettings")
-        service = service_maker.makeService(Options(), clock=None)
+        clock = Clock()
+
+        attempts = count()
+
+        def advance(seconds):
+            next(attempts)
+            clock.advance(seconds)
+
+        service = service_maker.makeService(
+            Options(), clock=clock, sleep=advance
+        )
         self.assertIsInstance(service, MultiService)
         self.assertEqual(service.namedServices, {})
         self.mock_generate_certificate.assert_not_called()
+        # All 300 attempts (e.g. 5 minutes) fail, next is 301
+        self.assertEqual(next(attempts), 301)
+
+    def test_makeService_eventual_shared_secret(self):
+        # First two times we look, there's no secret
+        self.mock_get_shared_secret.side_effect = [None, None, "secret"]
+        service_maker = ProvisioningServiceMaker("foo", "bar")
+        self.patch(service_maker, "_loadSettings")
+        clock = Clock()
+
+        attempts = count(1)
+
+        def advance(seconds):
+            next(attempts)
+            clock.advance(60)
+
+        service = service_maker.makeService(
+            Options(), clock=clock, sleep=advance
+        )
+        self.assertIsInstance(service, MultiService)
+        self.assertNotEqual(service.namedServices, {})
+        # First two fail, the third one succeeds
+        self.assertEqual(next(attempts), 3)
 
     def test_makeService_not_in_debug(self):
         """