Merge ~cgrabowski/maas:handle_all_connections_busy_in_getRegionClient into maas:master

[Christian Grabowski]

allow getRegionClient to use a busy connection

[Adam Collard (adam-collard)]

review: Approve

[Alberto Donato (ack)]

LGTM, +1

review: Approve

[MAAS Lander (maas-lander)]

UNIT TESTS
-b handle_all_connections_busy_in_getRegionClient lp:~cgrabowski/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: SUCCESS
COMMIT: e7653e53a746f2fd8b807e5a0096508840f33e9b

review: Approve

diff --git a/src/provisioningserver/rpc/__init__.py b/src/provisioningserver/rpc/__init__.py
index d5a36ed..0388356 100644
--- a/src/provisioningserver/rpc/__init__.py
+++ b/src/provisioningserver/rpc/__init__.py
@@ -25,4 +25,5 @@ def getRegionClient():
             "Cluster services are unavailable."
         )
     else:
-        return rpc_service.getClient()
+        # won't scale connections if existing connections are busy, but will always return connection if one exists
+        return rpc_service.getClient(busy_ok=True)
diff --git a/src/provisioningserver/rpc/clusterservice.py b/src/provisioningserver/rpc/clusterservice.py
index a7205db..d9baf8d 100644
--- a/src/provisioningserver/rpc/clusterservice.py
+++ b/src/provisioningserver/rpc/clusterservice.py
@@ -1230,7 +1230,7 @@ class ClusterClientService(TimerService):
         self.time_started = self.clock.seconds()
         super().startService()
 
-    def getClient(self):
+    def getClient(self, busy_ok=False):
         """Returns a :class:`common.Client` connected to a region.
 
         The client is chosen at random.
@@ -1246,9 +1246,13 @@ class ClusterClientService(TimerService):
                     self.connections.get_random_free_connection()
                 )
             except exceptions.AllConnectionsBusy as e:
-                for endpoint_conns in self.connections.values():
-                    if len(endpoint_conns) < self.connections._max_connections:
-                        raise e
+                if not busy_ok:
+                    for endpoint_conns in self.connections.values():
+                        if (
+                            len(endpoint_conns)
+                            < self.connections._max_connections
+                        ):
+                            raise e
                 # return a busy connection, assume it will free up or timeout
                 return common.Client(self.connections.get_random_connection())
 
diff --git a/src/provisioningserver/rpc/tests/test_clusterservice.py b/src/provisioningserver/rpc/tests/test_clusterservice.py
index 6f3e4f9..04b7d66 100644
--- a/src/provisioningserver/rpc/tests/test_clusterservice.py
+++ b/src/provisioningserver/rpc/tests/test_clusterservice.py
@@ -1300,6 +1300,24 @@ class TestClusterClientService(MAASTestCase):
         service.connections.connections = {}
         self.assertRaises(exceptions.NoConnectionsAvailable, service.getClient)
 
+    def test_getClient_returns_a_busy_connection_when_busy_ok_is_True(self):
+        service = ClusterClientService(Clock(), max_conns=1)
+        service.connections.connections = {
+            sentinel.eventloop01: [FakeBusyConnectionToRegion()],
+            sentinel.eventloop02: [FakeBusyConnectionToRegion()],
+            sentinel.eventloop03: [FakeBusyConnectionToRegion()],
+        }
+        client = service.getClient(busy_ok=True)
+        self.assertTrue(client._conn.in_use)
+        self.assertIn(
+            client,
+            {
+                common.Client(conn)
+                for conns in service.connections.values()
+                for conn in conns
+            },
+        )
+
     @inlineCallbacks
     def test_getClientNow_scales_connections_when_busy(self):
         service = ClusterClientService(Clock(), max_conns=2)