Merge ~cgrabowski/maas:LXD_create_cluster into maas:master

[Christian Grabowski]

correctly fetch commissioning info

cluster and pod naming tweaks

save discovered clusters and their containing pods

WIP save clusters

read discovered clusters from the region controller

return cluster on discover_pod

add discover cluster tests

discover all pods in a cluster

[MAAS Lander (maas-lander)]

UNIT TESTS
-b LXD_create_cluster lp:~cgrabowski/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci.internal:8080/job/maas/job/branch-tester/11115/console
COMMIT: de2476ab55b3a053b7207a691c8eaa4c62f7c631

review: Needs Fixing

[MAAS Lander (maas-lander)]

UNIT TESTS
-b LXD_create_cluster lp:~cgrabowski/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: SUCCESS
COMMIT: 237545542a6c49b6caa2274de3b1bb3c5928ff1f

review: Approve

[Alexsander de Souza (alexsander-souza)]

review: Approve

diff --git a/src/maasserver/clusterrpc/pods.py b/src/maasserver/clusterrpc/pods.py
index 68c37c0..f925fcb 100644
--- a/src/maasserver/clusterrpc/pods.py
+++ b/src/maasserver/clusterrpc/pods.py
@@ -69,7 +69,9 @@ def discover_pod(pod_type, context, pod_id=None, name=None, timeout=120):
 
     clients = getAllClients()
     dl = DeferredList(map(discover, clients), consumeErrors=True)
-    return dl.addCallback(_collect_results_and_failures, clients, "pod")
+    return dl.addCallback(
+        _collect_xor_results_and_failures, clients, "cluster", "pod"
+    )
 
 
 def get_best_discovered_result(discovered):
@@ -253,3 +255,24 @@ def _collect_results_and_failures(results, clients, result_key):
         else:
             failures[client.ident] = result.value
     return discovered, failures
+
+
+def _collect_xor_results_and_failures(
+    results, clients, result_key_a, result_key_b
+):
+    discovered, failures = {}, {}
+    for client, (success, result) in zip(clients, results):
+        if success:
+            if result_key_a in result:
+                discovered[client.ident] = result[result_key_a]
+            elif result_key_b in result:
+                discovered[client.ident] = result[result_key_b]
+            else:
+                raise ValueError(
+                    "neither %s or %s where found in result",
+                    result_key_a,
+                    result_key_b,
+                )
+        else:
+            failures[client.ident] = result.value
+    return discovered, failures
diff --git a/src/maasserver/clusterrpc/tests/test_pods.py b/src/maasserver/clusterrpc/tests/test_pods.py
index da49d55..fcb2ba6 100644
--- a/src/maasserver/clusterrpc/tests/test_pods.py
+++ b/src/maasserver/clusterrpc/tests/test_pods.py
@@ -37,6 +37,7 @@ from maastesting.matchers import MockCalledOnceWith
 from maastesting.testcase import MAASTestCase
 from metadataserver.models import NodeKey
 from provisioningserver.drivers.pod import (
+    DiscoveredCluster,
     DiscoveredPod,
     DiscoveredPodHints,
     DiscoveredPodProject,
@@ -216,6 +217,55 @@ class TestDiscoverPod(MAASTransactionServerTestCase):
             discovered[1], MatchesDict({rack_id: IsInstance(CancelledError)})
         )
 
+    @wait_for_reactor
+    @inlineCallbacks
+    def test_discovers_cluster(self):
+        rack_id = factory.make_name("system_id")
+        client = Mock()
+        client.ident = rack_id
+        pod_type = factory.make_name("pod")
+        cluster = DiscoveredCluster(
+            name=factory.make_name("cluster"),
+            project=factory.make_name("project"),
+            pods=[
+                DiscoveredPod(
+                    architectures=["amd64/generic"],
+                    cores=random.randint(1, 8),
+                    cpu_speed=random.randint(1000, 3000),
+                    memory=random.randint(1024, 4096),
+                    local_storage=random.randint(500, 1000),
+                    hints=DiscoveredPodHints(
+                        cores=random.randint(1, 8),
+                        cpu_speed=random.randint(1000, 3000),
+                        memory=random.randint(1024, 4096),
+                        local_storage=random.randint(500, 1000),
+                    ),
+                ),
+                DiscoveredPod(
+                    architectures=["amd64/generic"],
+                    cores=random.randint(1, 8),
+                    cpu_speed=random.randint(1000, 3000),
+                    memory=random.randint(1024, 4096),
+                    local_storage=random.randint(500, 1000),
+                    hints=DiscoveredPodHints(
+                        cores=random.randint(1, 8),
+                        cpu_speed=random.randint(1000, 3000),
+                        memory=random.randint(1024, 4096),
+                        local_storage=random.randint(500, 1000),
+                    ),
+                ),
+            ],
+        )
+        clients = []
+        client.return_value = succeed({"cluster": cluster})
+        clients.append(client)
+
+        self.patch(pods_module, "getAllClients").return_value = clients
+
+        discovered = yield discover_pod(pod_type, {})
+
+        self.assertEqual(({rack_id: cluster}, {}), discovered)
+
 
 class TestGetBestDiscoveredResult(MAASTestCase):
     def test_returns_one_of_the_discovered(self):
diff --git a/src/maasserver/models/bmc.py b/src/maasserver/models/bmc.py
index be9ba2a..3909ed5 100644
--- a/src/maasserver/models/bmc.py
+++ b/src/maasserver/models/bmc.py
@@ -665,8 +665,9 @@ class Pod(BMC):
         else:
             return None
 
-    def sync_hints(self, discovered_hints):
+    def sync_hints(self, discovered_hints, cluster=None):
         """Sync the hints with `discovered_hints`."""
+
         try:
             hints = self.hints
         except PodHints.DoesNotExist:
@@ -679,6 +680,8 @@ class Pod(BMC):
             hints.memory = discovered_hints.memory
         if discovered_hints.local_storage != -1:
             hints.local_storage = discovered_hints.local_storage
+        if cluster is not None:
+            hints.cluster = cluster
         hints.save()
 
     def add_tag(self, tag):
@@ -1478,7 +1481,7 @@ class Pod(BMC):
                 % (self.name, pool.name)
             )
 
-    def sync(self, discovered_pod, commissioning_user):
+    def sync(self, discovered_pod, commissioning_user, cluster=None):
         """Sync the pod and machines from the `discovered_pod`.
 
         This method ensures consistency with what is discovered by a pod
@@ -1497,7 +1500,7 @@ class Pod(BMC):
             self.power_parameters = power_params
         self.version = discovered_pod.version
         self.architectures = discovered_pod.architectures
-        if not self.name and discovered_pod.name:
+        if not self.name or cluster is not None and discovered_pod.name:
             self.name = discovered_pod.name
         self.capabilities = discovered_pod.capabilities
         if discovered_pod.cores != -1:
@@ -1510,7 +1513,7 @@ class Pod(BMC):
             self.local_storage = discovered_pod.local_storage
         self.tags = list(set(self.tags).union(discovered_pod.tags))
         self.save()
-        self.sync_hints(discovered_pod.hints)
+        self.sync_hints(discovered_pod.hints, cluster=cluster)
         self.sync_storage_pools(discovered_pod.storage_pools)
         self.sync_machines(discovered_pod.machines, commissioning_user)
         if discovered_pod.mac_addresses:
diff --git a/src/maasserver/models/tests/test_bmc.py b/src/maasserver/models/tests/test_bmc.py
index b12b0f5..460146b 100644
--- a/src/maasserver/models/tests/test_bmc.py
+++ b/src/maasserver/models/tests/test_bmc.py
@@ -1250,6 +1250,20 @@ class TestPod(MAASServerTestCase, PodTestMixin):
         self.assertNotEqual(-1, pod.hints.memory)
         self.assertNotEqual(-1, pod.hints.local_storage)
 
+    def test_sync_pod_with_cluster_saves_cluster_hint(self):
+        pod = factory.make_Pod()
+        cluster = factory.make_VMCluster()
+
+        discovered_pod = self.make_discovered_pod(
+            machines=[], storage_pools=[]
+        )
+        user = factory.make_User()
+        pod.sync(discovered_pod, user, cluster=cluster)
+
+        self.assertIsNotNone(pod.hints.cluster)
+        self.assertEqual(pod.hints.cluster.name, cluster.name)
+        self.assertEqual(pod.hints.cluster.project, cluster.project)
+
     def test_create_machine_ensures_unique_hostname(self):
         existing_machine = factory.make_Node()
         discovered_machine = self.make_discovered_machine()
diff --git a/src/maasserver/testing/factory.py b/src/maasserver/testing/factory.py
index c6cc75a..083d7d3 100644
--- a/src/maasserver/testing/factory.py
+++ b/src/maasserver/testing/factory.py
@@ -102,6 +102,7 @@ from maasserver.models import (
     VersionedTextFile,
     VirtualBlockDevice,
     VLAN,
+    VMCluster,
     VolumeGroup,
     Zone,
 )
@@ -2958,6 +2959,17 @@ class Factory(maastesting.factory.Factory):
             *args, group_type=FILESYSTEM_GROUP_TYPE.VMFS6, **kwargs
         )
 
+    def make_VMCluster(self, name=None, project=None):
+        if name is None:
+            name = factory.make_name("name")
+        if project is None:
+            project = factory.make_name("project")
+
+        return VMCluster.objects.create(
+            name=name,
+            project=project,
+        )
+
     def make_VirtualBlockDevice(
         self,
         name=None,
diff --git a/src/maasserver/tests/test_vmhost.py b/src/maasserver/tests/test_vmhost.py
index 7217ac6..8198315 100644
--- a/src/maasserver/tests/test_vmhost.py
+++ b/src/maasserver/tests/test_vmhost.py
@@ -9,6 +9,7 @@ from twisted.internet.defer import succeed
 from maasserver import vmhost as vmhost_module
 from maasserver.enum import BMC_TYPE
 from maasserver.exceptions import PodProblem
+from maasserver.models import PodHints
 from maasserver.testing.factory import factory
 from maasserver.testing.testcase import (
     MAASServerTestCase,
@@ -17,6 +18,7 @@ from maasserver.testing.testcase import (
 from maasserver.utils.threads import deferToDatabase
 from maastesting.crochet import wait_for
 from provisioningserver.drivers.pod import (
+    DiscoveredCluster,
     DiscoveredPod,
     DiscoveredPodHints,
     DiscoveredPodStoragePool,
@@ -75,6 +77,58 @@ def fake_pod_discovery(testcase):
     )
 
 
+def fake_cluster_discovery(testcase):
+    discovered_cluster = DiscoveredCluster(
+        name=factory.make_name("cluster"),
+        project=factory.make_name("project"),
+        pods=[
+            DiscoveredPod(
+                name=factory.make_name("pod"),
+                architectures=["amd64/generic"],
+                cores=random.randint(2, 4),
+                memory=random.randint(2048, 4096),
+                local_storage=random.randint(1024, 1024 * 1024),
+                cpu_speed=random.randint(2048, 4048),
+                hints=DiscoveredPodHints(
+                    cores=random.randint(2, 4),
+                    memory=random.randint(1024, 4096),
+                    local_storage=random.randint(1024, 1024 * 1024),
+                    cpu_speed=random.randint(2048, 4048),
+                ),
+                storage_pools=[
+                    DiscoveredPodStoragePool(
+                        id=factory.make_name("pool_id"),
+                        name=factory.make_name("name"),
+                        type=factory.make_name("type"),
+                        path="/var/lib/path/%s" % factory.make_name("path"),
+                        storage=random.randint(1024, 1024 * 1024),
+                    )
+                    for _ in range(3)
+                ],
+                clustered=True,
+            )
+            for _ in range(3)
+        ],
+        pod_addresses=["https://lxd-%d" % i for i in range(3)],
+    )
+    discovered_rack_1 = factory.make_RackController()
+    discovered_rack_2 = factory.make_RackController()
+    failed_rack = factory.make_RackController()
+    testcase.patch(vmhost_module, "post_commit_do")
+    testcase.patch(vmhost_module, "discover_pod").return_value = (
+        {
+            discovered_rack_1.system_id: discovered_cluster.pods[0],
+            discovered_rack_2.system_id: discovered_cluster.pods[0],
+        },
+        {failed_rack.system_id: factory.make_exception()},
+    )
+    return (
+        discovered_cluster,
+        [discovered_rack_1, discovered_rack_2],
+        [failed_rack],
+    )
+
+
 class TestDiscoverAndSyncVMHost(MAASServerTestCase):
     def test_sync_details(self):
         (
@@ -212,3 +266,142 @@ class TestDiscoverAndSyncVMHostAsync(MAASTransactionServerTestCase):
             self.assertEqual(str(exc), str(error))
         else:
             self.fail("No exception raised")
+
+
+class TestSyncVMCluster(MAASServerTestCase):
+    def test_sync_vmcluster_creates_cluster(self):
+        (
+            discovered_cluster,
+            discovered_racks,
+            failed_racks,
+        ) = fake_cluster_discovery(self)
+        zone = factory.make_Zone()
+        pod_info = make_pod_info()
+        power_parameters = {"power_address": pod_info["power_address"]}
+        orig_vmhost = factory.make_Pod(
+            zone=zone, pod_type=pod_info["type"], parameters=power_parameters
+        )
+        successes = {
+            rack_id: discovered_cluster for rack_id in discovered_racks
+        }
+        failures = {
+            rack_id: factory.make_exception() for rack_id in failed_racks
+        }
+        self.patch(vmhost_module, "discover_pod").return_value = (
+            successes,
+            failures,
+        )
+        vmhost = vmhost_module.discover_and_sync_vmhost(
+            orig_vmhost, factory.make_User()
+        )
+        self.assertEqual(vmhost.hints.cluster.name, discovered_cluster.name)
+        self.assertEqual(
+            vmhost.hints.cluster.project, discovered_cluster.project
+        )
+
+    def test_sync_vmcluster_creates_additional_pods(self):
+        (
+            discovered_cluster,
+            discovered_racks,
+            failed_racks,
+        ) = fake_cluster_discovery(self)
+        zone = factory.make_Zone()
+        pod_info = make_pod_info()
+        power_parameters = {"power_address": pod_info["power_address"]}
+        orig_vmhost = factory.make_Pod(
+            zone=zone, pod_type=pod_info["type"], parameters=power_parameters
+        )
+        successes = {
+            rack_id: discovered_cluster for rack_id in discovered_racks
+        }
+        failures = {
+            rack_id: factory.make_exception() for rack_id in failed_racks
+        }
+        self.patch(vmhost_module, "discover_pod").return_value = (
+            successes,
+            failures,
+        )
+        vmhost = vmhost_module.discover_and_sync_vmhost(
+            orig_vmhost, factory.make_User()
+        )
+        hints = PodHints.objects.filter(cluster=vmhost.hints.cluster)
+        pod_names = [hint.pod.name for hint in hints]
+        expected_names = [pod.name for pod in discovered_cluster.pods]
+        self.assertCountEqual(pod_names, expected_names)
+
+
+class TestSyncVMClusterAsync(MAASTransactionServerTestCase):
+
+    wait_for_reactor = wait_for(30)
+
+    @wait_for_reactor
+    async def test_sync_vmcluster_async_creates_cluster(self):
+        (
+            discovered_cluster,
+            discovered_racks,
+            failed_racks,
+        ) = await deferToDatabase(fake_cluster_discovery, self)
+        successes = {
+            rack_id: discovered_cluster for rack_id in discovered_racks
+        }
+        failures = {
+            rack_id: factory.make_exception() for rack_id in failed_racks
+        }
+        vmhost_module.discover_pod.return_value = succeed(
+            (successes, failures)
+        )
+        zone = await deferToDatabase(factory.make_Zone)
+        pod_info = await deferToDatabase(make_pod_info)
+        power_parameters = {"power_address": pod_info["power_address"]}
+        orig_vmhost = await deferToDatabase(
+            factory.make_Pod,
+            zone=zone,
+            pod_type=pod_info["type"],
+            parameters=power_parameters,
+        )
+        user = await deferToDatabase(factory.make_User)
+        vmhost = await vmhost_module.discover_and_sync_vmhost_async(
+            orig_vmhost, user
+        )
+        self.assertEqual(vmhost.hints.cluster.name, discovered_cluster.name)
+        self.assertEqual(
+            vmhost.hints.cluster.project, discovered_cluster.project
+        )
+
+    @wait_for_reactor
+    async def test_sync_vmcluster_async_creates_additional(self):
+        (
+            discovered_cluster,
+            discovered_racks,
+            failed_racks,
+        ) = await deferToDatabase(fake_cluster_discovery, self)
+        successes = {
+            rack_id: discovered_cluster for rack_id in discovered_racks
+        }
+        failures = {
+            rack_id: factory.make_exception() for rack_id in failed_racks
+        }
+        vmhost_module.discover_pod.return_value = succeed(
+            (successes, failures)
+        )
+        zone = await deferToDatabase(factory.make_Zone)
+        pod_info = await deferToDatabase(make_pod_info)
+        power_parameters = {"power_address": pod_info["power_address"]}
+        orig_vmhost = await deferToDatabase(
+            factory.make_Pod,
+            zone=zone,
+            pod_type=pod_info["type"],
+            parameters=power_parameters,
+        )
+        user = await deferToDatabase(factory.make_User)
+        vmhost = await vmhost_module.discover_and_sync_vmhost_async(
+            orig_vmhost, user
+        )
+
+        def _get_cluster_pod_names():
+            hints = PodHints.objects.filter(cluster=vmhost.hints.cluster)
+            return [hint.pod.name for hint in hints]
+
+        pod_names = await deferToDatabase(_get_cluster_pod_names)
+        expected_names = [pod.name for pod in discovered_cluster.pods]
+        self.assertCountEqual(pod_names, expected_names)
diff --git a/src/maasserver/vmhost.py b/src/maasserver/vmhost.py
index 1e55004..d907427 100644
--- a/src/maasserver/vmhost.py
+++ b/src/maasserver/vmhost.py
@@ -12,13 +12,16 @@ from maasserver.exceptions import PodProblem
 from maasserver.models import (
     BMCRoutableRackControllerRelationship,
     Event,
+    Pod,
     RackController,
+    VMCluster,
 )
 from maasserver.rpc import getClientFromIdentifiers
 from maasserver.utils import absolute_reverse
 from maasserver.utils.orm import post_commit_do, transactional
 from maasserver.utils.threads import deferToDatabase
 from metadataserver.models import NodeKey
+from provisioningserver.drivers.pod import DiscoveredCluster
 from provisioningserver.events import EVENT_TYPES
 
 
@@ -76,17 +79,21 @@ def discover_and_sync_vmhost(vmhost, user):
             "Unable to start the VM host discovery process. "
             "No rack controllers connected."
         )
-    _update_db(discovered_pod, discovered, vmhost, user)
-    # The data isn't committed to the database until the transaction is
-    # complete. The commissioning results must be sent after the
-    # transaction completes so the metadata server can process the
-    # data.
-    post_commit_do(
-        reactor.callLater,
-        0,
-        request_commissioning_results,
-        vmhost,
-    )
+    elif isinstance(discovered_pod, DiscoveredCluster):
+        vmhost = sync_vmcluster(discovered_pod, discovered, vmhost, user)
+    else:
+        vmhost = _update_db(discovered_pod, discovered, vmhost, user)
+        # The data isn't committed to the database until the transaction is
+        # complete. The commissioning results must be sent after the
+        # transaction completes so the metadata server can process the
+        # data.
+        post_commit_do(
+            reactor.callLater,
+            0,
+            request_commissioning_results,
+            vmhost,
+        )
+
     return vmhost
 
 
@@ -108,18 +115,110 @@ async def discover_and_sync_vmhost_async(vmhost, user):
             "Unable to start the VM host discovery process. "
             "No rack controllers connected."
         )
+    elif isinstance(discovered_pod, DiscoveredCluster):
+        vmhost = await sync_vmcluster_async(
+            discovered_pod, discovered, vmhost, user
+        )
+    else:
+        await deferToDatabase(
+            transactional(_update_db), discovered_pod, discovered, vmhost, user
+        )
+        await request_commissioning_results(vmhost)
+
+    return vmhost
+
+
+def _generate_cluster_power_params(pod, pod_address, first_host):
+    new_params = first_host.power_parameters.copy()
+    if pod_address.startswith("http://") or pod_address.startswith("https://"):
+        pod_address = pod_address.split("://")[1]
+    new_params["power_address"] = pod_address
+    new_params["instance_name"] = pod.name
+    return new_params
+
 
-    await deferToDatabase(
-        transactional(_update_db), discovered_pod, discovered, vmhost, user
+def sync_vmcluster(discovered_cluster, discovered, vmhost, user):
+    cluster = VMCluster.objects.create(
+        name=discovered_cluster.name or vmhost.name,
+        project=discovered_cluster.project,
     )
-    await request_commissioning_results(vmhost)
+    new_host = vmhost
+    for i, pod in enumerate(discovered_cluster.pods):
+        power_parameters = _generate_cluster_power_params(
+            pod, discovered_cluster.pod_addresses[i], vmhost
+        )
+        if (
+            power_parameters["power_address"]
+            != vmhost.power_parameters["power_address"]
+        ):
+            new_host = Pod.objects.create(
+                name=pod.name,
+                architectures=pod.architectures,
+                capabilities=pod.capabilities,
+                version=pod.version,
+                cores=pod.cores,
+                cpu_speed=pod.cpu_speed,
+                power_parameters=power_parameters,
+                power_type="lxd",  # VM clusters are only supported in LXD
+            )
+        new_host = _update_db(pod, discovered, new_host, user, cluster)
+        post_commit_do(
+            reactor.callLater,
+            0,
+            request_commissioning_results,
+            new_host,
+        )
+        if i == 0:
+            vmhost = new_host
     return vmhost
 
 
-def _update_db(discovered_pod, discovered, vmhost, user):
+async def sync_vmcluster_async(discovered_cluster, discovered, vmhost, user):
+    def _transaction(discovered_cluster, discovered, vmhost, user):
+        cluster = VMCluster.objects.create(
+            name=discovered_cluster.name or vmhost.name,
+            project=discovered_cluster.project,
+        )
+        new_hosts = []
+        for i, pod in enumerate(discovered_cluster.pods):
+            power_parameters = _generate_cluster_power_params(
+                pod, discovered_cluster.pod_addresses[i], vmhost
+            )
+            new_host = vmhost
+            if (
+                power_parameters["power_address"]
+                != vmhost.power_parameters["power_address"]
+            ):
+                new_host = Pod.objects.create(
+                    name=pod.name,
+                    architectures=pod.architectures,
+                    capabilities=pod.capabilities,
+                    version=pod.version,
+                    cores=pod.cores,
+                    cpu_speed=pod.cpu_speed,
+                    power_parameters=power_parameters,
+                    power_type="lxd",  # VM clusters are only supported in LXD
+                )
+            new_host = _update_db(pod, discovered, new_host, user, cluster)
+            new_hosts.append(new_host)
+        return new_hosts
+
+    new_hosts = await deferToDatabase(
+        transactional(_transaction),
+        discovered_cluster,
+        discovered,
+        vmhost,
+        user,
+    )
+    for new_host in new_hosts:
+        await request_commissioning_results(new_host)
+    return new_hosts[0]
+
+
+def _update_db(discovered_pod, discovered, vmhost, user, cluster=None):
     # If this is a new instance it will be stored in the database at the end of
     # sync.
-    vmhost.sync(discovered_pod, user)
+    vmhost.sync(discovered_pod, user, cluster=cluster)
 
     # Save which rack controllers can route and which cannot.
     discovered_rack_ids = [rack_id for rack_id, _ in discovered[0].items()]
diff --git a/src/provisioningserver/drivers/pod/__init__.py b/src/provisioningserver/drivers/pod/__init__.py
index fa85a33..7494950 100644
--- a/src/provisioningserver/drivers/pod/__init__.py
+++ b/src/provisioningserver/drivers/pod/__init__.py
@@ -295,6 +295,21 @@ class DiscoveredPod:
         converter=converter_list(DiscoveredPodStoragePool),
         default=attr.Factory(list),
     )
+    clustered = attr.ib(converter=bool, default=False)
+
+
+@attr.s
+class DiscoveredCluster:
+    """Discovered cluster information"""
+
+    name = attr.ib(converter=converter_obj(str, optional=True), default=None)
+    project = attr.ib(converter=str, default="")
+    pods = attr.ib(
+        converter=converter_list(DiscoveredPod), default=attr.Factory(list)
+    )
+    pod_addresses = attr.ib(
+        converter=converter_list(str), default=attr.Factory(list)
+    )
 
 
 @attr.s
diff --git a/src/provisioningserver/drivers/pod/lxd.py b/src/provisioningserver/drivers/pod/lxd.py
index 9296378..9261931 100644
--- a/src/provisioningserver/drivers/pod/lxd.py
+++ b/src/provisioningserver/drivers/pod/lxd.py
@@ -29,6 +29,7 @@ from provisioningserver.drivers import (
 )
 from provisioningserver.drivers.pod import (
     Capabilities,
+    DiscoveredCluster,
     DiscoveredMachine,
     DiscoveredMachineBlockDevice,
     DiscoveredMachineInterface,
@@ -310,7 +311,29 @@ class LXDPodDriver(PodDriver):
     def discover(self, pod_id: int, context: dict):
         """Discover all Pod host resources."""
         with self._get_client(pod_id, context) as client:
-            return self._discover(client, pod_id, context)
+            discovered_pod = self._discover(client, pod_id, context)
+            if discovered_pod.clustered:
+                return self._discover_cluster(client, context)
+            return discovered_pod
+
+    def _discover_cluster(self, client: Client, context: dict):
+        discovered_cluster = DiscoveredCluster(
+            name=context.get("instance_name"),
+            project=client.project,
+        )
+
+        cluster_members = client.cluster.members.all()
+
+        for member in cluster_members:
+            discovered_context = context.copy()
+            discovered_context["instance_name"] = member.server_name
+            discovered_context["power_address"] = member.url
+            with self._get_client(-1, discovered_context) as client:
+                discovered_cluster.pods.append(
+                    self._discover(client, -1, discovered_context)
+                )
+                discovered_cluster.pod_addresses.append(member.url)
+        return discovered_cluster
 
     def _discover(self, client: Client, pod_id: int, context: dict):
         self._check_required_extensions(client)
@@ -345,6 +368,7 @@ class LXDPodDriver(PodDriver):
                 Capabilities.OVER_COMMIT,
                 Capabilities.STORAGE_POOLS,
             ],
+            clustered=environment["server_clustered"],
         )
 
         # Discover networks. "unknown" interfaces are considered too to match
diff --git a/src/provisioningserver/drivers/pod/tests/test_lxd.py b/src/provisioningserver/drivers/pod/tests/test_lxd.py
index cee23a4..05e8397 100644
--- a/src/provisioningserver/drivers/pod/tests/test_lxd.py
+++ b/src/provisioningserver/drivers/pod/tests/test_lxd.py
@@ -147,15 +147,16 @@ class FakeClient:
 class FakeLXD:
     """A fake LXD server."""
 
-    def __init__(self):
+    def __init__(self, name="lxd-server", clustered=False):
         # global details
         self.host_info = {
             "api_extensions": sorted(lxd_module.LXD_REQUIRED_EXTENSIONS),
             "environment": {
                 "architectures": ["x86_64", "i686"],
                 "kernel_architecture": "x86_64",
-                "server_name": "lxd-server",
+                "server_name": name,
                 "server_version": "4.1",
+                "server_clustered": clustered,
             },
         }
         self.resources = {}
@@ -166,6 +167,9 @@ class FakeLXD:
         self.projects = MagicMock()
         self.storage_pools = MagicMock()
         self.virtual_machines = MagicMock()
+        if clustered:
+            self.cluster = MagicMock()
+            self.cluster.members = MagicMock()
 
         self._client_behaviors = None
 
@@ -203,9 +207,131 @@ class FakeLXD:
         self._client_behaviors.append(behaviors)
 
 
+class FakeLXDCluster:
+    """A fake cluster of LXD servers"""
+
+    def __init__(self, num_pods=1):
+        self.pods = [
+            FakeLXD(name="lxd-server-%d" % i, clustered=True)
+            for i in range(0, num_pods)
+        ]
+        self.pod_addresses = []
+
+        self.clients = []
+        self.client_idx = 0
+        for pod in self.pods:
+            self.clients.append(pod.make_client)
+
+        self._make_members()
+
+    def _make_members(self):
+        members = []
+        for i, pod in enumerate(self.pods):
+            member = Mock()
+            member.architectures = pod.host_info["environment"][
+                "architectures"
+            ]
+            member.server_name = pod.host_info["environment"]["server_name"]
+            member.url = "http://lxd-%d" % i
+            self.pod_addresses.append(member.url)
+            members.append(member)
+        for pod in self.pods:
+            pod.cluster.members.all.return_value = members
+
+    def make_client(
+        self,
+        endpoint="https://lxd",
+        project="default",
+        cert=None,
+        verify=False,
+    ):
+        if self.client_idx == len(self.clients):
+            self.client_idx = 0
+        client = self.clients[self.client_idx](endpoint, project, cert, verify)
+        client._PROXIES += ("cluster", "cluster/members")
+        self.client_idx += 1
+        return client
+
+
 SAMPLE_CERT = generate_certificate("maas")
 
 
+def _make_maas_certs(test_case):
+    tempdir = Path(test_case.useFixture(TempDir()).path)
+    test_case.useFixture(EnvironmentVariable("MAAS_ROOT", str(tempdir)))
+    test_case.certs_dir = tempdir / "etc/maas/certificates"
+    test_case.certs_dir.mkdir(parents=True)
+    maas_cert = test_case.certs_dir / "maas.crt"
+    maas_cert.touch()
+    maas_key = test_case.certs_dir / "maas.key"
+    maas_key.touch()
+    return str(maas_cert), str(maas_key)
+
+
+def _make_context(with_cert=True, with_password=True, extra=None):
+    params = {
+        "power_address": "".join(
+            [
+                factory.make_name("power_address"),
+                ":%s" % factory.pick_port(),
+            ]
+        ),
+        "instance_name": factory.make_name("instance_name"),
+        "project": factory.make_name("project"),
+    }
+    if with_cert:
+        params["certificate"] = SAMPLE_CERT.certificate_pem()
+        params["key"] = SAMPLE_CERT.private_key_pem()
+    if with_password:
+        params["password"] = factory.make_name("password")
+    if not extra:
+        extra = {}
+    return {**params, **extra}
+
+
+class TestClusteredLXDPodDriver(MAASTestCase):
+    run_tests_with = MAASTwistedRunTest.make_factory(timeout=5)
+
+    def setUp(self):
+        super().setUp()
+        self.fake_lxd_cluster = FakeLXDCluster(num_pods=3)
+        self.fake_lxd = self.fake_lxd_cluster.pods[0]
+        self.driver = lxd_module.LXDPodDriver()
+        self.driver._pylxd_client_class = self.fake_lxd_cluster.make_client
+
+    def make_maas_certs(self):
+        return _make_maas_certs(self)
+
+    def make_context(self, with_cert=True, with_password=True, extra=None):
+        return _make_context(with_cert, with_password, extra)
+
+    @inlineCallbacks
+    def test_discover_discovers_cluster(self):
+        mac_address = factory.make_mac_address()
+        lxd_net1 = Mock(type="physical")
+        lxd_net1.state.return_value = FakeNetworkState(mac_address)
+        # virtual interfaces are excluded
+        lxd_net2 = Mock(type="bridge")
+        lxd_net2.state.return_value = FakeNetworkState(
+            factory.make_mac_address()
+        )
+        self.fake_lxd.networks.all.return_value = [lxd_net1, lxd_net2]
+        context = self.make_context()
+        expected_names = [
+            pod.host_info["environment"]["server_name"]
+            for pod in self.fake_lxd_cluster.pods
+        ]
+        discovered_cluster = yield self.driver.discover(None, context)
+        self.assertEqual(context["instance_name"], discovered_cluster.name)
+        self.assertEqual(context["project"], discovered_cluster.project)
+        discovered_names = [pod.name for pod in discovered_cluster.pods]
+        self.assertCountEqual(expected_names, discovered_names)
+        self.assertCountEqual(
+            self.fake_lxd_cluster.pod_addresses,
+            discovered_cluster.pod_addresses,
+        )
+
+
 class TestLXDPodDriver(MAASTestCase):
 
     run_tests_with = MAASTwistedRunTest.make_factory(timeout=5)
@@ -217,35 +343,10 @@ class TestLXDPodDriver(MAASTestCase):
         self.driver._pylxd_client_class = self.fake_lxd.make_client
 
     def make_maas_certs(self):
-        tempdir = Path(self.useFixture(TempDir()).path)
-        self.useFixture(EnvironmentVariable("MAAS_ROOT", str(tempdir)))
-        self.certs_dir = tempdir / "etc/maas/certificates"
-        self.certs_dir.mkdir(parents=True)
-        maas_cert = self.certs_dir / "maas.crt"
-        maas_cert.touch()
-        maas_key = self.certs_dir / "maas.key"
-        maas_key.touch()
-        return str(maas_cert), str(maas_key)
+        return _make_maas_certs(self)
 
     def make_context(self, with_cert=True, with_password=True, extra=None):
-        params = {
-            "power_address": "".join(
-                [
-                    factory.make_name("power_address"),
-                    ":%s" % factory.pick_port(),
-                ]
-            ),
-            "instance_name": factory.make_name("instance_name"),
-            "project": factory.make_name("project"),
-        }
-        if with_cert:
-            params["certificate"] = SAMPLE_CERT.certificate_pem()
-            params["key"] = SAMPLE_CERT.private_key_pem()
-        if with_password:
-            params["password"] = factory.make_name("password")
-        if not extra:
-            extra = {}
-        return {**params, **extra}
+        return _make_context(with_cert, with_password, extra)
 
     def test_missing_packages(self):
         self.assertEqual(self.driver.detect_missing_packages(), [])
diff --git a/src/provisioningserver/rpc/cluster.py b/src/provisioningserver/rpc/cluster.py
index b245c61..7d208af 100644
--- a/src/provisioningserver/rpc/cluster.py
+++ b/src/provisioningserver/rpc/cluster.py
@@ -628,8 +628,17 @@ class DiscoverPod(amp.Command):
     response = [
         (
             b"pod",
-            AttrsClassArgument("provisioningserver.drivers.pod.DiscoveredPod"),
-        )
+            AttrsClassArgument(
+                "provisioningserver.drivers.pod.DiscoveredPod", optional=True
+            ),
+        ),
+        (
+            b"cluster",
+            AttrsClassArgument(
+                "provisioningserver.drivers.pod.DiscoveredCluster",
+                optional=True,
+            ),
+        ),
     ]
     errors = {
         exceptions.UnknownPodType: b"UnknownPodType",
diff --git a/src/provisioningserver/rpc/pods.py b/src/provisioningserver/rpc/pods.py
index 9173ae5..96c5e98 100644
--- a/src/provisioningserver/rpc/pods.py
+++ b/src/provisioningserver/rpc/pods.py
@@ -10,6 +10,7 @@ from twisted.internet.defer import Deferred, ensureDeferred
 from twisted.internet.threads import deferToThread
 
 from provisioningserver.drivers.pod import (
+    DiscoveredCluster,
     DiscoveredMachine,
     DiscoveredPod,
     DiscoveredPodHints,
@@ -85,6 +86,8 @@ def discover_pod(pod_type, context, pod_id=None, name=None):
         """Convert the result to send over RPC."""
         if result is None:
             raise PodActionFail("unable to discover pod information.")
+        elif isinstance(result, DiscoveredCluster):
+            return {"cluster": result}
         elif not isinstance(result, DiscoveredPod):
             raise PodActionFail(
                 "bad pod driver '%s'; 'discover' returned invalid result."
diff --git a/src/provisioningserver/rpc/tests/test_pods.py b/src/provisioningserver/rpc/tests/test_pods.py
index e0e078d..058005a 100644
--- a/src/provisioningserver/rpc/tests/test_pods.py
+++ b/src/provisioningserver/rpc/tests/test_pods.py
@@ -11,12 +11,13 @@ from unittest.mock import call, MagicMock
 from urllib.parse import urlparse
 
 from testtools import ExpectedException
-from twisted.internet.defer import fail, inlineCallbacks, succeed
+from twisted.internet.defer import fail, inlineCallbacks, returnValue, succeed
 
 from maastesting.factory import factory
 from maastesting.matchers import MockCallsMatch
 from maastesting.testcase import MAASTestCase, MAASTwistedRunTest
 from provisioningserver.drivers.pod import (
+    DiscoveredCluster,
     DiscoveredMachine,
     DiscoveredPod,
     DiscoveredPodHints,
@@ -173,6 +174,26 @@ class TestDiscoverPod(MAASTestCase):
         ):
             yield pods.discover_pod(fake_driver.name, {})
 
+    @inlineCallbacks
+    def test_handlers_driver_returning_cluster(self):
+        fake_driver = MagicMock()
+        fake_driver.name = factory.make_name("pod")
+        expected_cluster = DiscoveredCluster(
+            name=factory.make_name("name"),
+            project=factory.make_name("project"),
+            pods=[
+                DiscoveredPod(name=factory.make_name("pod-name"))
+                for _ in range(0, 3)
+            ],
+        )
+        fake_driver.discover.return_value = returnValue(expected_cluster)
+        self.patch(PodDriverRegistry, "get_item").return_value = fake_driver
+        result = yield pods.discover_pod(fake_driver.name, {})
+        discovered_cluster = result["cluster"]
+        self.assertEqual(expected_cluster.name, discovered_cluster.name)
+        self.assertEqual(expected_cluster.project, discovered_cluster.project)
+        self.assertCountEqual(expected_cluster.pods, discovered_cluster.pods)
+
 
 class TestComposeMachine(MAASTestCase):