Merge ~troyanov/maas:backport-21cf88e-3.5 into maas:3.5

[Anton Troyanov]

fix: get Ubuntu kernel for ephemeral deployments

fixes LP#2057939

(cherry picked from commit 21cf88e8a2f7855881728f965b7ca85ab59f4268)

[Anton Troyanov (troyanov)]

review: Approve

diff --git a/src/maasserver/rpc/boot.py b/src/maasserver/rpc/boot.py
index cee4453..f04da81 100644
--- a/src/maasserver/rpc/boot.py
+++ b/src/maasserver/rpc/boot.py
@@ -92,70 +92,95 @@ def event_log_pxe_request(machine, purpose):
     )
 
 
-def get_boot_filenames(
-    arch,
-    subarch,
-    osystem,
-    series,
-    commissioning_osystem=undefined,
-    commissioning_distro_series=undefined,
-):
-    """Return the filenames of the kernel, initrd, and boot_dtb for the boot
-    resource."""
-    if subarch == "generic":
-        # MAAS doesn't store in the BootResource table what subarch is the
-        # generic subarch so lookup what the generic subarch maps to.
-        try:
-            boot_resource_subarch = get_working_kernel(
-                subarch,
-                None,
-                f"{arch}/{subarch}",
-                osystem,
-                series,
-                commissioning_osystem=commissioning_osystem,
-                commissioning_distro_series=commissioning_distro_series,
-            )
-        except ValidationError:
-            # It's possible that no kernel's exist at all for this arch,
-            # subarch, osystem, series combination. In that case just fallback
-            # to 'generic'.
-            boot_resource_subarch = "generic"
-    else:
-        boot_resource_subarch = subarch
-
+def _get_files_map(osystem, oseries, arch, subarch, exclude=None):
+    exclude = exclude or []
     try:
-        # Get the filename for the kernel, initrd, and boot_dtb the rack should
-        # use when booting.
         boot_resource = BootResource.objects.get(
-            architecture=f"{arch}/{boot_resource_subarch}",
-            name=f"{osystem}/{series}",
+            architecture=f"{arch}/{subarch}",
+            name=f"{osystem}/{oseries}",
         )
-        boot_resource_set = boot_resource.get_latest_complete_set()
-        boot_resource_files = {
+        bset = boot_resource.get_latest_complete_set()
+        return {
             bfile.filetype: "/".join(
                 [
                     bfile.sha256,
                     osystem,
                     arch,
-                    boot_resource_subarch,
-                    series,
-                    boot_resource_set.label,
+                    subarch,
+                    oseries,
+                    bset.label,
                     bfile.filename,
                 ]
             )
-            for bfile in boot_resource_set.files.all()
+            for bfile in bset.files.all()
+            if bfile.filetype not in exclude
         }
     except ObjectDoesNotExist:
+        return {}
+
+
+def _xlate_generic_subarch(
+    arch,
+    subarch,
+    kernel_osystem,
+    kernel_release,
+    commissioning_osystem,
+    commissioning_distro_series,
+):
+    # MAAS doesn't store in the BootResource table what subarch is the
+    # generic subarch so lookup what the generic subarch maps to.
+    try:
+        return get_working_kernel(
+            subarch,
+            None,
+            f"{arch}/{subarch}",
+            kernel_osystem,
+            kernel_release,
+            commissioning_osystem=commissioning_osystem,
+            commissioning_distro_series=commissioning_distro_series,
+        )
+    except ValidationError:
+        # It's possible that no kernel's exist at all for this arch,
+        # subarch, osystem, series combination. In that case just fallback
+        # to 'generic'.
+        return "generic"
+
+
+def get_boot_filenames(
+    arch,
+    subarch,
+    osystem,
+    series,
+    commissioning_osystem=undefined,
+    commissioning_distro_series=undefined,
+):
+    """Return the filenames of the kernel, initrd, and boot_dtb for the boot
+    resource."""
+    if subarch == "generic":
+        kernel_subarch = _xlate_generic_subarch(
+            arch,
+            subarch,
+            osystem,
+            series,
+            commissioning_osystem,
+            commissioning_distro_series,
+        )
+    else:
+        kernel_subarch = subarch
+
+    res_files = _get_files_map(osystem, series, arch, kernel_subarch)
+    if not res_files and subarch != kernel_subarch:
+        res_files = _get_files_map(osystem, series, arch, subarch)
+
+    if not res_files:
         # If a filename can not be found return None to allow the rack to
         # figure out what todo.
         return None, None, None, None
-    kernel = boot_resource_files.pop(BOOT_RESOURCE_FILE_TYPE.BOOT_KERNEL, None)
-    initrd = boot_resource_files.pop(BOOT_RESOURCE_FILE_TYPE.BOOT_INITRD, None)
-    boot_dtb = boot_resource_files.pop(BOOT_RESOURCE_FILE_TYPE.BOOT_DTB, None)
-    if len(boot_resource_files) > 0:
-        rootfs = next(iter(boot_resource_files.values()))
-    else:
-        rootfs = None
+
+    kernel = res_files.pop(BOOT_RESOURCE_FILE_TYPE.BOOT_KERNEL, None)
+    initrd = res_files.pop(BOOT_RESOURCE_FILE_TYPE.BOOT_INITRD, None)
+    boot_dtb = res_files.pop(BOOT_RESOURCE_FILE_TYPE.BOOT_DTB, None)
+    rootfs = next(iter(res_files.values()), None)
     return kernel, initrd, boot_dtb, rootfs
 
 
@@ -698,6 +723,15 @@ def get_config(
     boot_purpose = get_final_boot_purpose(machine, arch, purpose)
 
     kernel_osystem, kernel_release = osystem, series
+
+    kernel, initrd, boot_dtb, rootfs = get_boot_filenames(
+        arch,
+        subarch,
+        osystem,
+        series,
+        commissioning_osystem=configs["commissioning_osystem"],
+        commissioning_distro_series=configs["commissioning_distro_series"],
+    )
     # For custom image ephemeral deployments we use the default (ubuntu) commissioning os/distro kernel
     if (
         machine is not None
@@ -708,16 +742,9 @@ def get_config(
             configs["commissioning_osystem"],
             configs["commissioning_distro_series"],
         )
-
-    # FIXME alexsander-souza: check the ephemeral deployment case
-    kernel, initrd, boot_dtb, rootfs = get_boot_filenames(
-        arch,
-        subarch,
-        osystem,
-        series,
-        commissioning_osystem=configs["commissioning_osystem"],
-        commissioning_distro_series=configs["commissioning_distro_series"],
-    )
+        kernel, initrd, boot_dtb, _ = get_boot_filenames(
+            arch, subarch, kernel_osystem, kernel_release
+        )
 
     # Return the params to the rack controller. Include the system_id only
     # if the machine was known.
diff --git a/src/maasserver/rpc/tests/test_boot.py b/src/maasserver/rpc/tests/test_boot.py
index e041906..ea0cc64 100644
--- a/src/maasserver/rpc/tests/test_boot.py
+++ b/src/maasserver/rpc/tests/test_boot.py
@@ -373,21 +373,29 @@ class TestGetConfig(MAASServerTestCase):
         local_ip = factory.make_ip_address()
         remote_ip = factory.make_ip_address()
         node = self.make_node_with_extra(
-            status=NODE_STATUS.DEPLOYED, netboot=False, ephemeral_deploy=True
+            status=NODE_STATUS.DEPLOYED,
+            netboot=False,
+            ephemeral_deploy=True,
+            boot_cluster_ip=local_ip,
+            osystem="centos",
+            distro_series="8",
         )
-        node.boot_cluster_ip = local_ip
-        node.osystem = "centos"
-        node.distro_series = "8"
-        node.architecture = "amd64/generic"
-        node.save()
+        arch = node.architecture.split("/")[0]
         mac = node.get_boot_interface().mac_address
         self.patch_autospec(boot_module, "event_log_pxe_request")
+        factory.make_default_ubuntu_release_bootable(arch=arch)
+        factory.make_usable_boot_resource(
+            name="centos/8",
+            architecture=f"{arch}/generic",
+            image_filetype=BOOT_RESOURCE_FILE_TYPE.ROOT_TGZ,
+        )
+
         config = get_config(
             rack_controller.system_id, local_ip, remote_ip, mac=mac
         )
         self.assertEqual(config["osystem"], "centos")
         self.assertEqual(config["release"], "8")
-        self.assertEqual(config["arch"], "amd64")
+        self.assertEqual(config["arch"], arch)
         self.assertEqual(config["subarch"], "generic")
         self.assertEqual(
             config["kernel_osystem"],
@@ -398,6 +406,9 @@ class TestGetConfig(MAASServerTestCase):
             Config.objects.get_config(name="commissioning_distro_series"),
         )
         self.assertEqual(config["purpose"], "xinstall")
+        self.assertIn(f"/ubuntu/{arch}/hwe-", config["kernel"])
+        self.assertIn(f"/ubuntu/{arch}/hwe-", config["initrd"])
+        self.assertIn(f"/centos/{arch}/generic/8", config["xinstall_path"])
 
     # See https://github.com/canonical/cloud-init/issues/4418 for more details
     def test_preseed_url_not_using_domain_names_for_custom_ephemeral_deployments(
@@ -1661,6 +1672,53 @@ class TestGetBootFilenames(MAASServerTestCase):
             rootfs,
         )
 
+    def test_custom_deployment(self):
+        architecture = make_usable_architecture(self)
+        arch = architecture.split("/")[0]
+        custom_res = factory.make_usable_boot_resource(
+            name="centos/8",
+            architecture=f"{arch}/generic",
+            image_filetype=BOOT_RESOURCE_FILE_TYPE.ROOT_TGZ,
+        )
+        custom_rset = custom_res.sets.first()
+        kernel, initrd, _, rootfs = get_boot_filenames(
+            arch,
+            "generic",
+            "centos",
+            "8",
+        )
+
+        self.assertEqual(
+            self.composeURL(
+                custom_res,
+                custom_rset,
+                custom_rset.files.get(
+                    filetype=BOOT_RESOURCE_FILE_TYPE.BOOT_KERNEL
+                ),
+            ),
+            kernel,
+        )
+        self.assertEqual(
+            self.composeURL(
+                custom_res,
+                custom_rset,
+                custom_rset.files.get(
+                    filetype=BOOT_RESOURCE_FILE_TYPE.BOOT_INITRD
+                ),
+            ),
+            initrd,
+        )
+        self.assertEqual(
+            self.composeURL(
+                custom_res,
+                custom_rset,
+                custom_rset.files.get(
+                    filetype=BOOT_RESOURCE_FILE_TYPE.ROOT_TGZ
+                ),
+            ),
+            rootfs,
+        )
+
     def test_get_filenames_finds_subarch_when_generic(self):
         release = factory.make_default_ubuntu_release_bootable()
         arch = release.architecture.split("/")[0]