Merge ~skatsaounis/maas-ci/+git/system-tests:download-boot-images into ~maas-committers/maas-ci/+git/system-tests:master

[Stamatis Katsaounis]

Add oseries support

[MAAS Lander (maas-lander)]

UNIT TESTS
-b download-boot-images lp:~skatsaounis/maas-ci/+git/system-tests into -b master lp:~maas-committers/maas-ci/+git/system-tests

STATUS: SUCCESS
COMMIT: 215a3c407e1f9c712d32ad211f3590d69104ca8f

review: Approve

[MAAS Lander (maas-lander)]

UNIT TESTS
-b download-boot-images lp:~skatsaounis/maas-ci/+git/system-tests into -b master lp:~maas-committers/maas-ci/+git/system-tests

STATUS: FAILED
LOG: http://maas-ci.internal:8080/job/system-tests-tester/759/console
COMMIT: a2399387bfe20d5c2d198c4ed2944956b8447202

review: Needs Fixing

[MAAS Lander (maas-lander)]

UNIT TESTS
-b download-boot-images lp:~skatsaounis/maas-ci/+git/system-tests into -b master lp:~maas-committers/maas-ci/+git/system-tests

STATUS: FAILED
LOG: http://maas-ci.internal:8080/job/system-tests-tester/760/console
COMMIT: c5372e8511c8780d2fbaa919e44b47d253501f07

review: Needs Fixing

[MAAS Lander (maas-lander)]

UNIT TESTS
-b download-boot-images lp:~skatsaounis/maas-ci/+git/system-tests into -b master lp:~maas-committers/maas-ci/+git/system-tests

STATUS: SUCCESS
COMMIT: deb013d04f91a14a5066fd5c27fb36027c1d0931

review: Approve

[MAAS Lander (maas-lander)]

UNIT TESTS
-b download-boot-images lp:~skatsaounis/maas-ci/+git/system-tests into -b master lp:~maas-committers/maas-ci/+git/system-tests

STATUS: SUCCESS
COMMIT: 40f5ee8ece9b0d1684205574b1f536fd7d811ebb

review: Approve

[Jack Lloyd-Walters (lloydwaltersj)]

LGTM

review: Approve

diff --git a/pyproject.toml b/pyproject.toml
index 4a38e7a..a3b6850 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -5,7 +5,8 @@ markers = [
     "skip_if_dns_unconfigured", # skips tests if DNS configuration isn't present
     "skip_if_installed_from_snap", # Skips tests if MAAS is installed as a snap.
     "skip_if_installed_from_deb_package", # Skips tests if MAAS is installed from package.
-    "skip_if_packer_unconfigured" # Skips tests if Packer-MAAS is not configured.
+    "skip_if_packer_unconfigured", # Skips tests if Packer-MAAS is not configured.
+    "skip_if_os_not_supported" # Skips tests if the OS chosen is not compatible with the machine architecture
 ]
 log_level = "INFO"
 log_file = "systemtests.log"
diff --git a/systemtests/api.py b/systemtests/api.py
index 4abf1c2..83163d1 100644
--- a/systemtests/api.py
+++ b/systemtests/api.py
@@ -6,7 +6,7 @@ from subprocess import CalledProcessError
 from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, TypedDict, Union
 
 from .tls import MAAS_CONTAINER_CERTS_PATH
-from .utils import wait_for_machine
+from .utils import retries, wait_for_machine
 
 if TYPE_CHECKING:
     from logging import Logger
@@ -32,6 +32,16 @@ class BootImages(TypedDict):
     status: str
 
 
+class BootResource(TypedDict):
+    id: int
+    type: str
+    name: str
+    architecture: str
+    resource_uri: str
+    last_deployed: str | None
+    subarches: str
+
+
 class Version(TypedDict):
     capabilities: list[str]
     version: str
@@ -276,6 +286,10 @@ class AuthenticatedAPIClient:
         result: str = self.execute(["boot-resources", "import"], json_output=False)
         return result
 
+    def list_boot_resources(self) -> list[BootResource]:
+        result: list[BootResource] = self.execute(["boot-resources", "read"])
+        return result
+
     def stop_importing_boot_resources(self) -> str:
         result: str = self.execute(["boot-resources", "stop-import"], json_output=False)
         return result
@@ -284,6 +298,20 @@ class AuthenticatedAPIClient:
         result: bool = self.execute(["boot-resources", "is-importing"])
         return result
 
+    def wait_for_images(self) -> None:
+        for retry_info in retries(timeout=20 * 60, delay=10):
+            if not self.is_importing_boot_resources():
+                break
+        else:
+            raise AssertionError(
+                f"Image couldn't be imported after {retry_info.attempt} attempts"
+                f" over {retry_info.elapsed} seconds."
+            )
+
+    def import_and_wait_for_images(self) -> None:
+        self.import_boot_resources()
+        self.wait_for_images()
+
     def list_machines(self, **kwargs: str) -> list[Machine]:
         """
         machines read -h to know parameters available for kwargs
@@ -336,6 +364,8 @@ class AuthenticatedAPIClient:
         assert result["status_name"] == "Deploying"
         if expected_osystem := kwargs.get("osystem"):
             assert result["osystem"] == expected_osystem
+            if expected_distro_series := kwargs.get("distro_series"):
+                assert result["distro_series"] == expected_distro_series
         return result
 
     def create_ssh_key(self, public_key: str) -> SSHKey:
@@ -390,6 +420,28 @@ class AuthenticatedAPIClient:
                     ]
                 )
 
+    def create_boot_source_selection(
+        self,
+        architectures: Iterable[str],
+        osystem: str = "ubuntu",
+        oseries: str = "lunar",
+    ) -> None:
+        arches = [f"arches={arch}" for arch in architectures]
+        boot_sources = self.execute(["boot-sources", "read"])
+        for boot_source in boot_sources:
+            self.execute(
+                [
+                    "boot-source-selections",
+                    "create",
+                    str(boot_source["id"]),
+                    f"os={osystem}",
+                    f"release={oseries}",
+                    *arches,
+                    "subarches=*",
+                    "labels=*",
+                ]
+            )
+
     def rescue_machine(self, machine: Machine) -> None:
         result = self.execute(["machine", "rescue-mode", machine["system_id"]])
         assert result["status_name"] == "Entering rescue mode"
diff --git a/systemtests/conftest.py b/systemtests/conftest.py
index 60dfa4a..72957ee 100644
--- a/systemtests/conftest.py
+++ b/systemtests/conftest.py
@@ -50,6 +50,7 @@ from .fixtures import (
     skip_if_dns_unconfigured,
     skip_if_installed_from_deb_package,
     skip_if_installed_from_snap,
+    skip_if_os_not_supported,
     ssh_key,
     tag_all,
     testlog,
@@ -102,6 +103,7 @@ __all__ = [
     "skip_if_dns_unconfigured",
     "skip_if_installed_from_deb_package",
     "skip_if_installed_from_snap",
+    "skip_if_os_not_supported",
     "ssh_key",
     "tag_all",
     "testlog",
diff --git a/systemtests/fixtures.py b/systemtests/fixtures.py
index e53ba45..1046980 100644
--- a/systemtests/fixtures.py
+++ b/systemtests/fixtures.py
@@ -17,6 +17,7 @@ from .api import AuthenticatedAPIClient, UnauthenticatedMAASAPIClient
 from .config import ADMIN_EMAIL, ADMIN_PASSWORD, ADMIN_USER
 from .dnstester import DNSTester
 from .lxd import Instance, get_lxd
+from .machine_config import MachineConfig
 from .o11y import is_o11y_enabled, setup_o11y
 from .packer import PackerMain
 from .region import MAASRegion
@@ -634,6 +635,16 @@ def skip_if_ansible_playbooks_unconfigured(
 
 
 @pytest.fixture(autouse=True)
+def skip_if_os_not_supported(request: Any) -> None:
+    if marker := request.node.get_closest_marker("skip_if_os_not_supported"):
+        # Avoid instantiating machine configurations unless we have the marker applied
+        machine_config: MachineConfig = request.getfixturevalue("machine_config")
+        reason = marker.args[0]
+        if machine_config.not_supported:
+            pytest.skip(reason)
+
+
[e-mail](autouse=True)
 def skip_if_packer_unconfigured(request: Any, config: dict[str, Any]) -> None:
     """Skip tests that require Packer configuration."""
     marker = request.node.get_closest_marker("skip_if_packer_unconfigured")
diff --git a/systemtests/machine_config.py b/systemtests/machine_config.py
index e567d12..31c249b 100644
--- a/systemtests/machine_config.py
+++ b/systemtests/machine_config.py
@@ -54,6 +54,8 @@ class MachineConfig:
     devices_config: tuple[DeviceConfig, ...] = field(compare=False)
     architecture: str = "amd64"
     osystem: str = "ubuntu"
+    oseries: Optional[str] = None
+    cpu_model: Optional[str] = None
     lxd_profile: Optional[str] = None
 
     def __str__(self) -> str:
@@ -86,6 +88,22 @@ class MachineConfig:
         )
 
     @property
+    def not_supported(self) -> bool:
+        # XXX config.yaml doesn't contain a CPU model yet
+        # we need to update that for this to be useful
+        return (
+            (
+                (self.architecture, self.osystem)
+                == (
+                    "ppc64el",
+                    "ubuntu",
+                )
+            )
+            and isinstance(self.oseries, str)
+            and self.oseries >= "jammy"
+        )
+
+    @property
     def maas_power_cmd_power_parameters(self) -> list[str]:
         return maas_power_cmd_power_parameters(self.power_type, self.power_parameters)
 
diff --git a/systemtests/state.py b/systemtests/state.py
index 7ca5be8..0af9641 100644
--- a/systemtests/state.py
+++ b/systemtests/state.py
@@ -65,9 +65,15 @@ def import_images_and_wait_until_synced(
 ) -> None:
     architectures = set()
     osystems = set()
+    oseriess = set()
     for machine, power_config in config.get("machines", {}).get("hardware", {}).items():
         architectures.add(power_config.get("architecture", "amd64"))
-        osystems.add(power_config.get("osystem", "ubuntu"))
+        osystem = power_config.get("osystem", "ubuntu")
+        osystems.add(osystem)
+        if osystem == "ubuntu":
+            oseries = power_config.get("oseries")
+            if oseries:
+                oseriess.add(oseries)
 
     started_importing_regex = "^Started importing of boot images from"
     # Sometimes the region has already started importing images, we
@@ -77,12 +83,15 @@ def import_images_and_wait_until_synced(
     # TODO Arguably MAAS should do this for us!
     if authenticated_admin.is_importing_boot_resources():
         authenticated_admin.stop_importing_boot_resources()
+        LOG.info("Aborted importing of unconfigured boot resources.")
     if stream_url := config.get("images", {}).get("stream-url"):
         LOG.info(f"Setting image source to {stream_url}")
         boot_sources = authenticated_admin.list_boot_sources()
         # XXX: Why do we have >1 boot source?!
         for boot_source in boot_sources:
             authenticated_admin.update_boot_source_url(boot_source["id"], stream_url)
+        # wait for the image stream to switch
+        authenticated_admin.import_and_wait_for_images()
     with waits_for_event_after(
         authenticated_admin,
         event_type="Region import info",
@@ -91,6 +100,20 @@ def import_images_and_wait_until_synced(
         authenticated_admin.update_architectures_in_boot_source_selections(
             architectures
         )
+        for oseries in oseriess:
+            authenticated_admin.create_boot_source_selection(
+                architectures, oseries=oseries
+            )
+        boot_sources = authenticated_admin.list_boot_sources()
+        boot_resources = ", ".join(
+            sorted(
+                set(
+                    resource["name"]
+                    for resource in authenticated_admin.list_boot_resources()
+                )
+            )
+        )
+        LOG.info(f"Available boot resources: {boot_resources}")
         authenticated_admin.import_boot_resources()
     windows_path = None
     if "windows" in osystems:
@@ -115,9 +138,16 @@ def import_images_and_wait_until_synced(
             LOG.error(f"Unsupported file-store scheme '{scheme}'")
 
     region_start_point = time.time()
-    while authenticated_admin.is_importing_boot_resources():
-        LOG.debug("Sleeping for 10s for region image import")
-        time.sleep(10)
+    authenticated_admin.import_and_wait_for_images()
+    boot_resources = ", ".join(
+        sorted(
+            set(
+                resource["name"]
+                for resource in authenticated_admin.list_boot_resources()
+            )
+        )
+    )
+    LOG.info(f"available images: {boot_resources}")
 
     region_time_taken = time.time() - region_start_point
     LOG.info(
@@ -136,6 +166,7 @@ def import_images_and_wait_until_synced(
         )
         windows_time_taken = time.time() - windows_start_point
         LOG.info(f"Took {windows_time_taken:0.1f}s to upload Windows")
+    # testing of custom (packer) images
     for img in custom_images:
         custom_image_start_point = time.time()
         img_params = {
diff --git a/systemtests/tests_per_machine/test_machine.py b/systemtests/tests_per_machine/test_machine.py
index e244b17..0ae7bdc 100644
--- a/systemtests/tests_per_machine/test_machine.py
+++ b/systemtests/tests_per_machine/test_machine.py
@@ -28,6 +28,7 @@ if TYPE_CHECKING:
     from ..machine_config import MachineConfig
 
 
[e-mail]_if_os_not_supported("Machine hardware is not supported by OS")
 @test_steps("enlist", "metadata", "commission", "deploy", "test_image", "rescue")
 def test_full_circle(
     maas_api_client: AuthenticatedAPIClient,
@@ -116,7 +117,7 @@ def test_full_circle(
     if image_to_test is None:
         deploy_osystem, deploy_oseries = (
             machine_config.osystem,
-            None,
+            machine_config.oseries,
         )
         ssh_username = "ubuntu"
     else:
@@ -133,7 +134,7 @@ def test_full_circle(
         status="Deployed",
         abort_status="Failed deployment",
         machine_id=machine_config.name,
-        timeout=TIMEOUT,
+        timeout=TIMEOUT * 2,
     )
 
     assert machine["ip_addresses"][0] not in dynamic_range
diff --git a/utils/gen_config.py b/utils/gen_config.py
index 041b1af..17d84c8 100755
--- a/utils/gen_config.py
+++ b/utils/gen_config.py
@@ -193,6 +193,12 @@ def main(argv: list[str]) -> int:
         metavar="URL",
         help="Configure MAAS to consume images from this URL",
     )
+    parser.add_argument(
+        "--oseries",
+        type=str,
+        metavar="SERIES",
+        help="Deploy and test this SERIES",
+    )
 
     args = parser.parse_args(argv)
 
@@ -208,8 +214,8 @@ def main(argv: list[str]) -> int:
                 "or in --containers-image option"
             )
 
-    if args.image_stream_url:
-        config.setdefault("images", {})["stream-url"] = args.image_stream_url
+    if stream_url := args.image_stream_url:
+        config.setdefault("images", {})["stream-url"] = stream_url.strip()
     else:
         if "images" not in config or "stream-url" not in config["images"]:
             parser.error(
@@ -339,6 +345,11 @@ def main(argv: list[str]) -> int:
                 for name, details in hardware.items()
                 if name in args.machine
             }
+        if oseries := args.oseries:
+            for details in machines["hardware"].values():
+                details["oseries"] = oseries
+                if details.get("osystem", ""):
+                    details["osystem"] = "ubuntu"
 
     if vms:
         # Filter out VMs with name not listed in specified vm_machines