Merge ~adam-collard/maas-ci/+git/system-tests:lxd-vm-bug into ~maas-committers/maas-ci/+git/system-tests:master

[Adam Collard]

Several small fixes to get system-tests green

- No need to actually look at the results from grep

- Timeout on login

Avoid the multiple hour hang that we hit in http://maas-integration-ci.internal:8080/job/maas-system-tests/1355/

- Retry device removal

- Reverse the event logs so most recent is at the bottom

[Jack Lloyd-Walters (lloydwaltersj)]

review: Approve

[MAAS Lander (maas-lander)]

UNIT TESTS
-b lxd-vm-bug lp:~adam-collard/maas-ci/+git/system-tests into -b master lp:~maas-committers/maas-ci/+git/system-tests

STATUS: SUCCESS
COMMIT: ad35e0bff700625ef2d85b1d6812fbb8bed45215

review: Approve

diff --git a/systemtests/api.py b/systemtests/api.py
index 46d9dad..8279739 100644
--- a/systemtests/api.py
+++ b/systemtests/api.py
@@ -108,7 +108,7 @@ class UnauthenticatedMAASAPIClient:
         cmd = ["login", session, self.url, token]
         if self.url.startswith("https://"):
             cmd += ["--cacerts", f"{MAAS_CONTAINER_CERTS_PATH}cacerts.pem"]
-        output = self.execute(cmd)
+        output = self.execute(cmd, base_cmd=["timeout", "5m", "maas"])
         return output, AuthenticatedAPIClient(self, session)
 
 
diff --git a/systemtests/fixtures.py b/systemtests/fixtures.py
index 0bbfaf9..8a73fa3 100644
--- a/systemtests/fixtures.py
+++ b/systemtests/fixtures.py
@@ -590,7 +590,7 @@ def maas_client_container(
             f"{MAAS_CONTAINER_CERTS_PATH}cacerts.pem",
         )
         try:
-            lxd.execute(container, ["sh", "-c", "grep maas.under.test /etc/hosts"])
+            lxd.execute(container, ["sh", "-c", "grep -q maas.under.test /etc/hosts"])
         except subprocess.CalledProcessError:
             lxd.execute(
                 container,
diff --git a/systemtests/lxd.py b/systemtests/lxd.py
index c2c1540..a7fec6e 100644
--- a/systemtests/lxd.py
+++ b/systemtests/lxd.py
@@ -9,7 +9,7 @@ from itertools import chain
 from pathlib import Path
 from typing import TYPE_CHECKING, Optional
 
-from retry.api import retry
+from retry import retry
 
 from .device_config import DeviceConfig, fmt_lxd_options
 from .subprocess import run_with_logging
@@ -82,7 +82,7 @@ class CLILXD:
         self.logger.info(f"Container {name} created.")
         self.logger.info("Waiting for boot to finish...")
 
-        @retry(tries=120, delay=1, exceptions=CloudInitDisabled, logger=self.logger)
+        @retry(exceptions=CloudInitDisabled, tries=120, delay=1, logger=self.logger)
         def _cloud_init_wait() -> None:
             process = self.execute(
                 name, ["timeout", "2000", "cloud-init", "status", "--wait", "--long"]
@@ -283,7 +283,20 @@ class CLILXD:
         )
 
     def remove_instance_device(self, instance_name: str, device_name: str) -> None:
-        self._run(["lxc", "config", "device", "remove", instance_name, device_name])
+        """Remove a device from an instance."""
+
+        @retry(
+            exceptions=subprocess.CalledProcessError,
+            tries=5,
+            delay=0.5,
+            logger=self.logger,
+        )
+        def _remove_device() -> subprocess.CompletedProcess[str]:
+            return self._run(
+                ["lxc", "config", "device", "remove", instance_name, device_name]
+            )
+
+        _remove_device()
 
     def list_instances(self) -> dict[str, dict[str, Any]]:
         result = self._run(["lxc", "list", "-f", "json"])
diff --git a/systemtests/utils.py b/systemtests/utils.py
index 37b2906..bc389ce 100644
--- a/systemtests/utils.py
+++ b/systemtests/utils.py
@@ -174,7 +174,7 @@ def wait_for_machine(
     debug_outputs.extend(
         f"- {event['type']}"
         + (f": {event['description']}" if event["description"] else "")
-        for event in events["events"]
+        for event in reversed(events["events"])
     )
     raise UnexpectedMachineStatus(
         machine["hostname"], status, retry_info.elapsed, debug_outputs