Merge autopkgtest-cloud:better-cleanup into autopkgtest-cloud:master

[Brian Murray]

When running cleanup-instances I ran into a python traceback:

Traceback (most recent call last):
  File "autopkgtest-cloud/tools/cleanup-instances", line 136, in <module>
    instance.delete()
  File "/usr/lib/python3/dist-packages/novaclient/v2/servers.py", line 65, in delete
    return self.manager.delete(self)
  File "/usr/lib/python3/dist-packages/novaclient/v2/servers.py", line 1538, in delete
    return self._delete("/servers/%s" % base.getid(server))
  File "/usr/lib/python3/dist-packages/novaclient/base.py", line 375, in _delete
    resp, body = self.api.client.delete(url)
  File "/usr/lib/python3/dist-packages/keystoneauth1/adapter.py", line 401, in delete
    return self.request(url, 'DELETE', **kwargs)
  File "/usr/lib/python3/dist-packages/novaclient/client.py", line 78, in request
    raise exceptions.from_response(resp, body, url, method)
novaclient.exceptions.NotFound: Instance abe6559c-6eda-4f33-b3cc-0b8eb5925df7 could not be found. (HTTP 404) (Request-ID: req-941e7d05-89af-4706-bfc8-2b544d4398ee)

That instance appears in `nova list` but has a "Task state" of scheduling and a "Power state" of NOSTATE. I think the script should continue cleaning up the instances rather than crashing when it encounters an error.

Additionally, I've reorganized the order of operations so that we don't write to influx until after the instance has actually been deleted.

I also noticed that we don't log anything when deleting instances without associated autopkgtests so fixed that too.

[Paride Legovini (paride)]

An inline comment/question, but this basically LGTM.

review: Needs Information

diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/cleanup-instances b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/cleanup-instances
index 96c574e..f2910ad 100755
--- a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/cleanup-instances
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/cleanup-instances
@@ -8,7 +8,7 @@ import subprocess
 import time
 from urllib.error import HTTPError
 
-import novaclient.client
+import novaclient.client, novaclient.exceptions
 from influxdb import InfluxDBClient
 from keystoneauth1 import session
 from keystoneauth1.identity import v2, v3
@@ -121,19 +121,23 @@ for instance in nova.servers.list():
             instance.name, instance.id, (float(age) / 3600)
         )
         logging.warning("{}, deleting".format(message))
-        measurements.append(
-            {
-                "measurement": "autopkgtest_delete_event",
-                "fields": {"deleted": True},
-                "tags": {
-                    "deletion_reason": "instance_too_old",
-                    "instance": INFLUXDB_CONTEXT,
-                    "message": message,
-                    "region": region,
-                },
-            }
-        )
-        instance.delete()
+        try:
+            instance.delete()
+            measurements.append(
+                {
+                    "measurement": "autopkgtest_delete_event",
+                    "fields": {"deleted": True},
+                    "tags": {
+                        "deletion_reason": "instance_too_old",
+                        "instance": INFLUXDB_CONTEXT,
+                        "message": message,
+                        "region": region,
+                    },
+                }
+            )
+        except novaclient.exceptions.NotFound:
+            logging.warning("Couldn't delete instance: not found")
+            pass
 
     # check matching adt-run process for instance name
     try:
@@ -152,19 +156,24 @@ for instance in nova.servers.list():
             message = "instance {} ({}) has no associated autopkgtest".format(
                 instance.name, instance.id
             )
-            measurements.append(
-                {
-                    "measurement": "autopkgtest_delete_event",
-                    "fields": {"deleted": True},
-                    "tags": {
-                        "deletion_reason": "instance_no_autopkgtest",
-                        "instance": INFLUXDB_CONTEXT,
-                        "message": message,
-                        "region": region,
-                    },
-                }
-            )
-            instance.delete()
+            logging.warning("{}, deleting".format(message))
+            try:
+                instance.delete()
+                measurements.append(
+                    {
+                        "measurement": "autopkgtest_delete_event",
+                        "fields": {"deleted": True},
+                        "tags": {
+                            "deletion_reason": "instance_no_autopkgtest",
+                            "instance": INFLUXDB_CONTEXT,
+                            "message": message,
+                            "region": region,
+                        },
+                    }
+                )
+            except novaclient.exceptions.NotFound:
+                logging.warning("Couldn't delete instance: not found")
+                pass
     except IndexError:
         logging.warning("instance %s has invalid name" % instance.name)