Merge ~andersson123/autopkgtest-cloud:ugj-check-running into autopkgtest-cloud:master

[Tim Andersson]

[Skia (hyask)]

review: Approve

diff --git a/charms/focal/autopkgtest-web/webcontrol/update-github-jobs b/charms/focal/autopkgtest-web/webcontrol/update-github-jobs
index c0f10fb..9aeb8e7 100755
--- a/charms/focal/autopkgtest-web/webcontrol/update-github-jobs
+++ b/charms/focal/autopkgtest-web/webcontrol/update-github-jobs
@@ -6,6 +6,7 @@ import io
 import json
 import logging
 import os
+import pathlib
 import sys
 import tarfile
 from datetime import datetime, timedelta
@@ -14,6 +15,7 @@ import swiftclient
 from request.submit import Submit
 
 PENDING_DIR = "/run/autopkgtest_webcontrol/github-pending"
+RUNNING_CACHE = "/run/amqp-status-collector/running.json"
 SWIFT_CREDS_FILE = "/home/ubuntu/public-swift-creds"
 MAX_DAY_DIFF = 30
 
@@ -102,6 +104,37 @@ def finish_job(jobfile, params, code, log_url):
     logging.debug("job %s finished!" % jobfile)
 
 
+def is_job_running(params):
+    job_match_me = (
+        params["arch"],
+        params["release"],
+        params["build-git"],
+        params["ppas"],
+        params["env"],
+    )
+
+    running_file = pathlib.Path(RUNNING_CACHE)
+    running_json = json.loads(running_file.read_text())
+    packages = running_json.keys()
+    if params["package"] not in packages:
+        return False
+    running_jobs = running_json[params["package"]]
+    for _, vals in running_jobs.items():
+        for release, vars in vals.items():
+            for arch, tests in vars.items():
+                test_params = tests[0]
+                this_test = (
+                    arch,
+                    release,
+                    test_params["build-git"],
+                    test_params["ppas"],
+                    test_params["env"],
+                )
+                if this_test == job_match_me:
+                    return True
+    return False
+
+
 def process_job(jobfile, swift_conn, ext_url):
     try:
         with open(jobfile) as f:
@@ -110,6 +143,10 @@ def process_job(jobfile, swift_conn, ext_url):
     except json.decoder.JSONDecodeError as e:
         logging.error("couldn't read %s, skipping: %s", jobfile, e)
         return
+    # if the job is currently running, it indicates a re-trigger has occurred!
+    if is_job_running(params):
+        logging.debug("job %s is currently running, skipping.")
+        return
 
     logging.debug(
         "\n\n--------------------\nprocessing job %s:\n   %s",
@@ -128,6 +165,7 @@ def process_job(jobfile, swift_conn, ext_url):
 
     _, object_list = swift_conn.get_container(container, full_listing=True)
 
+    result_for_finishing_job = {}
     for obj in object_list:
         if "result.tar" in obj["name"]:
             last_modified = obj["last_modified"].split(".")[0]
@@ -145,8 +183,30 @@ def process_job(jobfile, swift_conn, ext_url):
                 log_url = "/".join([ext_url, container, obj["name"]]).replace(
                     "result.tar", "log.gz"
                 )
-                finish_job(jobfile, params, code, log_url)
-                return
+                if not result_for_finishing_job:
+                    result_for_finishing_job = {
+                        "jobfile": jobfile,
+                        "params": params,
+                        "code": code,
+                        "log_url": log_url,
+                        "last_modified": obj_time_fmt.timestamp(),
+                    }
+                else:
+                    # this result is more recent, indicating a manual re-trigger
+                    if (
+                        obj_time_fmt.timestamp()
+                        > result_for_finishing_job["last_modified"]
+                    ):
+                        result_for_finishing_job = {
+                            "jobfile": jobfile,
+                            "params": params,
+                            "code": code,
+                            "log_url": log_url,
+                            "last_modified": obj_time_fmt.timestamp(),
+                        }
+    if result_for_finishing_job:
+        del result_for_finishing_job["last_modified"]
+        finish_job(**result_for_finishing_job)
 
 
 if __name__ == "__main__":