Merge ~alexmurray/ubuntu-security-tools:umt-adt-better-url-handling into ubuntu-security-tools:master

[Alex Murray]

Make the use of umt adt a bit easier as recommended by sbeattie.

[Alex Murray (alexmurray)]

Rebased this ontop of a recent change by sbeattie in the same vein, plus a minor change to make the code more pythonic as well... any reviews would be great :)

[Steve Beattie (sbeattie)]

LGTM, and limited testing worked fine.

Thanks!

review: Approve

diff --git a/build-tools/umt b/build-tools/umt
index 4ae47e4..40b5250 100755
--- a/build-tools/umt
+++ b/build-tools/umt
@@ -1913,7 +1913,7 @@ def check_adt_results(excuses, package, version, html=False, only_regressions=Fa
             else:
                 success(result_msg)
     else:
-        err("No excuses data found for %s %s" % (package, version))
+        raise ValueError("No excuses data found for %s %s" % (package, version))
 
 
 # Usage, e.g: umt adt --only-regressions --html - shows only those had
@@ -1943,50 +1943,57 @@ def cmd_adt():
     parser.add_argument("--html", default=False, action='store_true',
                         help="saves a html output to ../adt and open it in a browser")
     parser.add_argument("--url", default=None,
-                        help="Specify the URL to the update_excuses.yaml to use. %%s can be used as a placeholder for the release name.")
+                        help="Specify the URL to the update_excuses.yaml to use or 'security-proposed' / 'ubuntu' as shorthard for the security-proposed PPA and the ubuntu archive respectively. %%s can be used as a placeholder for the release name. e.g. --url 'http://security-britney.internal/current/security_%%s_excuses.yaml'.")
 
     (opt, args) = parser.parse_known_args()
 
     # validate we know what PPA is
     if release and release in valid_releases:
+        urls = []
         if opt.url is None:
             # pick the first upload file
             upload_files = glob.glob('./../source/*.upload')
             if len(upload_files) == 0:
-                err("No upload file found. Please upload first to generate adt results or specify --url manually.")
-                sys.exit(1)
-            upload = parse_upload_file(upload_files[0])
-            ppa = list(upload['upload'].keys())[0]
-            if ppa not in adt_urls:
-                err("unable to map PPA %s to a known location for autopkgtest results." % ppa)
-                err("please specify the update_excuses URL manually via the --url argument.")
-                sys.exit(1)
-            excuses_url = adt_urls[ppa] % release
+                warn("no upload file found.")
+            else:
+                upload = parse_upload_file(upload_files[0])
+                ppa = list(upload['upload'].keys())[0]
+                if ppa not in adt_urls:
+                    warn("unable to map upload destination '%s' to a known location for autopkgtest results." % ppa)
+                else:
+                    urls = [adt_urls[ppa] % release]
         else:
             if opt.url in adt_urls:
-                excuses_url = adt_urls[opt.url] % release
-            elif "%s" in opt.url:
-                excuses_url = opt.url % release
+                urls = [adt_urls[opt.url] % release]
             else:
-                excuses_url = opt.url
-        prepare_dir(adt_dest, True)
-
-        print("Getting {0} ...".format(excuses_url))
-        req = requests.get(excuses_url)
-        if not req.status_code == 200:
-            err("failed to get the excuses yaml page: %s" % req.url)
-            req.raise_for_status()
-            sys.exit(1)
-
-        # handle decoding zstd compressed excuses
-        if excuses_url.endswith('xz'):
-            excuses_yaml = subprocess.check_output(["/usr/bin/xz", "-d", "-"],
-                                                   input=req.content)
-        else:
-            excuses_yaml = req.content.decode()
-        excuses = yaml.safe_load(excuses_yaml)
-        check_adt_results(excuses, details["package"], details["version"],
-                          opt.html, opt.only_regressions, opt.verbose)
+                urls = [opt.url % release]
+        # default to looking in all known locations
+        if len(urls) == 0:
+            warn("trying all known excuses...")
+            urls = map(lambda s: s % release, adt_urls.values())
+        for url in urls:
+            prepare_dir(adt_dest, True)
+
+            print("Getting {0} ...".format(url))
+            req = requests.get(url)
+            if not req.status_code == 200:
+                err("failed to get the excuses yaml page: %s" % req.url)
+                # try next URL in the list
+                continue
+
+            # handle decoding zstd compressed excuses
+            if url.endswith('xz'):
+                excuses_yaml = subprocess.check_output(["/usr/bin/xz", "-d", "-"],
+                                                       input=req.content)
+            else:
+                excuses_yaml = req.content.decode()
+            excuses = yaml.safe_load(excuses_yaml)
+            try:
+                check_adt_results(excuses, details["package"], details["version"],
+                                  opt.html, opt.only_regressions, opt.verbose)
+            except ValueError as e:
+                err(str(e))
+                continue
 
     else:
        err("Not a valid release")