Merge ~alexmurray/ubuntu-security-tools:testflinger-support into ubuntu-security-tools:master

[Alex Murray]

[Seth Arnold (seth-arnold)]

This looks very useful, thanks!

[Alex Murray (alexmurray)]

Thanks for the review Seth, I've tried to address your comments in subsequent commits. `runcmdopt()` would respect opt.dry_run and opt.debug but I realise this was perhaps a bit too non-obvious, so instead I have refactored this function out and instead enhanced the existing `runcmd()` to allow this behaviour. Let me know what you think.

[Seth Arnold (seth-arnold)]

Thanks, I'm liking the look of this. One last thought, if the testflinger snap executable isn't in the PATH when this is run, how much mess does it make? Is it enough to justify a "search for the executable" check very early in the function?

Thanks

[Alex Murray (alexmurray)]

Ah good point - I've added a check for testflinger in PATH plus a few other clean ups.. I think this is about ready now :)

[Seth Arnold (seth-arnold)]

There's an rm that feels a bit out of place to me, but otherwise looks good. Thanks

review: Approve

[Seth Arnold (seth-arnold)]

Hmm seems my previous comment didn't survive, try again.

[Alex Murray (alexmurray)]

Thanks Seth, this is now merged.

diff --git a/build-tools/umt b/build-tools/umt
index 927009b..9a3febc 100755
--- a/build-tools/umt
+++ b/build-tools/umt
@@ -23,6 +23,9 @@ import re, optparse, subprocess, tempfile, glob, shutil, collections
 import resource
 import requests
 import time
+import json
+import yaml
+import threading
 from collections import namedtuple
 
 BinaryPackages = collections.namedtuple('BinaryPackages', 'binaries pkg_versions')
@@ -41,6 +44,7 @@ previous_dest = '../previous'
 coverity_dest = '../coverity'
 qrt_dest = '../qrt'
 autopkgtest_dest = '../autopkgtest'
+testflinger_dest = '../testflinger'
 
 # Per-package overrides for the sbuild resolver
 # <sbuild resolver> = sbuild_dep_resolver_overrides[<srcpkg>][<ubuntu release>]
@@ -1723,19 +1727,6 @@ def cmd_qrt():
                 warn("Please ensure you consult these")
 
 
-def runcmdopt(cmd, opt, okrc=[0]):
-    output = ""
-    if opt.debug:
-        print("[" + " ".join(cmd) + "]")
-    if not opt.dry_run:
-        (rc, output) = runcmd(cmd)
-        if rc not in okrc:
-            raise Exception("Failed to execute command: " + " ".join(cmd) + " rc [" + str(rc) + "] not in " + str(okrc) + ": " + output)
-        if opt.debug:
-            print(output)
-    return output
-
-
 def run_qrt_tests(opt, args, details):
     okrc = [0]
     if opt.ignore_failures:
@@ -1793,64 +1784,68 @@ def run_qrt_tests(opt, args, details):
             if not opt.no_snapshot:
                 cmd.append("-f")
             print("Ensuring uvt VM " + vm + " is stopped...")
-            runcmdopt(cmd, opt)
+            runcmd(cmd, debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
             print("Updating uvt VM " + vm + "...")
             cmd = [uvt, "update", "-f", vm]
             if opt.no_snapshot:
                 cmd.append("--nosnapshot")
-            runcmdopt(cmd, opt)
+            runcmd(cmd, debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
         print("Launching uvt VM " + vm + "...")
         # run headless since we don't use the GUI and wait for ssh
         # availability
-        runcmdopt([uvt, "start", "-v", "-w", vm], opt)
+        runcmd([uvt, "start", "-v", "-w", vm],
+               debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
         print("Packaging QRT test...")
-        runcmdopt([os.path.join(opt.qrt_path, "scripts", "make-test-tarball"),
-                        qrt_test], opt)
+        runcmd([os.path.join(opt.qrt_path, "scripts", "make-test-tarball"),
+                        qrt_test],
+               debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
         print("Deploying QRT test...")
-        runcmdopt(["/usr/bin/scp",
-                   os.path.join(tempfile.gettempdir(), "qrt-" + test_name + ".tar.gz"),
-                   user + "@" + vm + ":"], opt)
+        runcmd(["/usr/bin/scp",
+                os.path.join(tempfile.gettempdir(), "qrt-" + test_name + ".tar.gz"),
+                user + "@" + vm + ":"],
+               debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
         print("Extracting QRT test...")
-        runcmdopt(["/usr/bin/ssh", "-T", user + "@" + vm,
-                   "tar -xvf qrt-" + test_name + ".tar.gz" ],
-                  opt)
+        runcmd(["/usr/bin/ssh", "-T", user + "@" + vm,
+                "tar -xvf qrt-" + test_name + ".tar.gz" ],
+               debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
 
         # enable proposed if required
         if opt.enable_proposed:
             print("Enabling -proposed...")
             # find which mirror is being used
-            mirror = runcmdopt(["/usr/bin/ssh", "-T", user + "@" + vm,
-                                "grep '^deb .*" + details["release"] + " main' /etc/apt/sources.list |" +
-                                "awk '{print $2}'"],
-                               opt).strip()
+            (rc, mirror) = runcmd(["/usr/bin/ssh", "-T", user + "@" + vm,
+                                   "grep '^deb .*" + details["release"] + " main' /etc/apt/sources.list |" +
+                                   "awk '{print $2}'"],
+                                  debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
+            mirror = mirror.strip()
             if mirror == "":
                 err("Failed to determine primary mirror used by VM... using archive.ubuntu.com")
                 mirror = "http://archive.ubuntu.com/ubuntu/"
-            runcmdopt(["/usr/bin/ssh", "-T", user + "@" + vm,
-                       "cp /etc/apt/sources.list /tmp/sources.list"],
-                      opt)
-            runcmdopt(["/usr/bin/ssh", "-T", user + "@" + vm,
-                       "echo 'deb " + mirror + " focal-proposed main restricted universe multiverse' >> /tmp/sources.list"],
-                      opt)
-            runcmdopt(["/usr/bin/ssh", "-T", user + "@" + vm,
-                       "echo ubuntu | sudo -S mv /tmp/sources.list /etc/apt/sources.list"],
-                      opt)
-            runcmdopt(["/usr/bin/ssh", "-T", user + "@" + vm,
-                       "echo ubuntu | sudo -S apt update"],
-                      opt)
+            runcmd(["/usr/bin/ssh", "-T", user + "@" + vm,
+                    "cp /etc/apt/sources.list /tmp/sources.list"],
+                   debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
+            runcmd(["/usr/bin/ssh", "-T", user + "@" + vm,
+                    "echo 'deb " + mirror + " focal-proposed main restricted universe multiverse' >> /tmp/sources.list"],
+                   debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
+            runcmd(["/usr/bin/ssh", "-T", user + "@" + vm,
+                    "echo ubuntu | sudo -S mv /tmp/sources.list /etc/apt/sources.list"],
+                   debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
+            runcmd(["/usr/bin/ssh", "-T", user + "@" + vm,
+                    "echo ubuntu | sudo -S apt update"],
+                   debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
             if not opt.no_update:
                 print("Updating from -proposed...")
                 # update again from -proposed so we use this as a baseline
-                runcmdopt(["/usr/bin/ssh", "-T", user + "@" + vm,
-                           "echo ubuntu | sudo -S apt -y upgrade"],
-                          opt)
+                runcmd(["/usr/bin/ssh", "-T", user + "@" + vm,
+                        "echo ubuntu | sudo -S apt -y upgrade"],
+                       debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
 
         print("Installing packages for QRT test...")
         # always install packages via sudo
-        runcmdopt(["/usr/bin/ssh", "-T", user + "@" + vm,
-                   "cd ./qrt-" + test_name + "; " +
-                   "echo ubuntu | sudo -S ./install-packages ./" + test_name + ".py"],
-                  opt)
+        runcmd(["/usr/bin/ssh", "-T", user + "@" + vm,
+                "cd ./qrt-" + test_name + "; " +
+                "echo ubuntu | sudo -S ./install-packages ./" + test_name + ".py"],
+               debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
 
         # get the list of binary packages
         binaries = [deb.split("/")[-1].split("_")[0] for deb in glob.glob('./../binary/*.deb')]
@@ -1858,9 +1853,11 @@ def run_qrt_tests(opt, args, details):
             err('No binaries exist in ../binary - please build the package first.')
         if not opt.skip_baseline:
             print("Install prior binaries with version %s..." % details["version_prior"])
-            report = runcmdopt(["/usr/bin/ssh", "-T", "root@" + vm,
-                "apt install --yes --allow-downgrades", " ".join([binary + "=" +
-                    details["version_prior"] for binary in binaries])], opt)
+            (rc, report) = runcmd(["/usr/bin/ssh", "-T", "root@" + vm,
+                                   "apt install --yes --allow-downgrades",
+                                   " ".join([binary + "=" + details["version_prior"]
+                                             for binary in binaries])],
+                                  debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
             if opt.debug:
                 print(report)
 
@@ -1868,10 +1865,10 @@ def run_qrt_tests(opt, args, details):
             okrc = [0]
             if opt.ignore_failures:
                 okrc.append(1)
-            report = runcmdopt(["/usr/bin/ssh", "-T", user + "@" + vm,
-                                "cd ./qrt-" + test_name + ";" +
-                                sudo + " ./" + test_name + ".py -v " + " ".join(args)],
-                               opt, okrc)
+            (rc, report) = runcmd(["/usr/bin/ssh", "-T", user + "@" + vm,
+                                   "cd ./qrt-" + test_name + ";" +
+                                   sudo + " ./" + test_name + ".py -v " + " ".join(args)],
+                                  debug=opt.debug, dry_run=opt.dry_run, okrc=okrc)
             if not opt.dry_run:
                 # save the report so we can compare it later
                 path = os.path.join(qrt_dest, "qrt-" + test_name + '-' + opt.arch + '-orig.txt')
@@ -1892,28 +1889,32 @@ def run_qrt_tests(opt, args, details):
                 copy_to_repo(opt, details, quiet=not opt.debug, source_dest=source_dest, binary_dest=binary_dest)
 
         print("Enabling repo for uvt VM...")
-        runcmdopt([uvt, "repo", "-e", vm], opt)
+        runcmd([uvt, "repo", "-e", vm],
+               debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
 
         if not opt.no_update:
             print("Upgrading packages in uvt VM...")
-            report = runcmdopt(["/usr/bin/ssh", "-T", "root@" + vm,
-                                "apt-get dist-upgrade -y"], opt)
+            (rc, report) = runcmd(["/usr/bin/ssh", "-T", "root@" + vm,
+                                   "apt-get dist-upgrade -y"],
+                                  debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
             path = os.path.join(qrt_dest, "apt-dist-upgrade-log.txt")
             with open(path,"w+") as handle:
                 handle.write(report)
                 handle.flush()
 
         print("Install target binaries with version %s..." % details["version"])
-        report = runcmdopt(["/usr/bin/ssh", "-T", "root@" + vm,
-            "apt install --yes --allow-downgrades", " ".join([binary + "=" +
-                details["version"] for binary in binaries])], opt)
+        (rc, report) = runcmd(["/usr/bin/ssh", "-T", "root@" + vm,
+                               "apt install --yes --allow-downgrades",
+                               " ".join([binary + "=" + details["version"]
+                                         for binary in binaries])],
+                              debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
         if opt.debug:
             print(report)
         print("Re-executing QRT test...")
-        report = runcmdopt(["/usr/bin/ssh", "-T", user + "@" + vm,
-                            "cd ./qrt-" + test_name + ";" +
-                            sudo + " ./" + test_name + ".py -v " + " ".join(args)], opt,
-                           okrc)
+        (rc, report) = runcmd(["/usr/bin/ssh", "-T", user + "@" + vm,
+                               "cd ./qrt-" + test_name + ";" +
+                               sudo + " ./" + test_name + ".py -v " + " ".join(args)],
+                              debug=opt.debug, dry_run=opt.dry_run, okrc=okrc)
         if not opt.dry_run:
             # save the report so we can compare it later
             path = os.path.join(qrt_dest, "qrt-" + test_name + '-' + opt.arch +'.txt')
@@ -1925,12 +1926,12 @@ def run_qrt_tests(opt, args, details):
         if (os.path.exists(os.path.join(qrt_dest, "qrt-" + test_name + '-' + opt.arch + '-orig.txt')) and
             os.path.exists(os.path.join(qrt_dest, "qrt-" + test_name + '-' + opt.arch + '.txt'))):
             print("Generating diff of QRT tests output...")
-            report = runcmdopt(["/usr/bin/diff", "-u",
-                                os.path.join(qrt_dest, "qrt-" + test_name + '-' + opt.arch + '-orig.txt'),
-                                os.path.join(qrt_dest, "qrt-" + test_name + '-' + opt.arch + '.txt')],
-                               # diff returns 1 if different and 0 if same - so
-                               # both are valid ok return codes
-                               opt, [0, 1])
+            (rc, report) = runcmd(["/usr/bin/diff", "-u",
+                                   os.path.join(qrt_dest, "qrt-" + test_name + '-' + opt.arch + '-orig.txt'),
+                                   os.path.join(qrt_dest, "qrt-" + test_name + '-' + opt.arch + '.txt')],
+                                   # diff returns 1 if different and 0 if same - so
+                                   # both are valid ok return codes
+                                  debug=opt.debug, dry_run=opt.dry_run, okrc=[0, 1])
             # save the diff
             path = os.path.join(qrt_dest, "qrt-" + test_name + '-' + opt.arch + '.diff')
             with open(path,"w+") as handle:
@@ -1939,24 +1940,28 @@ def run_qrt_tests(opt, args, details):
             print("QRT test run diff in " + path)
 
         print("Testing reboot of uvt VM...")
-        report = runcmdopt(["/usr/bin/ssh", "-T", "root@" + vm,
-                            "shutdown -r now"], opt)
+        (rc, report) = runcmd(["/usr/bin/ssh", "-T", "root@" + vm,
+                               "shutdown -r now & exit"],
+                              debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
         print("Waiting for 20 seconds for uvt VM to finish rebooting...")
         time.sleep(20)
-        report += runcmdopt(["/usr/bin/ssh", "-T", vm,
-                             "uptime"], opt)
+        (rc, report2) = runcmd(["/usr/bin/ssh", "-T", vm,
+                                "uptime"],
+                               debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
+        report += report2
 
     except Exception as e:
         err(str(e))
         if opt.debug:
             print("Dropping into a remote shell to debug...")
             # drop the user into a shell to debug the failure...
-            rc, report = runcmd(["/usr/bin/ssh", "-t", user + "@" + vm],
-                                stderr=sys.stderr,
-                                stdout=sys.stdout,
-                                stdin=sys.stdin)
+            (rc, report) = runcmd(["/usr/bin/ssh", "-t", user + "@" + vm],
+                                  stderr=sys.stderr,
+                                  stdout=sys.stdout,
+                                  stdin=sys.stdin)
     print("Stopping uvt VM " + vm + "...")
-    runcmdopt([uvt, "stop", vm], opt)
+    runcmd([uvt, "stop", vm],
+           debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
 
 
 def cmd_autopkgtest():
@@ -2014,11 +2019,11 @@ def run_autopkgtest_tests(opt, args, details):
             print("")
             if answer.startswith("y"):
                 print("Creating autopkgtest qemu image '%s'" % image)
-                runcmdopt(["/usr/bin/autopkgtest-buildvm-ubuntu-cloud",
+                runcmd(["/usr/bin/autopkgtest-buildvm-ubuntu-cloud",
                            "-a", opt.arch,
                            "-r", details["release"],
                            "-o", os.path.dirname(image)],
-                          opt)
+                          debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
                 print("Created autopkgtest qemu image '%s'" % image)
         # autopkgtest requires the config file to be prefixed by @ to
         # designate it
@@ -2039,9 +2044,8 @@ def run_autopkgtest_tests(opt, args, details):
                    details["package"]]
             if opt.autopkgtest_config is not None:
                 cmd.append(opt.autopkgtest_config)
-            report = runcmdopt(cmd +
-                               ["--", "qemu", image] + args,
-                               opt, okrc)
+            (rc, report) = runcmd(cmd + ["--", "qemu", image] + args,
+                                  debug=opt.debug, dry_run=opt.dry_run, okrc=okrc)
             if not opt.dry_run:
                 # save the report so we can compare it later
                 path = os.path.join(autopkgtest_dest, "autopkgtest-test-" + details["package"] + "-" + prev_version + ".txt")
@@ -2061,9 +2065,8 @@ def run_autopkgtest_tests(opt, args, details):
                    changes]
             if opt.autopkgtest_config is not None:
                 cmd.append(opt.autopkgtest_config)
-            report = runcmdopt(cmd +
-                               ["--", "qemu", image] + args,
-                               opt, okrc)
+            (rc, report) = runcmd(cmd + ["--", "qemu", image] + args,
+                                  debug=opt.debug, dry_run=opt.dry_run, okrc=okrc)
             if not opt.dry_run:
                 # save the report so we can compare it later
                 path = os.path.join(autopkgtest_dest, "autopkgtest-test-" + details["package"] + "-" + version + ".txt")
@@ -2076,12 +2079,12 @@ def run_autopkgtest_tests(opt, args, details):
             os.path.exists(os.path.join(autopkgtest_dest, "autopkgtest-test-" + details["package"] + "-" + prev_version + ".txt")) and
             os.path.exists(os.path.join(autopkgtest_dest, "autopkgtest-test-" + details["package"] + "-" + version + ".txt"))):
             print("Generating diff of autopkgtest tests output...")
-            report = runcmdopt(["/usr/bin/diff", "-u",
-                                os.path.join(autopkgtest_dest, "autopkgtest-test-" + details["package"] + "-" + prev_version + ".txt"),
-                                os.path.join(autopkgtest_dest, "autopkgtest-test-" + details["package"] + "-" + version + ".txt")],
-                               # diff returns 1 if different and 0 if same - so
-                               # both are valid ok return codes
-                               opt, [0, 1])
+            (rc, report) = runcmd(["/usr/bin/diff", "-u",
+                                   os.path.join(autopkgtest_dest, "autopkgtest-test-" + details["package"] + "-" + prev_version + ".txt"),
+                                   os.path.join(autopkgtest_dest, "autopkgtest-test-" + details["package"] + "-" + version + ".txt")],
+                                  # diff returns 1 if different and 0 if same - so
+                                  # both are valid ok return codes
+                                  debug=opt.debug, dry_run=opt.dry_run, okrc=[0,1])
             if not opt.dry_run:
                 # save the diff
                 path = os.path.join(autopkgtest_dest, "autopkgtest-test-" + details["package"] + '-' + opt.arch + '.diff')
@@ -2092,6 +2095,332 @@ def run_autopkgtest_tests(opt, args, details):
     except Exception as e:
         err(str(e))
 
+# this is a list of known working testflinger queues from
+# https://docs.google.com/spreadsheets/d/11WNsLuwG6z5fcAJQWq8RoSD2mwkmL2zvJdLR-um-MiM
+# that can be provisioned via maas along with the images they support and
+# any other ancillary data - for now let's just stick to the stock images
+# but in the future we can look at adding oem images too
+testflinger_queues = {
+    "201302-12728": {"images": ["xenial", "bionic", "focal"]},
+    "201506-18555": {"images": ["xenial", "bionic", "focal"]},
+    "201507-18599": {"images": ["xenial", "bionic", "focal"]},
+    "201507-18697": {"images": ["xenial", "bionic", "focal"]},
+    "201606-22340": {"images": ["xenial"]},
+    "201606-22344": {"images": ["xenial", "bionic", "focal"]},
+    "201606-22346": {"images": ["xenial", "bionic", "focal"]},
+    "201606-22347": {"images": ["xenial"]},
+    "201606-22459": {"images": ["xenial", "bionic", "focal"]},
+    #"201606-22528": {}, - only xenial-oem for now
+    "201702-25401": {"images": ["bionic", "focal"]},
+    "201708-25694": {"images": ["bionic"]},
+    "201708-25695": {"images": ["bionic"]},
+    "201708-25696": {"images": ["bionic"]},
+    "201711-25989": {"images": ["bionic", "focal"]},
+    #"201802-26107": {}, - only xenial-oem for now
+    "201807-26342": {"images": ["bionic", "focal"]},
+    "201903-26932": {"images": ["focal"]},
+    #"201904-26941": {}, - only bionic-eom for now
+    "201904-26953": {"images": ["bionic", "focal"]},
+    "201906-27109": {"images": ["xenial", "bionic", "focal"]},
+    "201906-27131": {"images": ["focal"]},
+    "201907-27241": {"images": ["bionic", "focal"]},
+    "201911-27542": {"images": ["xenial", "bionic"]},
+    "201912-27608": {"images": ["xenial", "bionic"]},
+    "201912-27622": {"images": ["bionic", "focal"]},
+    "201912-27623": {"images": ["bionic", "focal"]},
+    "202001-27665": {"images": ["bionic", "focal"]},
+    "202001-27683": {"images": ["focal"]},
+    "202002-27717": {"images": ["focal"]},
+    "202002-27718": {"images": ["bionic", "focal"]},
+    "202004-27810": {"images": ["bionic", "focal"]},
+    "202004-27811": {"images": ["bionic", "focal"]},
+    "202004-27812": {"images": ["bionic", "focal"]},
+    "202007-28045": {"images": ["bionic", "focal"]},
+    # intel nuc with CPUID 0x000706a1 / 06-7a-01
+    "dawson-j": {"images": ["bionic", "focal"]},
+    # these are not currently provisionable by maas but we can drive them
+    # directly
+    "201406-15262": {"images": []}, # 06-3c-03
+    "201610-25145": {"images": []}, # 06-9e-09
+    "201702-25425": {"images": []}, # 06-9e-09
+    "201803-26173": {"images": []}, # 06-9e-0a
+    "201806-26288": {"images": []}, # 06-55-04
+    "201807-26296": {"images": []}, # 06-8e-0b
+    "201902-26852": {"images": []}, # 06-8e-0c
+    "201907-27239": {"images": []}, # 06-7e-05
+}
+
+def cmd_testflinger():
+    '''Run TESTFLINGER tests for the package in the current directory of unpacked source'''
+    parser = umt_optparse("usage: %prog testflinger [options]")
+    parser.add_option("-l", "--list-queues", dest="list_queues", action='store_true', default=False,
+                      help="List supported testflinger queues")
+    parser.add_option("-q", "--queues", dest="queues", default=','.join(testflinger_queues.keys()),
+                      help="Comma separated list of testflinger queues to use (default: all)")
+    parser.add_option("-n", "--dry-run",  dest="dry_run",  default=False, action='store_true',
+                      help="Don't actually execute tests, instead print what would be run")
+    parser.add_option("--debug", default=False, action='store_true',
+                      help="Report additional debug details")
+    parser.add_option("-f", "--force",     dest="force",      default=False, action='store_true',
+                      help="force deletion of ../testflinger before running")
+    parser.add_option("-F", "--ignore-failures", dest="ignore_failures", default=False, action='store_true',
+                      help="continue even if test failures encountered when running testflinger")
+    parser.add_option("-T", "--template", dest="template", default=None,
+                      help="A testflinger job template to use - this allows to specify extra commands etc")
+    parser.add_option("--repo", dest="repo", default=None,
+                      metavar="Anything suitable for `add-apt-repository REPO` (ie. 'ppa:ubuntu-security-proposed/ppa' or 'https://https://USER:[e-mail]/ubuntu-security/ppa/ubuntu' etc)",
+                      help="Add REPO to test machine to test new binaries from it rather than using yantok")
+    parser.add_option("-j", "--jump-host", dest="jump_host", default=os.getenv("USER") + "@yantok.canonical.com:/srv/enablement/www",
+                      help="A jump host to temporarily store debs to be loaded on test machine (default $[e-mail]:/srv/enablement/www)")
+    parser.add_option("-i", "--jump-host-internal-ip", dest="jump_host_ip", default="10.101.47.1",
+                      help="The internal IP of JUMP_HOST (default 10.101.47.1 for yanktok)")
+    (opt, args) = parser.parse_args()
+
+    validate_toplevel()
+    details = parse_package_details(skip_sanity=True)
+    # check we can find testflinger in PATH
+    if shutil.which("testflinger") is None:
+        err("Unable to find testflinger in PATH, install it with `sudo snap install testflinger-cli`")
+    else:
+        run_testflinger_tests(opt, args, details)
+
+
+def run_testflinger_tests(opt, args, details):
+    # choose only those queues which support this release image as well as
+    # the unprovisionalble ones
+    queues = [queue for queue in opt.queues.split(',')
+              if details["release"] in testflinger_queues[queue]["images"] or
+              len(testflinger_queues[queue]["images"]) == 0]
+    jobs = {}
+    template = ''
+    if len(queues) == 0:
+        err('No queues in %s support release (%s)' % (opt.queues, details["release"]))
+        return 1
+    if opt.list_queues:
+        print(queues)
+        return 0
+    if not opt.dry_run:
+        try:
+            os.mkdir(testflinger_dest)
+        except FileExistsError:
+            pass
+    try:
+        tempdir = None
+        # get the list of binary packages
+        local_binaries = glob.glob('./../binary/*.deb')
+        binaries = [deb.split("/")[-1] for deb in local_binaries]
+        apt_binaries = [deb.split("_")[0] + "=" + deb.split("_")[1] for deb in binaries]
+        if len(binaries) == 0:
+            err('No binaries exist in ../binary - please build the package first.')
+
+        if opt.repo is None:
+            # copy binaries to the jump host
+            user = opt.jump_host.split("@")[0]
+            server = opt.jump_host.split(":")[0]
+            path = opt.jump_host.split(":")[1]
+            # generate a temp path to store per-user
+            print("Copying binaries to jump host...")
+            (rc, tempdir) = runcmd(["ssh", server,
+                                    "mktemp", "-d", "-p", path, user + "XXXX"],
+                                   debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
+            tempdir = tempdir.strip()
+            if opt.dry_run:
+                # generate a fake tempdir so that we keep the code
+                # simpler if this is a dry-run
+                tempdir = os.path.join(path, user + "XXXX")
+            # ensure can be read from by other users, ie www
+            runcmd(["ssh", server, "chmod", "+rx", tempdir],
+                   debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
+            runcmd(["scp"] + local_binaries + [server + ":" + tempdir],
+                   debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
+            tempurl = "http://" + opt.jump_host_ip + tempdir.replace(path, "/")
+
+        if opt.template is not None:
+            with open(opt.template, 'r') as f:
+                template = f.read()
+        # generate test config
+        config = yaml.safe_load(template) or {"test_data": {"test_cmds": []}}
+        test_data = config["test_data"] if "test_data" in config else {"test_cmds": []}
+        test_cmds = test_data["test_cmds"]
+        # basic test commands to install the package and reboot and check
+        # the device is still functional
+        cmds = ["mkdir artifacts",
+                "echo Getting device details...",
+                "ssh $DEVICE_IP lsb_release -a > artifacts/lsb_release",
+                "ssh $DEVICE_IP uname -a > artifacts/uname",
+                "echo Getting initial dmesg and cpuinfo...",
+                "ssh $DEVICE_IP sudo dmesg > artifacts/dmesg.orig",
+                "ssh $DEVICE_IP sudo cat /proc/cpuinfo > artifacts/cpuinfo.orig"]
+        if not opt.ignore_failures:
+            # fail on any failing command
+            cmds = ["set -e"] + cmds
+        if opt.repo is None:
+            cmds.append("echo Manually installing binaries for %s..." % details["package"])
+            for binary in binaries:
+                # don't try too hard since it should be on the same network
+                # and will only slow down debugging when this fails...
+                cmds.append("wget %s/%s" % (tempurl, binary))
+            cmds.append("scp %s $DEVICE_IP:" % " ".join(binaries))
+            cmds.append("rm %s" % " ".join(binaries))
+            cmds.append("ssh $DEVICE_IP sudo dpkg -i %s" % " ".join(binaries))
+        else:
+            cmds.append("echo Adding apt repository %s..." % opt.repo)
+            cmds.append("ssh $DEVICE_IP sudo add-apt-repository --yes %s" % opt.repo)
+            cmds.append("echo Installing %s..." % details["package"])
+            cmds.append("ssh $DEVICE_IP sudo apt-get install %s" % " ".join(apt_binaries))
+        # we can't just reboot since this will close the ssh connection
+        # before ssh gets a chance to exit and the command will appear to
+        # fail - so instead schedule a reboot in 1 minute
+        cmds.append("echo Rebooting test device...")
+        cmds.append("ssh $DEVICE_IP sudo shutdown -r +1")
+        cmds.append("echo Waiting for device to reboot for 300 seconds...")
+        cmds.append("sleep 300")
+        cmds.append("echo Getting subsequent dmesg and cpuinfo...")
+        cmds.append("ssh $DEVICE_IP sudo dmesg > artifacts/dmesg.rebooted")
+        cmds.append("ssh $DEVICE_IP sudo cat /proc/cpuinfo > artifacts/cpuinfo.rebooted")
+        cmds.append("echo Getting diff of cpuinfo...")
+        cmds.append("diff -u artifacts/cpuinfo.orig artifacts/cpuinfo.rebooted > artifacts/cpuinfo.diff || true")
+        test_cmds = cmds + test_cmds
+        # concatenate commands so they get run in a single session so
+        # things like set -e etc work as expected
+        test_data["test_cmds"] = "\n".join(test_cmds)
+        config["test_data"] = test_data
+        for queue in queues:
+            # set provision_data per queue since some may not support
+            # provisioning (ie. are not managed by maas)
+            if details["release"] in testflinger_queues[queue]["images"]:
+                config["provision_data"] = {"distro": details["release"]}
+            else:
+                try:
+                    del config["provision_data"]
+                except KeyError:
+                    pass
+            config["job_queue"] = queue
+            # place the config somewhere the testflinger snap can find it
+            job_yaml = os.path.expanduser("~/snap/testflinger-cli/common/%s.yaml" % queue)
+            with open(job_yaml, "w+") as fp:
+                yaml.safe_dump(config, fp, default_style="|", default_flow_style=False)
+                fp.close()
+            print("Generated job config for queue %s in %s" % (queue, job_yaml))
+            if not opt.dry_run:
+                prepare_dir(os.path.join(testflinger_dest, queue), opt.force)
+            (rc, job) = runcmd(["testflinger", "submit", "-q", job_yaml],
+                               debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
+            job = job.strip()
+            if opt.dry_run:
+                job = queue
+            jobs[queue] = {"id": job, "status": "unknown", "test_status": None}
+
+        # setup to handle SIGHUP so we can print status
+        exit = threading.Event()
+        def interrupted(signo, _frame):
+            print("Interrupted by signal %d..." % signo)
+            remaining = [queue for queue in jobs if
+                         jobs[queue]["status"] != "complete" and
+                         jobs[queue]["status"] != "cancelled"]
+            if signo == signal.SIGHUP:
+                print("Waiting for %d jobs to complete out of %d jobs: %s" % (len(remaining), len(jobs), jobs))
+            else:
+                print("Cancelling remaining jobs: " + " ".join(remaining))
+                for queue in remaining:
+                    runcmd(["testflinger", "cancel", jobs[queue]["id"]],
+                           debug=opt.debug, dry_run=opt.dry_run)
+                exit.set()
+        for sig in [signal.SIGTERM, signal.SIGHUP, signal.SIGINT]:
+            signal.signal(sig, interrupted)
+        print("Waiting for %d jobs to complete... (send me PID %d a SIGHUP to get status)..." %
+              (len(jobs), os.getpid()))
+        while not exit.is_set():
+            for queue in [queue for queue in jobs if
+                          jobs[queue]["status"] != "complete" and
+                          jobs[queue]["status"] != "cancelled"]:
+                job = jobs[queue]["id"]
+                (rc, status) = runcmd(["testflinger", "status", job],
+                                      debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
+                status = status.strip()
+                jobs[queue]["status"] = status
+                if opt.dry_run:
+                    status = "complete"
+                if status == "complete" or status == "cancelled":
+                    (rc, res) = runcmd(["testflinger", "results", job],
+                                       debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
+                    if opt.dry_run:
+                        res = '{"test_output": "", "test_status": 0}'
+                    results = json.loads(res)
+                    results_json = os.path.join(testflinger_dest, queue, "results.json")
+                    print("Results for %s available in %s" % (queue, results_json))
+                    # get artifacts - this may fail
+                    artifacts_tgz = os.path.join(testflinger_dest, queue, "artifacts.tgz")
+                    (rc, output) = runcmd(["testflinger", "artifacts", job,
+                                           "--filename", artifacts_tgz],
+                                          debug=opt.debug, dry_run=opt.dry_run)
+                    if rc == 0:
+                        artifacts_dir = os.path.dirname(artifacts_tgz)
+                        # helpfully extract artifacts
+                        (rc, output) = runcmd(["tar", "-C", artifacts_dir,
+                                               "-x", "-f", artifacts_tgz])
+                        if rc == 0:
+                            os.unlink(artifacts_tgz)
+                            # artifacts is self-contained in it's own dir
+                            print("Artifacts for %s available in %s" % (queue, os.path.join(artifacts_dir, "artifacts")))
+                        else:
+                            err("Failed to extract artifacts for %s from %s: %s" % (queue, artifacts_tgz, output))
+                            print("Artifacts for %s available in %s" % (queue, artifacts_tgz))
+                    else:
+                        err("Failed to download artifacts for %s: %s" % (queue, output))
+                    # get test status / output
+                    try:
+                        status = results["test_status"]
+                        jobs[queue]["test_status"] = "PASS" if status == 0 else "FAIL"
+                        if status != 0 and not opt.ignore_failures:
+                            err("Test failed for %s [%d]: see output for more details" % (queue, status))
+                    except KeyError:
+                        err("No test_status found for %s" % queue)
+                    try:
+                        output = results["test_output"]
+                    except KeyError:
+                        output = "No test_output found - see results.json for more details"
+                    output_txt = os.path.join(testflinger_dest, queue, "output.txt")
+                    if not opt.dry_run:
+                        with open(results_json, "w+") as fp:
+                            json.dump(results, fp)
+                            fp.close()
+                        with open(output_txt, "w+") as fp:
+                            fp.write(output)
+                            fp.close()
+                    print("Test output for %s available in %s" % (queue, output_txt))
+            # bail if nothing left to do
+            if len([queue for queue in jobs if
+                    jobs[queue]["status"] != "complete" and
+                    jobs[queue]["status"] != "cancelled"]) == 0:
+                exit.set()
+            else:
+                exit.wait(60)
+
+    except Exception as e:
+        err(str(e))
+        print("Cancelling remaining jobs: " + " ".join(jobs))
+        for queue in jobs:
+            runcmd(["testflinger", "cancel", jobs[queue]["id"]],
+                   debug=opt.debug, dry_run=opt.dry_run)
+    finally:
+        # ensure we cleanup after ourselves on the jump host
+        if opt.repo is None and tempdir is not None:
+            print("Cleaning up binaries from jump host...")
+            runcmd(["ssh", server, "rm", "-rf", tempdir],
+                   debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
+        print("### Results: ###")
+        # print summary
+        for queue in jobs:
+            job = jobs[queue]["id"]
+            (rc, status) = runcmd(["testflinger", "status", job],
+                                  debug=opt.debug, dry_run=opt.dry_run, okrc=[0])
+            status = status.strip()
+            jobs[queue]["status"] = status
+            print("%s: %s [%s]" % (queue, jobs[queue]["status"],
+                                   jobs[queue]["test_status"]))
+
+
 #
 # Misc functions
 #
@@ -2663,19 +2992,28 @@ def recursive_rm(dirPath):
             recursive_rm(path)
     os.rmdir(dirPath)
 
-def runcmd(command, input = None, stderr = subprocess.STDOUT, stdout = subprocess.PIPE, stdin = None, shell = False):
+
+def runcmd(command, input = None, stderr = subprocess.STDOUT, stdout = subprocess.PIPE, stdin = None, shell = False,
+           debug = False, dry_run = False, okrc=None):
     '''Try to execute given command (array) and return its stdout, or return
     a textual error if it failed.'''
+    rc = 0
+    out = ''
+    if debug:
+        print("[" + " ".join(command) + "]")
 
-    try:
-        sp = subprocess.Popen(command, stdin=stdin, stdout=stdout, stderr=stderr, close_fds=True, shell=shell, preexec_fn=subprocess_setup)
-    except OSError as e:
-        return [127, str(e)]
-
-    out = sp.communicate(input)[0]
-    if out is not None:
-        out = out.decode()
-    return [sp.returncode,out]
+    if not dry_run:
+        try:
+            sp = subprocess.Popen(command, stdin=stdin, stdout=stdout, stderr=stderr, close_fds=True, shell=shell, preexec_fn=subprocess_setup)
+        except OSError as e:
+            return [127, str(e)]
+        out = sp.communicate(input)[0]
+        if out is not None:
+            out = out.decode()
+        rc = sp.returncode
+    if okrc is not None and rc not in okrc:
+        raise Exception("Failed to execute command: '" + " ".join(command) + "' rc [" + str(rc) + "] not in " + str(okrc) + ": " + out)
+    return [rc, out]
 
 def mkstemp_fill(contents,suffix='',prefix='umt-',dir=None):
     '''As tempfile.mkstemp does, return a (file, name) pair, but with
@@ -3784,8 +4122,9 @@ compare-bin  Compare the binary debs against the prior version's binaries
 sign         Sign the packages
 check        Do a check-source-package
 repo         Copy all built packages into local repository
-qrt          Run qa-regression-test for a built package
+qrt          Run qa-regression-test for a built package in a uvt VM
 autopkgtest  Run autopkgtests for a built package
+testflinger  Test package via testflinger on real hardware in the cert lab
 upload       Uploads with dput
 
 OPTIONS:
@@ -3852,6 +4191,7 @@ commands = {
     'upload'      : cmd_upload,
     'qrt'         : cmd_qrt,
     'autopkgtest' : cmd_autopkgtest,
+    'testflinger' : cmd_testflinger,
     'open'        : cmd_open,
     'read'        : cmd_read,
     'sing'        : cmd_sing,