Merge ~alexmurray/ubuntu-security-tools:cmd-autopkgtest into ubuntu-security-tools:master

[Alex Murray]

Like the recent qrt subcommand, add a new autopkgtest subcommand which will try and launch autopkgtest's using the built binary packages and if available, also run a baseline test against the build-orig binaries if these have been retained.

[Alex Murray (alexmurray)]

No review after over a week, merging anyway (this is self-contained so is unlikely to break anything anyway).

review: Approve

diff --git a/build-tools/umt b/build-tools/umt
index b42696c..0ae824e 100755
--- a/build-tools/umt
+++ b/build-tools/umt
@@ -37,6 +37,7 @@ reports_dest = '../reports'
 previous_dest = '../previous'
 coverity_dest = '../coverity'
 qrt_dest = '../qrt'
+autopkgtest_dest = '../autopkgtest'
 
 # Per-package overrides for the sbuild resolver
 # <sbuild resolver> = sbuild_dep_resolver_overrides[<srcpkg>][<ubuntu release>]
@@ -1748,6 +1749,114 @@ def run_qrt_tests(opt, details):
     print("Stopping uvt VM " + vm + "...")
     runcmdopt([uvt, "stop", vm], opt)
 
+
+def cmd_autopkgtest():
+    '''Run AUTOPKGTEST tests for the package in the current directory of unpacked source'''
+    parser = umt_optparse("usage: %prog autopkgtest [options]")
+    parser.add_option("-a", "--arch", dest="arch", default="amd64",
+                      help="Architecture to test (default: amd64)")
+    parser.add_option("-p", "--images-path", dest="images_path", default="~/images/",
+                      help="Location QEMU autopkgtest images to use for tests (default: ~/images/)")
+    parser.add_option("-i", "--image", dest="image", default=None,
+                      help="QEMU autopkgtest image name to use for tests (default: {images-path}/autopkgtest-{release}-{arch}.img)")
+    parser.add_option("-s", "--skip-baseline",  dest="skip_baseline",  default=False, action='store_true',
+                      help="Don't try and run the tests against the previous build as a baseline")
+    parser.add_option("-n", "--dry-run",  dest="dry_run",  default=False, action='store_true',
+                      help="Don't actually execute tests, instead print what would be run")
+    parser.add_option("--debug", default=False, action='store_true',
+                      help="Report additional debug details")
+    parser.add_option("-f", "--force",     dest="force",      default=False, action='store_true',
+                      help="force deletion of ../tests/autopkgtest before running")
+    (opt, args) = parser.parse_args()
+
+    validate_toplevel()
+    details = parse_package_details(skip_sanity=True)
+    run_autopkgtest_tests(opt, details)
+
+
+def run_autopkgtest_tests(opt, details):
+    autopkgtest_test = os.path.join("debian", "tests", "control")
+    if not os.path.exists(autopkgtest_test):
+        print("No autopkgtest found for " + details["package"] + ": expected to find in " + autopkgtest_test)
+        return
+    if not opt.dry_run:
+        prepare_dir(autopkgtest_dest, opt.force)
+    if opt.image is not None:
+        image = opt.image
+    else:
+        image = os.path.join(os.path.expanduser(opt.images_path), "autopkgtest-" + details["release"] + "-" + opt.arch + ".img")
+    try:
+        if not os.path.exists(image):
+            print("autopkgtest qemu image '%s' does not exist - do you wish to create it (y|N)? " % image, end=' ', flush=True)
+            answer = sys.stdin.readline().strip().lower()
+            print("")
+            if answer.startswith("y"):
+                print("Creating autopkgtest qemu image '%s'" % image)
+                runcmdopt(["/usr/bin/autopkgtest-buildvm-ubuntu-cloud",
+                           "-r", details["release"],
+                           "-o", os.path.dirname(image)],
+                          opt)
+                print("Created autopkgtest qemu image '%s'" % image)
+        # for the baseline test, use the orig (aka previous) .changes
+        if not opt.skip_baseline:
+            prev_changes = glob.glob("../previous/binary/*" + opt.arch + ".changes")
+            prev_version = None
+            if len(prev_changes) > 0:
+                prev_changes = prev_changes[0]
+                prev_version = os.path.splitext(os.path.basename(prev_changes))[0].replace(details["package"] + "_", "").replace("_source", "")
+                print("Executing autopkgtest baseline test using %s on %s..." % (prev_changes, image))
+                report = runcmdopt(["/usr/bin/autopkgtest", "-U",
+                                    "-o", os.path.join(autopkgtest_dest, prev_version),
+                                    prev_changes,
+                                    "--",
+                                    "qemu", image], opt)
+                if not opt.dry_run:
+                    # save the report so we can compare it later
+                    path = os.path.join(autopkgtest_dest, "autopkgtest-test-" + details["package"] + "-" + prev_version + ".txt")
+                    with open(path,"w+") as handle:
+                        handle.write(report)
+                        handle.flush()
+            else:
+                warn("No previous build found - unable to run baseline autopkgtest (try running `umt build-orig --leave`) ...")
+        changes = glob.glob("../binary/*" + opt.arch + ".changes")
+        version = None
+        if len(changes) > 0:
+            changes = changes[0]
+            version = os.path.splitext(os.path.basename(changes))[0].replace(details["package"] + "_", "").replace("_source", "")
+            print("Executing autopkgtest test using %s on %s..." % (changes, image))
+            report = runcmdopt(["/usr/bin/autopkgtest", "-U",
+                                "-o", os.path.join(autopkgtest_dest, version),
+                                changes,
+                                "--",
+                                "qemu", image], opt)
+            if not opt.dry_run:
+                # save the report so we can compare it later
+                path = os.path.join(autopkgtest_dest, "autopkgtest-test-" + details["package"] + "-" + version + ".txt")
+                with open(path,"w+") as handle:
+                    handle.write(report)
+                    handle.flush()
+        else:
+            warn("No current binary build found - not running autopkgtest...")
+        if (prev_version is not None and version is not None and
+            os.path.exists(os.path.join(autopkgtest_dest, "autopkgtest-test-" + details["package"] + "-" + prev_version + ".txt")) and
+            os.path.exists(os.path.join(autopkgtest_dest, "autopkgtest-test-" + details["package"] + "-" + version + ".txt"))):
+            print("Generating diff of autopkgtest tests output...")
+            report = runcmdopt(["/usr/bin/diff", "-u",
+                                os.path.join(autopkgtest_dest, "autopkgtest-test-" + details["package"] + "-" + prev_version + ".txt"),
+                                os.path.join(autopkgtest_dest, "autopkgtest-test-" + details["package"] + "-" + version + ".txt")],
+                               # diff returns 1 if different and 0 if same - so
+                               # both are valid ok return codes
+                               opt, [0, 1])
+            if not opt.dry_run:
+                # save the diff
+                path = os.path.join(autopkgtest_dest, "qrt-test-" + details["package"] + '.diff')
+                with open(path,"w+") as handle:
+                    handle.write(report)
+                    handle.flush()
+            print("autopkgtest test run diff in " + path)
+    except Exception as e:
+        err(str(e))
+
 #
 # Misc functions
 #
@@ -3466,6 +3575,7 @@ commands = {
     'repo'        : cmd_repo,
     'upload'      : cmd_upload,
     'qrt'         : cmd_qrt,
+    'autopkgtest' : cmd_autopkgtest,
     'open'        : cmd_open,
     'read'        : cmd_read,
     'help'        : cmd_help
diff --git a/build-tools/umt-completion.bash b/build-tools/umt-completion.bash
index bd73861..b62b8b1 100644
--- a/build-tools/umt-completion.bash
+++ b/build-tools/umt-completion.bash
@@ -12,7 +12,7 @@ _umt_complete()
     COMPREPLY=( $( compgen -W "$( grep Package: /var/lib/apt/lists/*Sources | awk '{print $2}' | sort | uniq )" "$cur") )
   }
 
-  umt_commands="search download changelog source binary build build-orig compare-log compare-bin sign check upload qrt"
+  umt_commands="search download changelog source binary build build-orig compare-log compare-bin sign check upload qrt autopkgtest"
 
   if [ $COMP_CWORD -eq 1 ]; then
     COMPREPLY=( $(compgen -W "$umt_commands" -- ${COMP_WORDS[COMP_CWORD]}) )