Merge ~alexmurray/ubuntu-security-tools:cmd-qrt into ubuntu-security-tools:master

[Alex Murray]

[Alex Murray (alexmurray)]

Thanks for the review Marc - I have pushed extra commits to address your comments - see https://git.launchpad.net/~alexmurray/ubuntu-security-tools/commit/?id=d8b061e61a96bce2d22829c41fb4bf39302af080 which removes the revert part - and https://git.launchpad.net/~alexmurray/ubuntu-security-tools/commit/?id=2ebdc31cb89e21acd4f35d78816823fe13e00f84 which always adds -v when running tests so we always use verbose output. Perhaps in the future we might want a way to turn this off if the diffs get too messy with it but I think for now this an improvement.

[Emilia Torino (emitorino)]

This is great Alex! I added 2 inline comments with just refactoring suggestions + one question. Thanks!

[Eduardo Barretto (ebarretto)]

Adding a comment to Emi's suggestion as discussed on IRC.

[Eduardo Barretto (ebarretto)]

sorry for the spam, my comment didn't work, commenting again!

[Alex Murray (alexmurray)]

emitorino + ebarretto - I just pushed https://git.launchpad.net/~alexmurray/ubuntu-security-tools/commit/?id=67f072cbf84c8479e1ce67d85a732b3101f3f504 which should resolve your concerns.

[Alex Murray (alexmurray)]

Whoops - accidentally left in too much old code - fixed in subsequent commit https://git.launchpad.net/~alexmurray/ubuntu-security-tools/commit/?id=8b03bce

[Eduardo Barretto (ebarretto)]

LGTM

review: Approve

[Steve Beattie (sbeattie)]

I haven't looked at the implementation, so no real comment on that, but I did look briefly and tried to guess how it would be used; can we call the option 'test' rather than 'qrt', since the qrt tree is an implementation detail?

[Emilia Torino (emitorino)]

review: Approve

[Alex Murray (alexmurray)]

> I haven't looked at the implementation, so no real comment on that, but I did
> look briefly and tried to guess how it would be used; can we call the option
> 'test' rather than 'qrt', since the qrt tree is an implementation detail?

Well I want to add autopkgtest support as well and so was thinking it was easier to add both as separate sub-commands, otherwise if there was a single 'test' command this would have to have multiple mutually incompatible options depending on whether it ran qrt or autopkgtest etc - so to keep it simpler, I figure it is better to have separate qrt and autopkgtest commands.

diff --git a/build-tools/umt b/build-tools/umt
index 5aea099..c8939bc 100755
--- a/build-tools/umt
+++ b/build-tools/umt
@@ -36,6 +36,7 @@ binary_dest = '../binary'
 reports_dest = '../reports'
 previous_dest = '../previous'
 coverity_dest = '../coverity'
+tests_dest = '../tests'
 
 # Per-package overrides for the sbuild resolver
 # <sbuild resolver> = sbuild_dep_resolver_overrides[<srcpkg>][<ubuntu release>]
@@ -1364,19 +1365,11 @@ Acquire::Languages "none";
     recursive_rm(tempdir)
 
 
-def cmd_repo():
+def copy_to_repo(opt, details, quiet=False):
     '''Copy all built packages into local repository'''
-    parser = umt_optparse("usage: %prog repo [options]")
-
-    parser.add_option("-r", "--release",   dest="release",    default=False,
-                          help="specify repository to copy to")
-    parser.add_option("--purge",           dest="purge",      default=False, action='store_true',
-                          help="purge repository of all packages before copying")
-
-    (opt, args) = parser.parse_args()
     repo_base = ust['package_tools_repo_base']
-    details = parse_package_details(release = opt.release, skip_sanity=True)
-    print_details(details)
+    if not quiet:
+        print_details(details)
 
     repo_dest = os.path.join(repo_base, details['release'])
 
@@ -1404,7 +1397,8 @@ def cmd_repo():
         err("Could not find 'update_repo' (is '$UST' set?). Aborting.")
         sys.exit(1)
 
-    print("Repository: %s" % (repo_dest))
+    if not quiet:
+        print("Repository: %s" % (repo_dest))
 
     # Copy the source files, ignoring stuff we don't want
     files = os.listdir(source_dest)
@@ -1412,7 +1406,8 @@ def cmd_repo():
         (root, ext) = os.path.splitext(f)
         if ext in [ ".build", ".upload", ".debdiff" ]:
             continue
-        print("Copying '%s'..." % (f))
+        if not quiet:
+            print("Copying '%s'..." % (f))
         shutil.copy(os.path.join(source_dest, f), repo_dest)
 
     # Copy the files, as we may have built multiple architectures and the
@@ -1420,7 +1415,8 @@ def cmd_repo():
     files = os.listdir(binary_dest)
     for f in files:
         if f.endswith("deb"):
-            print("Copying '%s'..." % (f))
+            if not quiet:
+                print("Copying '%s'..." % (f))
             shutil.copy(os.path.join(binary_dest, f), repo_dest)
 
     (rc, report) = runcmd([update_repo, details['release']], stdin=sys.stdin, stdout=None)
@@ -1428,6 +1424,22 @@ def cmd_repo():
         err("failure running '%s':\n%s" % (update_repo, report))
         sys.exit(1)
 
+
+def add_repo_arguments(parser):
+    '''Add the command-line arguments used by repo'''
+
+    parser.add_option("-r", "--release",   dest="release",    default=False,
+                          help="specify repository to copy to")
+    parser.add_option("--purge",           dest="purge",      default=False, action='store_true',
+                          help="purge repository of all packages before copying")
+
+def cmd_repo():
+    '''Copy all built packages into local repository'''
+    parser = umt_optparse("usage: %prog repo [options]")
+    add_repo_arguments(parser)
+    (opt, args) = parser.parse_args()
+    details = parse_package_details(release = opt.release, skip_sanity=True)
+    copy_to_repo(opt, details)
     success("SUCCESS")
 
 def cmd_upload():
@@ -1556,6 +1568,178 @@ def cmd_search():
             print("%s: %s, Pocket: %s, Component: %s" % (release,
                 pkg_list[release][0], pkg_list[release][1], pkg_list[release][2]))
 
+def cmd_qrt():
+    '''Run QRT tests for the package in the current directory of unpacked source'''
+    parser = umt_optparse("usage: %prog qrt [options]")
+    add_repo_arguments(parser)
+    parser.add_option("-s", "--sudo",  dest="sudo",  default=False, action='store_true',
+                      help="Whether to run the QRT tests via sudo as a regular user")
+    parser.add_option("-u", "--user",  dest="user",  default=os.getlogin(), metavar='USER',
+                      help="The username to run the QRT tests as (default: current user)")
+    parser.add_option("-q", "--qrt-path",  dest="qrt_path",  default=os.getenv("QRT", ""), metavar='QRT_PATH',
+                      help="Path to the local copy of the qa-regression-tools (QRT) repo")
+    parser.add_option("--no-update",  dest="no_update",  default=False, action='store_true',
+                      help="Don't update UVT VM before initial test run")
+    parser.add_option("--no-snapshot",  dest="no_snapshot",  default=False, action='store_true',
+                      help="Don't revert or update snapshot of UVT VM on update")
+    parser.add_option("-n", "--dry-run",  dest="dry_run",  default=False, action='store_true',
+                      help="Don't actually execute tests, instead print what would be run")
+    parser.add_option("-p", "--uvt-prefix", dest="uvt_prefix", default="sec",
+                      help="Prefix to use for UVT VM names (default: sec)")
+    parser.add_option("-v", "--uvt-vm", dest="uvt_vm", default=None,
+                      help="VM use for UVT tests (default: use {uvt-prefix}-release-{arch})")
+    parser.add_option("--debug", default=False, action='store_true',
+                      help="Report additional debug details")
+    parser.add_option("-f", "--force",     dest="force",      default=False, action='store_true',
+                      help="force deletion of ../tests before running")
+    (opt, args) = parser.parse_args()
+
+    validate_toplevel()
+    details = parse_package_details(skip_sanity=True)
+    run_qrt_tests(opt, details)
+    # also alert if there appear to be other bits in QRT worth looking at
+    for d in ['build_testing', 'notes_testing']:
+        testing_dir = os.path.join(opt.qrt_path, d, details["package"])
+        try:
+            files = os.listdir(testing_dir)
+        except:
+            files = []
+        finally:
+            if len(files) > 0:
+                warn("QRT also contains the following relevant files for " + details["package"] + ":")
+                for f in files:
+                    warn(" " + os.path.join(d, details["package"], f))
+                warn("Please ensure you consult these")
+
+
+def runcmdopt(cmd, opt, okrc=[0]):
+    output = ""
+    if opt.debug:
+        print("[" + " ".join(cmd) + "]")
+    if not opt.dry_run:
+        (rc, output) = runcmd(cmd)
+        if rc not in okrc:
+            raise Exception("Failed to execute command: " + " ".join(cmd) + ": " + output)
+        if opt.debug:
+            print(output)
+    return output
+
+
+def run_qrt_tests(opt, details):
+    qrt_test = os.path.join(opt.qrt_path, "scripts", "test-" + details["package"] + ".py")
+    if not os.path.exists(qrt_test):
+        print("No QRT test found for " + details["package"] + ": expected to find in " + qrt_test)
+        return
+    qrt_tests_dest = os.path.join(tests_dest, "qrt")
+    prepare_dir(qrt_tests_dest, opt.force)
+    user = opt.user
+    sudo = ""
+    if opt.sudo:
+        sudo = 'echo ubuntu | sudo -S '
+    print("Running QRT test " + qrt_test + " for " + details["package"] + " as " + user + (" via sudo" if opt.sudo else ""))
+    if opt.uvt_vm is not None:
+        vm = opt.uvt_vm
+    else:
+        # TODO suport different arch's
+        vm = opt.uvt_prefix + "-" + details["release"] + "-amd64"
+    # find path to uvt binary - first see if UQT is defined, otherwise try
+    # in relative location to the current script
+    uqt = os.getenv("UQT")
+    if uqt != None and os.path.exists(os.path.join(uqt, "vm-tools", "uvt")):
+        uvt = os.path.join(uqt, "vm-tools", "uvt")
+    elif os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))),
+                      "ubuntu-qa-tools", "vm-tools", "uvt"):
+        uvt = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))),
+                           "ubuntu-qa-tools", "vm-tools", "uvt")
+    else:
+        err("Unable to find uvt - please define QRT as the path to ubuntu-qa-tools repo")
+    # catch any exception which occurred during execution
+    try:
+        if not opt.no_update:
+                # ensure vm is stopped first - unless no_snapshot we can stop
+                # it by force
+                cmd = [uvt, "stop", vm]
+                if not opt.no_snapshot:
+                    cmd.append("-f")
+                print("Ensuring uvt VM " + vm + " is stopped...")
+                runcmdopt(cmd, opt)
+                print("Updating uvt VM " + vm + "...")
+                cmd = [uvt, "update", "-f", vm]
+                if opt.no_snapshot:
+                    cmd.append("--nosnapshot")
+                runcmdopt(cmd, opt)
+        print("Launching uvt VM " + vm + "...")
+        # run headless since we don't use the GUI and wait for ssh
+        # availability
+        runcmdopt([uvt, "start", "-v", "-w", vm], opt)
+        print("Packaging QRT test...")
+        runcmdopt([os.path.join(opt.qrt_path, "scripts", "make-test-tarball"),
+                        qrt_test], opt)
+        print("Deploying QRT test...")
+        runcmdopt(["/usr/bin/scp",
+                   "/tmp/qrt-test-" + details["package"] + ".tar.gz",
+                   user + "@" + vm + ":"], opt)
+        print("Extracting QRT test...")
+        runcmdopt(["/usr/bin/ssh", "-T", user + "@" + vm,
+                   "tar -xvf qrt-test-" + details["package"] + ".tar.gz" ],
+                  opt)
+        print("Installing packages for QRT test...")
+        # always install packages via sudo
+        runcmdopt(["/usr/bin/ssh", "-T", user + "@" + vm,
+                   "cd ./qrt-test-" + details["package"] + "; " +
+                   "echo ubuntu | sudo -S ./install-packages ./test-" + details["package"] + ".py"],
+                  opt)
+        print("Executing QRT baseline test...")
+        report = runcmdopt(["/usr/bin/ssh", "-T", user + "@" + vm,
+                            "cd ./qrt-test-" + details["package"] + ";" +
+                            sudo + " ./test-" + details["package"] + ".py -v"],
+                           opt)
+        # save the report so we can compare it later
+        path = os.path.join(qrt_tests_dest, "qrt-test-" + details["package"] + '-orig.txt')
+        with open(path,"w+") as handle:
+            handle.write(report)
+            handle.flush()
+
+        print("Copying packages to umt repo...")
+        if not opt.dry_run:
+            copy_to_repo(opt, details, quiet=not opt.debug)
+
+        print("Enabling repo for uvt VM...")
+        runcmdopt([uvt, "repo", "-e", vm], opt)
+
+        print("Upgrading packages in uvt VM...")
+        runcmdopt(["/usr/bin/ssh", "-T", "root@" + vm,
+                   "apt-get dist-upgrade -y --force-yes"], opt)
+
+        print("Re-executing QRT test...")
+        report = runcmdopt(["/usr/bin/ssh", "-T", user + "@" + vm,
+                            "cd ./qrt-test-" + details["package"] + ";" +
+                            sudo + " ./test-" + details["package"] + ".py -v"], opt)
+        # save the report so we can compare it later
+        path = os.path.join(qrt_tests_dest, "qrt-test-" + details["package"] + '.txt')
+        with open(path,"w+") as handle:
+            handle.write(report)
+            handle.flush()
+
+        print("Generating diff of QRT tests output...")
+        report = runcmdopt(["/usr/bin/diff", "-u",
+                            os.path.join(qrt_tests_dest, "qrt-test-" + details["package"] + '-orig.txt'),
+                            os.path.join(qrt_tests_dest, "qrt-test-" + details["package"] + '.txt')],
+                           # diff returns 1 if different and 0 if same - so
+                           # both are valid ok return codes
+                           opt, [0, 1])
+        # save the diff
+        path = os.path.join(qrt_tests_dest, "qrt-test-" + details["package"] + '.diff')
+        with open(path,"w+") as handle:
+            handle.write(report)
+            handle.flush()
+        print("QRT test run diff in " + path)
+
+    except Exception as e:
+        err(str(e))
+    print("Stopping uvt VM " + vm + "...")
+    runcmdopt([uvt, "stop", vm], opt)
+
 #
 # Misc functions
 #
@@ -3273,6 +3457,7 @@ commands = {
     'compare-bin' : cmd_compare_bin,
     'repo'        : cmd_repo,
     'upload'      : cmd_upload,
+    'qrt'         : cmd_qrt,
     'open'        : cmd_open,
     'read'        : cmd_read,
     'help'        : cmd_help
diff --git a/build-tools/umt-completion.bash b/build-tools/umt-completion.bash
index 7935679..bd73861 100644
--- a/build-tools/umt-completion.bash
+++ b/build-tools/umt-completion.bash
@@ -12,7 +12,7 @@ _umt_complete()
     COMPREPLY=( $( compgen -W "$( grep Package: /var/lib/apt/lists/*Sources | awk '{print $2}' | sort | uniq )" "$cur") )
   }
 
-  umt_commands="search download changelog source binary build build-orig compare-log compare-bin sign check upload"
+  umt_commands="search download changelog source binary build build-orig compare-log compare-bin sign check upload qrt"
 
   if [ $COMP_CWORD -eq 1 ]; then
     COMPREPLY=( $(compgen -W "$umt_commands" -- ${COMP_WORDS[COMP_CWORD]}) )