Merge lp:~sil2100/ubuntu-archive-tools/diff-master into lp:ubuntu-archive-tools

[Łukasz Zemczak]

Add an option to perform a brief diff between reviewed kernel and its master kernel, if present. This is useful when reviewing derivatives/backports, to get a better sense of if all the master changes are present.

Add an option to perform a brief diff between reviewed kernel and its master kernel, if present. This is useful when reviewing derivatives/backports, to get a better sense of if all the master changes are present.

[lp:~sil2100/ubuntu-archive-tools/diff-master]

1145. By Łukasz Zemczak on 2018-01-22

Some fixes for issues noticed during first test runs of the new functionality.

[Łukasz Zemczak (sil2100)]

Initial test run done. It should be good-ish now for merging.

[lp:~sil2100/ubuntu-archive-tools/diff-master]

1146. By Łukasz Zemczak on 2018-01-31

For kernel reviews, only match per 'Published' status when no version number is given.

1147. By Łukasz Zemczak on 2018-02-06

Fix working with ESM.

=== modified file 'kernel-sru-review'
--- kernel-sru-review	2017-12-12 11:37:10 +0000
+++ kernel-sru-review	2018-02-06 09:55:13 +0000
@@ -45,6 +45,87 @@
 from kernel_workflow import *
 
 
+def get_master_kernel(lp, bugnum):
+    current = lp.bugs[bugnum]
+    master = None
+    backport_re = re.compile(r'^kernel-sru-backport-of-(\d+)$')
+    derivative_re = re.compile(r'^kernel-sru-derivative-of-(\d+)$')
+
+    for tag in current.tags:
+        num = derivative_re.match(tag)
+        if not num:
+            num = backport_re.match(tag)
+        if num:
+            master = lp.bugs[num.group(1)]
+
+    if not master:
+        print("No master kernel.")
+        return (None, None)
+    return get_name_and_version_from_bug(master)
+
+
+def get_kernel_dsc(me, archive, source, series=None, version=None):
+    kwargs = {
+        'order_by_date': True,
+        'exact_match': True,
+        'source_name': source
+    }
+    if version:
+        kwargs['version'] = version
+    if series:
+        kwargs['status'] = 'Published'
+        kwargs['distro_series'] = series
+
+    # in cases where we have a separate archive for proposed and release,
+    # we need to check both places in the order proposed -> release
+    target = archive['proposed']
+    srcpkgs = target.getPublishedSources(**kwargs)
+    if len(srcpkgs) == 0:
+        target = archive['release']
+        srcpkgs = target.getPublishedSources(**kwargs)
+        if len(srcpkgs) == 0 and 'non-esm' in archive:
+            target = archive['non-esm']
+            srcpkgs = target.getPublishedSources(**kwargs)
+    srcpkg = srcpkgs[0]
+    source_ver = srcpkg.source_package_version
+    source_dsc = list(filter(
+                         lambda x: x.endswith('.dsc'),
+                         srcpkg.sourceFileUrls()))[0]
+    if target.private:
+        priv_url = me.getArchiveSubscriptionURL(archive=target)
+        dsc_file = os.path.basename(source_dsc)
+        source_dsc = os.path.join(priv_url, 'pool/main/l', source, dsc_file)
+
+    return (source_dsc, source_ver)
+
+
+def generate_diff_from_master(me, archive, master_source, master_version,
+                              new_source, new_upstream,
+                              work_dir, tardir, start_dir):
+    master_upstream = master_version.split('-')[0]
+
+    master_dsc, master_version = get_kernel_dsc(
+        me, archive, master_source, version=master_version)
+    fetch_tarball_from_cache(
+        work_dir, tardir, master_source, master_upstream, start_dir)
+
+    # grab the old source first
+    dget_cmd = ['dget', '-u', master_dsc]
+    try:
+        subprocess.check_call(dget_cmd)
+    except subprocess.CalledProcessError as e:
+        print("Failed to get master source for %s at version %s" %
+              (master_source, master_version))
+        raise e
+
+    # generate the diff
+    print("Generating brief diff between new kernel and master (%s) to %s" %
+          (master_version, os.path.join(work_dir, 'master_diff')))
+    diff_cmd = ('diff -rq "{}-{}" "{}-{}" >master_diff').format(
+        master_source, master_upstream, new_source, new_upstream)
+    subprocess.call(diff_cmd, shell=True)
+
+
 def review_task_callback(lp, bugnum, task, context):
     if str(task.target) != \
        ('%skernel-sru-workflow/promote-to-proposed' % str(lp._root_uri)):
@@ -71,11 +152,16 @@
     # as per LP: #1290543, we need to evaluate (load) lp.me for
     # getArchiveSubscritionURL to work
     me = lp.load(lp.me.self_link)
+    master_source = None
+    master_version = None
+    if context['diff']:
+        master_source, master_version = get_master_kernel(lp, bugnum)
     for source in full_packages:
         process_source_package(
             source, release, me, context['archive'], context['ppa'],
             context['ubuntu'], context['startdir'], context['workdir'],
-            context['tardir'], context['esm'], context['tarcache'])
+            context['tardir'], context['esm'], context['tarcache'],
+            master_source, master_version)
     tasks['proposed'].status = 'Fix Committed'
     tasks['proposed'].assignee = me
     tasks['proposed'].lp_save()
@@ -123,7 +209,8 @@
 
 def process_source_package(source, release, me, archive, ppa, ubuntu,
                            start_dir, work_dir, tardir,
-                           esm=False, tar_cache=False):
+                           esm=False, tar_cache=False,
+                           master_source=None, master_version=None):
     series = ubuntu.getSeries(name_or_version=release)
 
     ppa_src = ppa.getPublishedSources(order_by_date=True,
@@ -155,30 +242,7 @@
                 return
         scanned.add(pocket.self_link)
 
-    # in cases where we have a separate archive for proposed and release,
-    # we need to check both places in the order proposed -> release
-    target = archive['proposed']
-    srcpkgs = target.getPublishedSources(order_by_date=True,
-                                         status='Published',
-                                         exact_match=True,
-                                         distro_series=series,
-                                         source_name=source)
-    if len(srcpkgs) == 0:
-        target = archive['release']
-        srcpkgs = target.getPublishedSources(order_by_date=True,
-                                             status='Published',
-                                             exact_match=True,
-                                             distro_series=series,
-                                             source_name=source)
-    srcpkg = srcpkgs[0]
-    source_ver = srcpkg.source_package_version
-    source_dsc = list(filter(
-                         lambda x: x.endswith('.dsc'),
-                         srcpkg.sourceFileUrls()))[0]
-    if target.private:
-        priv_url = me.getArchiveSubscriptionURL(archive=target)
-        dsc_file = os.path.basename(source_dsc)
-        source_dsc = os.path.join(priv_url, 'pool/main/l', source, dsc_file)
+    source_dsc, source_ver = get_kernel_dsc(me, archive, source, series=series)
 
     new_fullabi = ppa_ver.split('~')[0]
     new_majorabi = re.sub(r"\.[^.]+$", '', new_fullabi)
@@ -239,6 +303,14 @@
     if not real_tardir and tar_cache:
         save_tarball_to_cache(work_dir, tardir, source, new_upstream)
 
+    if (master_source and master_version and
+            '-meta' not in source and '-signed' not in source):
+        # if requested, we also generate a brief diff between the new kernel
+        # and its 'master' kernel
+        generate_diff_from_master(
+            me, archive, master_source, master_version, source, new_upstream,
+            work_dir, tardir, start_dir)
+
     # look at the diff
     diff_cmd = ('diff -uNr "{}-{}.old" "{}-{}" | filterdiff -x \'**/abi/**\''
                 + ' | sensible-pager').format(
@@ -273,7 +345,7 @@
         # only include uefi binaries that have appeared since we started the
         # copy to avoid accepting something that might have been improperly
         # copied into the queue by an "attacker" with upload rights.
-        uefis = series.getPackageUploads(archive=target,
+        uefis = series.getPackageUploads(archive=archive['release'],
                                          pocket='Proposed',
                                          status='Unapproved',
                                          custom_type='uefi',
@@ -304,6 +376,9 @@
         "-t", "--tarball-directory", dest="tardir", default=cachedir)
     parser.add_option(
         "-e", "--esm", dest="esm", action="store_true")
+    parser.add_option(
+        "-d", "--diff-against-master", dest="diff_master",
+        action="store_true")
 
     opts, bugs = parser.parse_args()
 
@@ -334,6 +409,7 @@
             distribution=ubuntu, name='proposed')
         archive['release'] = launchpad.people['ubuntu-esm'].getPPAByName(
             distribution=ubuntu, name='esm')
+        archive['non-esm'] = ubuntu.main_archive
     else:
         team = 'canonical-kernel-team'
         archive['proposed'] = archive['release'] = ubuntu.main_archive
@@ -344,7 +420,7 @@
     context = {
       'archive': archive, 'ppa': ppa, 'ubuntu': ubuntu,
       'tardir': tardir, 'tarcache': opts.caching, 'startdir': start_dir,
-      'esm': opts.esm
+      'esm': opts.esm, 'diff': opts.diff_master
       }
     for bugnum in bugs:
         with ExitStack() as resources:

=== modified file 'kernel_workflow.py'
--- kernel_workflow.py	2016-12-21 15:21:20 +0000
+++ kernel_workflow.py	2018-02-06 09:55:13 +0000
@@ -23,13 +23,25 @@
 class KernelWorkflowError(Exception):
     """An exception occurred with the state of the workflow bug"""
 
+
+def get_name_and_version_from_bug(bug):
+    title_re = re.compile(
+        r'^(?P<package>[a-z0-9.-]+): (?P<version>[0-9.-]+[0-9a-z.~-]*)'
+        + ' -proposed tracker$')
+    match = title_re.search(bug.title)
+    if not match:
+        print("Ignoring bug %s, not a kernel SRU tracking bug" % bugnum)
+        return (None, None)
+    package = match.group('package')
+    version = match.group('version')
+    # FIXME: check that the package version is correct for the suite
+    return (package, version)
+
+
 def process_sru_bug(lp, bugnum, task_callback, source_callback, context=None):
     """Process the indicated bug and call the provided helper functions
     as needed
     """
-    title_re = re.compile(
-        r'^(?P<package>[a-z0-9.-]+): (?P<version>[0-9.-]+[0-9a-z.~-]*)'
-         + ' -proposed tracker$')
     package_re = re.compile(
         (r'^%subuntu/(?P<release>[0-9a-z.-]+)/'
          + '\+source/(?P<package>[a-z0-9.-]+)$') % str(lp._root_uri))
@@ -43,13 +55,9 @@
     updates_task = None
     security_task = None
     bug = lp.bugs[int(bugnum)]
-    match = title_re.search(bug.title)
-    if not match:
-        print("Ignoring bug %s, not a kernel SRU tracking bug" % bugnum)
+    package, version = get_name_and_version_from_bug(bug)
+    if not package or not version:
         return
-    package = match.group('package')
-    version = match.group('version')
-    # FIXME: check that the package version is correct for the suite
 
     task_results = {}
     for task in bug.bug_tasks: