Merge lp:~sil2100/ubuntu-archive-tools/kernel-sru-review-esm into lp:ubuntu-archive-tools

[Łukasz Zemczak]

Add support for ESM into the kernel-sru-review tool.

Add support for ESM into the kernel-sru-review tool.

This branch adds the --esm option to the kernel review tool. The way it's done is by using the subscription URL and dget for all downloads from the security PPAs. For this to work, lazr.restfulclient version 0.13.5 or higher is required due to LP: #1714960. But this branch has been tested 'in production' and works with the required library version.

[lp:~sil2100/ubuntu-archive-tools/kernel-sru-review-esm]

1116. By Łukasz Zemczak on 2017-09-14

Some additional fixes to the ESM support in kernel-sru-review - handle the case of looking for ESM packages in the ESM release PPA.

1117. By Łukasz Zemczak on 2017-09-22

Forgot to commit a typo fix.

=== modified file 'kernel-sru-review'
--- kernel-sru-review	2017-07-14 10:10:08 +0000
+++ kernel-sru-review	2017-09-22 20:57:55 +0000
@@ -25,7 +25,7 @@
 ABI strings in debian/control et al.)
 
 USAGE:
-    kernel-sru-review [-s precise] linux
+    kernel-sru-review <bug number>
 """
 
 import glob
@@ -68,13 +68,16 @@
 
 
 def review_source_callback(lp, bugnum, tasks, full_packages, release, context):
+    # as per LP: #1290543, we need to evaluate (load) lp.me for
+    # getArchiveSubscritionURL to work
+    me = lp.load(lp.me.self_link)
     for source in full_packages:
         process_source_package(
-            source, release, context['archive'], context['ppa'],
+            source, release, me, context['archive'], context['ppa'],
             context['ubuntu'], context['startdir'], context['workdir'],
-            context['tardir'], context['tarcache'])
+            context['esm'], context['tardir'], context['tarcache'])
     tasks['proposed'].status = 'Fix Committed'
-    tasks['proposed'].assignee = lp.me
+    tasks['proposed'].assignee = me
     tasks['proposed'].lp_save()
 
 
@@ -121,8 +124,9 @@
                 shutil.copy(tarball, target)
 
 
-def process_source_package(source, release, archive, ppa, ubuntu,
-                           start_dir, work_dir, tardir=None, tar_cache=False):
+def process_source_package(source, release, me, archive, ppa, ubuntu,
+                           start_dir, work_dir, esm=False,
+                           tardir=None, tar_cache=False):
     series = ubuntu.getSeries(name_or_version=release)
 
     ppa_src = ppa.getPublishedSources(order_by_date=True,
@@ -132,25 +136,52 @@
     ppa_ver = ppa_src.source_package_version
     ppa_dsc = list(filter(
                   lambda x: x.endswith('.dsc'), ppa_src.sourceFileUrls()))[0]
-
-    archive_uploads = series.getPackageUploads(version=ppa_ver,
-                                               name=source,
-                                               archive=archive,
-                                               exact_match=True)
-    for upload in archive_uploads:
-        if upload.status != 'Rejected':
-            print("%s_%s already copied to Ubuntu archive (%s), skipping" %
-                  (source, ppa_ver, upload.status))
-            return
-
-    srcpkg = archive.getPublishedSources(order_by_date=True,
-                                         status='Published', exact_match=True,
+    if ppa.private:
+        priv_url = me.getArchiveSubscriptionURL(archive=ppa)
+        dsc_file = os.path.basename(ppa_dsc)
+        ppa_dsc = os.path.join(priv_url, 'pool/main/l', source, dsc_file)
+
+    # since we can have one archive for more than one 'pocket', no need to do
+    # API calls more than once
+    scanned = set()
+    for pocket in archive.values():
+        if pocket.self_link in scanned:
+            continue
+        archive_uploads = series.getPackageUploads(version=ppa_ver,
+                                                   name=source,
+                                                   archive=pocket,
+                                                   exact_match=True)
+        for upload in archive_uploads:
+            if upload.status != 'Rejected':
+                print("%s_%s already copied to Ubuntu archive (%s), skipping" %
+                      (source, ppa_ver, upload.status))
+                return
+        scanned.add(pocket.self_link)
+
+    # in cases where we have a separate archive for proposed and release,
+    # we need to check both places in the order proposed -> release
+    target = archive['proposed']
+    srcpkgs = target.getPublishedSources(order_by_date=True,
+                                         status='Published',
+                                         exact_match=True,
                                          distro_series=series,
-                                         source_name=source)[0]
+                                         source_name=source)
+    if len(srcpkgs) == 0:
+        target = archive['release']
+        srcpkgs = target.getPublishedSources(order_by_date=True,
+                                             status='Published',
+                                             exact_match=True,
+                                             distro_series=series,
+                                             source_name=source)
+    srcpkg = srcpkgs[0]
     source_ver = srcpkg.source_package_version
     source_dsc = list(filter(
                          lambda x: x.endswith('.dsc'),
                          srcpkg.sourceFileUrls()))[0]
+    if target.private:
+        priv_url = me.getArchiveSubscriptionURL(archive=target)
+        dsc_file = os.path.basename(source_dsc)
+        source_dsc = os.path.join(priv_url, 'pool/main/l', source, dsc_file)
 
     new_fullabi = ppa_ver.split('~')[0]
     new_majorabi = re.sub(r"\.[^.]+$", '', new_fullabi)
@@ -164,7 +195,12 @@
         work_dir, tardir, source, old_upstream, start_dir)
 
     # grab the old source first
-    pull_cmd = ['pull-lp-source', source, source_ver]
+    if esm:
+        pull_cmd = ['dget', '-u', source_dsc]
+    else:
+        # for non-ESM cases, it's just more reliable to use pull-lp-source
+        pull_cmd = ['pull-lp-source', source, source_ver]
+
     try:
         subprocess.check_call(pull_cmd)
     except subprocess.CalledProcessError as e:
@@ -217,6 +253,8 @@
     response = sys.stdin.readline()
     if response.strip().lower().startswith('y'):
         copy_cmd = ['copy-proposed-kernel', release, source]
+        if esm:
+            copy_cmd.append('--esm')
         copy_time = datetime.datetime.now(tz=pytz.utc)
         try:
             subprocess.check_call(copy_cmd)
@@ -238,7 +276,7 @@
         # only include uefi binaries that have appeared since we started the
         # copy to avoid accepting something that might have been improperly
         # copied into the queue by an "attacker" with upload rights.
-        uefis = series.getPackageUploads(archive=archive,
+        uefis = series.getPackageUploads(archive=target,
                                          pocket='Proposed',
                                          status='Unapproved',
                                          custom_type='uefi',
@@ -266,6 +304,8 @@
         "-C", "--cache-tarballs", dest="caching", action="store_true")
     parser.add_option(
         "-t", "--tarball-directory", dest="tardir", default=cachedir)
+    parser.add_option(
+        "-e", "--esm", dest="esm", action="store_true")
 
     opts, bugs = parser.parse_args()
 
@@ -290,14 +330,27 @@
         'ubuntu-archive-tools', opts.launchpad_instance, version='devel')
 
     ubuntu = launchpad.distributions['ubuntu']
-    archive = ubuntu.main_archive
-    ppa = launchpad.people['canonical-kernel-team'].getPPAByName(
-                distribution=ubuntu, name='ppa')
+    # for ESM (precise) we use special PPAs for CKT testing, -proposed and
+    # release
+    archive = {}
+    if opts.esm:
+        team = 'canonical-kernel-esm'
+        archive['proposed'] = launchpad.people[team].getPPAByName(
+            distribution=ubuntu, name='proposed')
+        archive['release'] = launchpad.people['ubuntu-esm'].getPPAByName(
+            distribution=ubuntu, name='esm')
+    else:
+        team = 'canonical-kernel-team'
+        archive['proposed'] = archive['release'] = ubuntu.main_archive
+    ppa = launchpad.people[team].getPPAByName(
+        distribution=ubuntu, name='ppa')
 
     start_dir = os.getcwd()
     context = {
       'archive': archive, 'ppa': ppa, 'ubuntu': ubuntu,
-      'tardir': tardir, 'tarcache': opts.caching, 'startdir': start_dir }
+      'tardir': tardir, 'tarcache': opts.caching, 'startdir': start_dir,
+      'esm': opts.esm
+      }
     for bugnum in bugs:
         with ExitStack() as resources:
             cwd = mkdtemp(prefix='kernel-sru-%s-' % bugnum, dir=start_dir)