Merge lp:~apw/ubuntu-archive-tools/copy-proposed-kernels-routing-updates into lp:ubuntu-archive-tools

[Andy Whitcroft]

Update to support full routing for the linux-fips and linux-ibm-gt kernels. Also brings routing for the split copy from build-ppa -> signing-ppa -> proposed for ESM and FIPS. By default copy-proposed-kernel will copy from the build-ppa to the next step; either proposed or signing as needed. The second copy is triggered (once built in the signing PPA) via a second copy-proposed-kernel --from-signing pass.

This fits with deployed changes to the kernel workflow manager which changes the meaning of promote-to-proposed to mean "do the AA review and move to the next step", and adds promote-signing-to-proposed to request the second copy. It monitors builds in the signing PPAs.

[Steve Langasek (vorlon)]

Various suggestions for improvement, but nothing that I think should necessarily block landing this.

review: Needs Fixing

[Andy Whitcroft (apw)]

Ok updated version pushed which covers the easily fixable things.

[Steve Langasek (vorlon)]

review: Approve

[lp:~apw/ubuntu-archive-tools/copy-proposed-kernels-routing-updates]

1212. By Andy Whitcroft on 2019-04-09

copy-proposed-kernel: add support for routing via signing PPAs

Update to support full routing for the linux-fips and linux-ibm-gt
kernels. Also brings routing for the split copy from build-ppa ->
signing-ppa -> proposed for ESM and FIPS. By default copy-proposed-kernel
will copy from the build-ppa to the next step; either proposed or signing
as needed. The second copy is triggered (once built in the signing PPA)
via a second copy-proposed-kernel --from-signing pass.

This fits with deployed changes to the kernel workflow manager which
changes the meaning of promote-to-proposed to mean "do the AA review and
move to the next step", and adds promote-signing-to-proposed to request
the second copy. It monitors builds in the signing PPAs.

Signed-off-by: Andy Whitcroft <email address hidden>

=== modified file 'copy-proposed-kernel'
--- copy-proposed-kernel	2018-08-14 08:31:43 +0000
+++ copy-proposed-kernel	2019-04-03 20:06:41 +0000
@@ -24,47 +24,310 @@
 from __future__ import print_function
 
 import argparse
+from contextlib import contextmanager
+from copy import copy
+from StringIO import StringIO
 import sys
+import unittest
 
 from launchpadlib.launchpad import Launchpad
 
 
+class TestBase(unittest.TestCase):
+    class FakeArgs:
+        def __init__(self, **kwargs):
+            self.series = None
+            self.source = None
+            self.security = False
+            self.security2 = False
+            self.esm = False
+            self.fips = False
+            self.ibmgt = False
+            self.to_signing = False
+            self.from_signing = False
+
+            self.update(**kwargs)
+
+        def update(self, **kwargs):
+            for (key, value) in kwargs.items():
+                setattr(self, key, value)
+            return self
+
+    @contextmanager
+    def capture(self):
+        new_out, new_err = StringIO(), StringIO()
+        old_out, old_err = sys.stdout, sys.stderr
+        try:
+            sys.stdout, sys.stderr = new_out, new_err
+            yield sys.stdout, sys.stderr
+        finally:
+            sys.stdout, sys.stderr = old_out, old_err
+
+
+class TestRouting(TestBase):
+    def test_default(self):
+        expected = ('~canonical-kernel-team/ubuntu/ppa', 'ubuntu', False)
+        result = routing(self.FakeArgs())
+        self.assertEqual(expected, result)
+
+    def test_security(self):
+        expected = ('~canonical-kernel-security-team/ubuntu/ppa', 'ubuntu', True)
+        result = routing(self.FakeArgs(security=True))
+        self.assertEqual(expected, result)
+
+    def test_security2(self):
+        expected = ('~canonical-kernel-security-team/ubuntu/ppa2', 'ubuntu', True)
+        result = routing(self.FakeArgs(security2=True))
+        self.assertEqual(expected, result)
+
+    def test_to_signing(self):
+        expected = ('~canonical-kernel-team/ubuntu/ppa', None, False)
+        result = routing(self.FakeArgs(to_signing=True))
+        self.assertEqual(expected, result)
+
+    def test_from_signing(self):
+        expected = (None, 'ubuntu', False)
+        result = routing(self.FakeArgs(from_signing=True))
+        self.assertEqual(expected, result)
+
+    def test_esm(self):
+        expected = ('~canonical-kernel-esm/ubuntu/ppa', '~canonical-kernel-esm/ubuntu/proposed', False)
+        result = routing(self.FakeArgs(esm=True))
+        self.assertEqual(expected, result)
+
+    def test_esm_security(self):
+        expected = ('~canonical-kernel-security-team/ubuntu/esm', '~canonical-kernel-esm/ubuntu/proposed', False)
+        result = routing(self.FakeArgs(esm=True, security=True))
+        self.assertEqual(expected, result)
+
+    def test_esm_security2(self):
+        expected = (None, '~canonical-kernel-esm/ubuntu/proposed', False)
+        result = routing(self.FakeArgs(esm=True, security2=True))
+        self.assertEqual(expected, result)
+
+    def test_esm_to_signing(self):
+        expected = ('~canonical-kernel-esm/ubuntu/ppa', '~canonical-signing/ubuntu/esm', False)
+        result = routing(self.FakeArgs(esm=True, to_signing=True))
+        self.assertEqual(expected, result)
+
+    def test_esm_from_signing(self):
+        expected = ('~canonical-signing/ubuntu/esm', '~canonical-kernel-esm/ubuntu/proposed', False)
+        result = routing(self.FakeArgs(esm=True, from_signing=True))
+        self.assertEqual(expected, result)
+
+    # Autorouting will enable to_signing, the user will then want to switch us
+    # to from_signing in order to perform phase two copies.  To ensure this is
+    # simple we make from_signing take presidence over to_signing.  Test this
+    # is honoured correctly.
+    def test_esm_from_signing_override_to_signing(self):
+        expected = ('~canonical-signing/ubuntu/esm', '~canonical-kernel-esm/ubuntu/proposed', False)
+        result = routing(self.FakeArgs(esm=True, to_signing=True, from_signing=True))
+        self.assertEqual(expected, result)
+
+    def test_fips(self):
+        expected = ('~ubuntu-advantage/ubuntu/fips-kernel-source', '~ubuntu-advantage/ubuntu/fips-proposed', False)
+        result = routing(self.FakeArgs(fips=True))
+        self.assertEqual(expected, result)
+
+    def test_fips_security(self):
+        expected = ('~canonical-kernel-security-team/ubuntu/ppa', '~ubuntu-advantage/ubuntu/fips-proposed', False)
+        result = routing(self.FakeArgs(fips=True, security=True))
+        self.assertEqual(expected, result)
+
+    def test_fips_security2(self):
+        expected = ('~canonical-kernel-security-team/ubuntu/ppa2', '~ubuntu-advantage/ubuntu/fips-proposed', False)
+        result = routing(self.FakeArgs(fips=True, security2=True))
+        self.assertEqual(expected, result)
+
+    def test_fips_to_signing(self):
+        expected = ('~ubuntu-advantage/ubuntu/fips-kernel-source', '~canonical-signing/ubuntu/fips', False)
+        result = routing(self.FakeArgs(fips=True, to_signing=True))
+        self.assertEqual(expected, result)
+
+    def test_fips_from_signing(self):
+        expected = ('~canonical-signing/ubuntu/fips', '~ubuntu-advantage/ubuntu/fips-proposed', False)
+        result = routing(self.FakeArgs(fips=True, from_signing=True))
+        self.assertEqual(expected, result)
+
+    def test_ibmgt(self):
+        expected = ('~ibm-cloud/ubuntu/build', '~ibm-cloud/ubuntu/proposed', False)
+        result = routing(self.FakeArgs(ibmgt=True))
+        self.assertEqual(expected, result)
+
+    def test_ibmgt_security(self):
+        expected = ('~canonical-kernel-security-team/ubuntu/ppa', '~ibm-cloud/ubuntu/proposed', False)
+        result = routing(self.FakeArgs(ibmgt=True, security=True))
+        self.assertEqual(expected, result)
+
+    def test_ibmgt_security2(self):
+        expected = ('~canonical-kernel-security-team/ubuntu/ppa2', '~ibm-cloud/ubuntu/proposed', False)
+        result = routing(self.FakeArgs(ibmgt=True, security2=True))
+        self.assertEqual(expected, result)
+
+
+def routing(args):
+    unembargo = False
+
+    # Default routing.
+    security_name = (
+        '~canonical-kernel-security-team/ubuntu/ppa',
+        '~canonical-kernel-security-team/ubuntu/ppa2',
+        )
+    signing = None
+
+    # ESM specific routing
+    if args.esm:
+        ppa_name = '~canonical-kernel-esm/ubuntu/ppa'
+        to = '~canonical-kernel-esm/ubuntu/proposed'
+        signing = '~canonical-signing/ubuntu/esm'
+        security_name = (
+            '~canonical-kernel-security-team/ubuntu/esm',
+            None,
+            )
+
+    # FIPS specific routing
+    elif args.fips:
+        ppa_name = '~ubuntu-advantage/ubuntu/fips-kernel-source'
+        to = '~ubuntu-advantage/ubuntu/fips-proposed'
+        signing = '~canonical-signing/ubuntu/fips'
+
+    # IBMGT specific routing
+    elif args.ibmgt:
+        ppa_name = '~ibm-cloud/ubuntu/build'
+        to = '~ibm-cloud/ubuntu/proposed'
+
+    # Default routing.
+    else:
+        ppa_name = '~canonical-kernel-team/ubuntu/ppa'
+        to = 'ubuntu'
+        if args.security or args.security2:
+            # Allow us to unembargo when releasing from security to ubuntu.
+            unembargo = True
+
+    # Handle security routing.
+    if args.security:
+        ppa_name = security_name[0]
+    elif args.security2:
+        ppa_name = security_name[1]
+
+    # Handle signing routing.
+    if args.from_signing:
+        ppa_name = signing
+    elif args.to_signing:
+        to = signing
+
+    return (ppa_name, to, unembargo)
+
+class TestAutoRoute(TestBase):
+    def test_default(self):
+        results = self.FakeArgs(series='bionic', source='linux')
+        expected = copy(results).update()
+        with self.capture() as (out, err):
+            auto_route(results)
+        self.assertEqual(expected.__dict__, results.__dict__)
+        self.assertEqual('', out.getvalue().strip())
+
+    def test_esm_precise_linux(self):
+        results = self.FakeArgs(series='precise', source='linux')
+        expected = copy(results).update(esm=True)
+        with self.capture() as (out, err):
+            auto_route(results)
+        self.assertEqual(expected.__dict__, results.__dict__)
+        self.assertIn('from and to ESM', out.getvalue().strip())
+        self.assertNotIn('via signing', out.getvalue().strip())
+
+    def test_esm_precise_linux_lts_trusty(self):
+        results = self.FakeArgs(series='precise', source='linux-lts-trusty')
+        expected = copy(results).update(esm=True, to_signing=True)
+        with self.capture() as (out, err):
+            auto_route(results)
+        self.assertEqual(expected.__dict__, results.__dict__)
+        self.assertIn('from and to ESM', out.getvalue().strip())
+        self.assertIn('via signing', out.getvalue().strip())
+
+    def test_fips_bionic_linux_fips(self):
+        results = self.FakeArgs(series='bionic', source='linux-fips')
+        expected = copy(results).update(fips=True, to_signing=True)
+        with self.capture() as (out, err):
+            auto_route(results)
+        self.assertEqual(expected.__dict__, results.__dict__)
+        self.assertIn('from and to FIPS', out.getvalue().strip())
+        self.assertIn('via signing', out.getvalue().strip())
+
+    def test_ibmgt_bionic_linux_ibmgt(self):
+        results = self.FakeArgs(series='bionic', source='linux-ibm-gt')
+        expected = copy(results).update(ibmgt=True)
+        with self.capture() as (out, err):
+            auto_route(results)
+        self.assertEqual(expected.__dict__, results.__dict__)
+        self.assertIn('from and to IBM-GT', out.getvalue().strip())
+        self.assertNotIn('via signing', out.getvalue().strip())
+
+def auto_route(args):
+    before = (args.esm, args.fips, args.ibmgt, args.to_signing)
+
+    if args.series == 'precise':
+        args.esm = True
+    if args.esm:
+        if not (args.series == 'precise' and args.source in ('linux', 'linux-meta')):
+            args.to_signing = True
+
+    if args.source.endswith('-fips'):
+        args.fips = True
+    if args.fips:
+        args.to_signing = True
+
+    if args.source.endswith('-ibm-gt'):
+        args.ibmgt = True
+
+    if before != (args.esm, args.fips, args.ibmgt, args.to_signing):
+        msg = "NOTE: directing copy "
+        if args.esm:
+            msg += "from and to ESM"
+        elif args.fips:
+            msg += "from and to FIPS"
+        elif args.ibmgt:
+            msg += "from and to IBM-GT PPAs"
+        if args.to_signing:
+            msg += " via signing"
+        print(msg)
+
+
+# SELF-TESTS:
+if len(sys.argv) >= 2 and sys.argv[1] == '--self-test':
+    unittest.main(argv=[sys.argv[0]] + sys.argv[2:])
+    sys.exit(0)
+
 parser = argparse.ArgumentParser(description='Copy a proposed kernel to the apropriate archive pocket')
 parser.add_argument('--dry-run', action='store_true', help='Do everything but actually copy the package')
 parser.add_argument('--security', '-S', action='store_true', help='Copy from the kernel security PPA')
 parser.add_argument('--security2', action='store_true', help='Copy from the kernel security PPA2')
 parser.add_argument('--esm', '-E', action='store_true', help='Copy from the kernel ESM PPA and to the kernel ESM proposed PPA')
+parser.add_argument('--fips', action='store_true', help='Copy from the kernel FIPS PPA and to the kernel FIPS proposed PPA')
+parser.add_argument('--ibmgt', action='store_true', help='Copy from the kernel IBM-GT build PPA to the corresponding proposed PPA')
 parser.add_argument('--no-auto', action='store_true', help='Turn off automatic detection of ESM et al based on series')
+parser.add_argument('--to-signing', action='store_true', help='Copy from the kernel ESM/FIPS PPA to the ESM/FIPS signing PPA')
+parser.add_argument('--from-signing', action='store_true', help='Copy from the ESM/FIPS signing PPA to the ESM/FIPS proposed PPA')
 parser.add_argument('series', action='store', help='The series the source package is in')
 parser.add_argument('source', action='store', help='The source package name')
 
 args = parser.parse_args()
 
-to = 'ubuntu'
-ppa_name = '~canonical-kernel-team/ubuntu/ppa'
-security = False
-
 # If we are allowed to intuit destinations do so:
 # 1) precise is now destined for the ESM PPAs
 if not args.no_auto:
-    if args.series == 'precise' and not args.esm:
-        print("NOTE: directing copy from and to ESM for precise")
-        args.esm = True
-
-if args.esm:
-    ppa_name = '~canonical-kernel-esm/ubuntu/ppa'
-    to = '~canonical-kernel-esm/ubuntu/proposed'
-    to_pocket = 'release'
-if args.security:
-    ppa_name = '~canonical-kernel-security-team/ubuntu/ppa'
-    if not args.esm:
-        security = True
-    else:
-        ppa_name = '~canonical-kernel-security-team/ubuntu/esm'
-if args.security2:
-    ppa_name = '~canonical-kernel-security-team/ubuntu/ppa2'
-    if not args.esm:
-        security = True
+    auto_route(args)
+
+(ppa_name, to, security) = routing(args)
+
+##print("ppa_name<{}> to<{}>".format(ppa_name, to))
+
+if ppa_name is None:
+    print("ERROR: bad source PPA")
+    sys.exit(1)
+if to is None:
+    print("ERROR: bad destination")
+    sys.exit(1)
 
 (release, pkg) = (args.series, args.source)
 
@@ -75,26 +338,34 @@
 kernel_ppa = launchpad.archives.getByReference(
     reference=ppa_name)
 
+# Grab a reference to the 'to' archive and select a pocket.
+to_archive = launchpad.archives.getByReference(reference=to)
+if to == 'ubuntu':
+    to_pocket = 'proposed'
+else:
+    to_pocket = 'release'
+
 # get current version in PPA for that series
 versions = kernel_ppa.getPublishedSources(
     source_name=pkg, exact_match=True, status='Published', pocket='Release',
     distro_series=distro_series)
-assert versions.total_size == 1
-version = versions[0].source_package_version
+version = None
+if versions.total_size == 1:
+    version = versions[0].source_package_version
 
 include_binaries = (pkg not in ('debian-installer')
                     and not pkg.startswith('linux-signed'))
-
-# Grab a reference to the 'to' archive and select a pocket.
-to_archive = launchpad.archives.getByReference(reference=to)
-if to == 'ubuntu':
-    to_pocket = 'proposed'
-else:
-    to_pocket = 'release'
+if args.from_signing:
+    include_binaries = True
 
 print("""Copying {}/{}:
  From: {} release
- To: {} {}""".format(pkg, version, kernel_ppa, to_archive, to_pocket))
+ To: {} {}
+ Binaries: {}""".format(pkg, version, kernel_ppa, to_archive, to_pocket, include_binaries))
+
+if not version:
+    print("ERROR: no version to copy")
+    sys.exit(1)
 
 if args.dry_run:
     print("Dry run; no packages copied.")