Merge lp:~apw/launchpad/signing-reinstate-raw-uefi-custom-upload into lp:launchpad

[Andy Whitcroft]

Reinstate the raw-uefi custom upload as a distinct type.

Reinstate the raw-uefi custom upload as a distinct type. This utilises the same machinery as raw-signing. The only difference is the location in dists, with raw-uefi landing in dists/*/uefi and raw-signing landing in dists/*/signing.

[Colin Watson (cjwatson)]

review: Approve

[Andy Whitcroft (apw)]

Changes as requested pushed to the branch. Thanks for your reviews.

=== modified file 'lib/lp/archivepublisher/configure.zcml'
--- lib/lp/archivepublisher/configure.zcml	2016-05-23 10:46:55 +0000
+++ lib/lp/archivepublisher/configure.zcml	2016-05-31 12:46:39 +0000
@@ -88,6 +88,13 @@
             interface="lp.soyuz.interfaces.queue.ICustomUploadHandler"/>
     </securedutility>
     <securedutility
+        class="lp.archivepublisher.signing.UefiUpload"
+        provides="lp.soyuz.interfaces.queue.ICustomUploadHandler"
+        name="UEFI">
+        <allow
+            interface="lp.soyuz.interfaces.queue.ICustomUploadHandler"/>
+    </securedutility>
+    <securedutility
         class="lp.archivepublisher.signing.SigningUpload"
         provides="lp.soyuz.interfaces.queue.ICustomUploadHandler"
         name="SIGNING">
 
=== modified file 'lib/lp/archivepublisher/signing.py'
--- lib/lp/archivepublisher/signing.py	2016-05-26 10:46:04 +0000
+++ lib/lp/archivepublisher/signing.py	2016-05-31 12:46:39 +0000
@@ -15,6 +15,7 @@
 
 __all__ = [
     "SigningUpload",
+    "UefiUpload",
     ]
 
 import os
@@ -63,6 +64,8 @@
     """
     custom_type = "signing"
 
+    dists_directory = "signed"
+
     @staticmethod
     def parsePath(tarfile_path):
         tarfile_base = os.path.basename(tarfile_path)
@@ -94,8 +97,8 @@
 
         self.setComponents(tarfile_path)
 
-        dists_signed = os.path.join(
-            pubconf.archiveroot, "dists", suite, "main", "signed")
+        dists_signed = os.path.join(pubconf.archiveroot, "dists",
+            suite, "main", self.dists_directory)
         self.targetdir = os.path.join(
             dists_signed, "%s-%s" % (self.package, self.arch))
         self.archiveroot = pubconf.archiveroot
@@ -302,3 +305,20 @@
 
     def shouldInstall(self, filename):
         return filename.startswith("%s/" % self.version)
+
+
+class UefiUpload(SigningUpload):
+    """Legacy UEFI Signing custom upload.
+
+    Provides backwards compatibility UEFI signing uploads. Existing
+    packages use the raw-uefi custom upload and expect the results
+    to be published to dists/*/uefi.  These are a functional subset of
+    raw-signing custom uploads differing only in where they are published
+    in the archive.
+
+    We expect to be able to remove this upload type once all existing
+    packages are converted to the new form and location.
+    """
+    custom_type = "uefi"
+
+    dists_directory = "uefi"
 
=== modified file 'lib/lp/archivepublisher/tests/test_signing.py'
--- lib/lp/archivepublisher/tests/test_signing.py	2016-05-26 10:46:39 +0000
+++ lib/lp/archivepublisher/tests/test_signing.py	2016-05-31 12:46:39 +0000
@@ -15,7 +15,10 @@
     CustomUploadAlreadyExists,
     CustomUploadBadUmask,
     )
-from lp.archivepublisher.signing import SigningUpload
+from lp.archivepublisher.signing import (
+    SigningUpload,
+    UefiUpload,
+    )
 from lp.services.osutils import write_file
 from lp.services.tarfile_helpers import LaunchpadWriteTarFile
 from lp.testing import TestCase
@@ -98,10 +101,10 @@
         self.signingautokey = True
 
 
-class TestSigning(TestCase):
+class TestSigningHelpers(TestCase):
 
     def setUp(self):
-        super(TestSigning, self).setUp()
+        super(TestSigningHelpers, self).setUp()
         self.temp_dir = self.makeTemporaryDirectory()
         self.signing_dir = self.makeTemporaryDirectory()
         self.pubconf = FakeConfigPrimary(self.temp_dir, self.signing_dir)
@@ -135,43 +138,42 @@
         self.buffer = open(self.path, "wb")
         self.archive = LaunchpadWriteTarFile(self.buffer)
 
-    def process_emulate(self):
-        self.archive.close()
-        self.buffer.close()
-        upload = SigningUpload()
-        # Under no circumstances is it safe to execute actual commands.
-        self.fake_call = FakeMethod(result=0)
-        upload.callLog = FakeMethodCallLog(upload=upload)
-        self.useFixture(MonkeyPatch("subprocess.call", self.fake_call))
-        upload.process(self.pubconf, self.path, self.suite)
-
-        return upload
-
-    def process(self):
-        self.archive.close()
-        self.buffer.close()
-        upload = SigningUpload()
-        upload.signUefi = FakeMethod()
-        upload.signKmod = FakeMethod()
-        # Under no circumstances is it safe to execute actual commands.
-        fake_call = FakeMethod(result=0)
-        self.useFixture(MonkeyPatch("subprocess.call", fake_call))
-        upload.process(self.pubconf, self.path, self.suite)
-        self.assertEqual(0, fake_call.call_count)
-
-        return upload
-
     def getDistsPath(self):
         return os.path.join(self.pubconf.archiveroot, "dists",
             self.suite, "main")
 
+
+class TestSigning(TestSigningHelpers):
+
     def getSignedPath(self, loader_type, arch):
         return os.path.join(self.getDistsPath(), "signed",
             "%s-%s" % (loader_type, arch))
 
-    def getUefiPath(self, loader_type, arch):
-        return os.path.join(self.getDistsPath(), "uefi",
-            "%s-%s" % (loader_type, arch))
+    def process_emulate(self):
+        self.archive.close()
+        self.buffer.close()
+        upload = SigningUpload()
+        # Under no circumstances is it safe to execute actual commands.
+        self.fake_call = FakeMethod(result=0)
+        upload.callLog = FakeMethodCallLog(upload=upload)
+        self.useFixture(MonkeyPatch("subprocess.call", self.fake_call))
+        upload.process(self.pubconf, self.path, self.suite)
+
+        return upload
+
+    def process(self):
+        self.archive.close()
+        self.buffer.close()
+        upload = SigningUpload()
+        upload.signUefi = FakeMethod()
+        upload.signKmod = FakeMethod()
+        # Under no circumstances is it safe to execute actual commands.
+        fake_call = FakeMethod(result=0)
+        self.useFixture(MonkeyPatch("subprocess.call", fake_call))
+        upload.process(self.pubconf, self.path, self.suite)
+        self.assertEqual(0, fake_call.call_count)
+
+        return upload
 
     def test_archive_copy(self):
         # If there is no key/cert configuration, processing succeeds but
@@ -599,3 +601,35 @@
         self.assertTrue(os.path.exists(self.kmod_x509))
         self.assertEqual(stat.S_IMODE(os.stat(self.kmod_pem).st_mode), 0o600)
         self.assertEqual(stat.S_IMODE(os.stat(self.kmod_x509).st_mode), 0o644)
+
+
+class TestUefi(TestSigningHelpers):
+
+    def getSignedPath(self, loader_type, arch):
+        return os.path.join(self.getDistsPath(), "uefi",
+            "%s-%s" % (loader_type, arch))
+
+    def process(self):
+        self.archive.close()
+        self.buffer.close()
+        upload = UefiUpload()
+        upload.signUefi = FakeMethod()
+        upload.signKmod = FakeMethod()
+        # Under no circumstances is it safe to execute actual commands.
+        fake_call = FakeMethod(result=0)
+        self.useFixture(MonkeyPatch("subprocess.call", fake_call))
+        upload.process(self.pubconf, self.path, self.suite)
+        self.assertEqual(0, fake_call.call_count)
+
+        return upload
+
+    def test_installed(self):
+        # Files in the tarball are installed correctly.
+        self.setUpUefiKeys()
+        self.openArchive("test", "1.0", "amd64")
+        self.archive.add_file("1.0/empty.efi", "")
+        self.process()
+        self.assertTrue(os.path.isdir(os.path.join(
+            self.getDistsPath(), "uefi")))
+        self.assertTrue(os.path.exists(os.path.join(
+            self.getSignedPath("test", "amd64"), "1.0", "empty.efi")))
 
=== modified file 'lib/lp/archiveuploader/nascentuploadfile.py'
--- lib/lp/archiveuploader/nascentuploadfile.py	2016-05-06 09:16:30 +0000
+++ lib/lp/archiveuploader/nascentuploadfile.py	2016-05-31 12:46:39 +0000
@@ -33,7 +33,10 @@
 from lp.archivepublisher.debian_installer import DebianInstallerUpload
 from lp.archivepublisher.dist_upgrader import DistUpgraderUpload
 from lp.archivepublisher.rosetta_translations import RosettaTranslationsUpload
-from lp.archivepublisher.signing import SigningUpload
+from lp.archivepublisher.signing import (
+    SigningUpload,
+    UefiUpload,
+    )
 from lp.archiveuploader.utils import (
     determine_source_file_type,
     prefix_multi_line_string,
@@ -258,8 +261,8 @@
             PackageUploadCustomFormat.STATIC_TRANSLATIONS,
         'raw-meta-data':
             PackageUploadCustomFormat.META_DATA,
+        'raw-uefi': PackageUploadCustomFormat.UEFI,
         'raw-signing': PackageUploadCustomFormat.SIGNING,
-        'raw-uefi': PackageUploadCustomFormat.SIGNING,  # Compatibility
         }
 
     custom_handlers = {
@@ -268,6 +271,7 @@
         PackageUploadCustomFormat.DDTP_TARBALL: DdtpTarballUpload,
         PackageUploadCustomFormat.ROSETTA_TRANSLATIONS:
             RosettaTranslationsUpload,
+        PackageUploadCustomFormat.UEFI: UefiUpload,
         PackageUploadCustomFormat.SIGNING: SigningUpload,
         }
 
@@ -309,7 +313,8 @@
         """Return whether this custom upload can be automatically approved."""
         # Signing uploads will be signed, and must therefore be approved
         # by a human.
-        if self.custom_type == PackageUploadCustomFormat.SIGNING:
+        if self.custom_type in (PackageUploadCustomFormat.UEFI,
+                                PackageUploadCustomFormat.SIGNING):
             return False
         return True
 
 
=== modified file 'lib/lp/soyuz/browser/queue.py'
--- lib/lp/soyuz/browser/queue.py	2016-05-09 18:07:49 +0000
+++ lib/lp/soyuz/browser/queue.py	2016-05-31 12:46:39 +0000
@@ -574,6 +574,7 @@
             (self.contains_installer, ("Installer", 'ubuntu-icon')),
             (self.contains_upgrader, ("Upgrader", 'ubuntu-icon')),
             (self.contains_ddtp, (ddtp, 'ubuntu-icon')),
+            (self.contains_uefi, ("UEFI Objects for Signing", 'ubuntu-icon')),
             (self.contains_signing, ("Objects for Signing", 'ubuntu-icon')),
             ]
         return [
 
=== modified file 'lib/lp/soyuz/enums.py'
--- lib/lp/soyuz/enums.py	2016-05-03 14:44:33 +0000
+++ lib/lp/soyuz/enums.py	2016-05-31 12:46:39 +0000
@@ -481,7 +481,13 @@
         the Software Center.
         """)
 
-    SIGNING = DBItem(6, """
+    UEFI = DBItem(6, """
+        uefi
+
+        A tarball containing EFI images to be signed.
+        """)
+
+    SIGNING = DBItem(7, """
         signing
 
         A tarball containing images to be signed.
 
=== modified file 'lib/lp/soyuz/model/queue.py'
--- lib/lp/soyuz/model/queue.py	2016-05-23 11:18:16 +0000
+++ lib/lp/soyuz/model/queue.py	2016-05-31 12:46:39 +0000
@@ -654,12 +654,15 @@
         return PackageUploadCustomFormat.DDTP_TARBALL in self._customFormats
 
     @cachedproperty
+    def contains_uefi(self):
+        """See `IPackageUpload`."""
+        return PackageUploadCustomFormat.UEFI in self._customFormats
+
+    @cachedproperty
     def contains_signing(self):
         """See `IPackageUpload`."""
         return PackageUploadCustomFormat.SIGNING in self._customFormats
 
-    contains_uefi = contains_signing
-
     @property
     def package_name(self):
         """See `IPackageUpload`."""
 
=== modified file 'lib/lp/soyuz/scripts/custom_uploads_copier.py'
--- lib/lp/soyuz/scripts/custom_uploads_copier.py	2016-05-06 09:16:30 +0000
+++ lib/lp/soyuz/scripts/custom_uploads_copier.py	2016-05-31 12:46:39 +0000
@@ -18,7 +18,10 @@
 from lp.archivepublisher.debian_installer import DebianInstallerUpload
 from lp.archivepublisher.dist_upgrader import DistUpgraderUpload
 from lp.archivepublisher.rosetta_translations import RosettaTranslationsUpload
-from lp.archivepublisher.signing import SigningUpload
+from lp.archivepublisher.signing import (
+    SigningUpload,
+    UefiUpload,
+    )
 from lp.registry.interfaces.pocket import PackagePublishingPocket
 from lp.services.database.bulk import load_referencing
 from lp.soyuz.enums import PackageUploadCustomFormat
@@ -40,6 +43,7 @@
         PackageUploadCustomFormat.DDTP_TARBALL: DdtpTarballUpload,
         PackageUploadCustomFormat.ROSETTA_TRANSLATIONS:
             RosettaTranslationsUpload,
+        PackageUploadCustomFormat.UEFI: UefiUpload,
         PackageUploadCustomFormat.SIGNING: SigningUpload,
         }
 
@@ -63,7 +67,8 @@
             return True
         # Signing uploads will be signed, and must therefore be approved
         # by a human.
-        if custom.customformat == PackageUploadCustomFormat.SIGNING:
+        if custom.customformat in (PackageUploadCustomFormat.UEFI,
+                                   PackageUploadCustomFormat.SIGNING):
             return False
         return True
 
 
=== modified file 'lib/lp/soyuz/scripts/tests/test_custom_uploads_copier.py'
--- lib/lp/soyuz/scripts/tests/test_custom_uploads_copier.py	2016-05-06 09:16:30 +0000
+++ lib/lp/soyuz/scripts/tests/test_custom_uploads_copier.py	2016-05-31 12:46:39 +0000
@@ -409,7 +409,7 @@
         self.assertEqual(PackageUploadStatus.ACCEPTED, copied_pu.status)
 
     def test_copyUpload_unapproves_signing_from_different_archive(self):
-        # Copies of UEFI custom uploads to a primary archive are set to
+        # Copies of signing custom uploads to a primary archive are set to
         # UNAPPROVED, since they will normally end up being signed.
         target_series = self.factory.makeDistroSeries()
         archive = self.factory.makeArchive(
@@ -422,22 +422,60 @@
         self.assertEqual(PackageUploadStatus.UNAPPROVED, copied_pu.status)
 
     def test_copyUpload_approves_signing_from_same_archive(self):
+        # Copies of signing custom uploads within the same archive are
+        # automatically accepted, since they have already been signed.
+        original_upload = self.makeUpload(
+            custom_type=PackageUploadCustomFormat.SIGNING)
+        target_series = self.factory.makeDistroSeries()
+        copier = CustomUploadsCopier(target_series)
+        copied_pu = copier.copyUpload(original_upload).packageupload
+        self.assertEqual(PackageUploadStatus.ACCEPTED, copied_pu.status)
+
+    def test_copyUpload_approves_signing_to_ppa(self):
+        # Copies of signing custom uploads to a PPA are automatically accepted,
+        # since PPAs have much more limited upload permissions than the main
+        # archive, and in any case PPAs do not have an upload approval
+        # workflow.
+        original_upload = self.makeUpload(
+            custom_type=PackageUploadCustomFormat.SIGNING)
+        target_series = self.factory.makeDistroSeries()
+        target_archive = self.factory.makeArchive(
+            distribution=target_series.distribution)
+        copier = CustomUploadsCopier(
+            target_series, target_archive=target_archive)
+        copied_pu = copier.copyUpload(original_upload).packageupload
+        self.assertEqual(PackageUploadStatus.ACCEPTED, copied_pu.status)
+
+    def test_copyUpload_unapproves_uefi_from_different_archive(self):
+        # Copies of UEFI custom uploads to a primary archive are set to
+        # UNAPPROVED, since they will normally end up being signed.
+        target_series = self.factory.makeDistroSeries()
+        archive = self.factory.makeArchive(
+            distribution=target_series.distribution)
+        original_upload = self.makeUpload(
+            archive=archive, custom_type=PackageUploadCustomFormat.UEFI)
+        copier = CustomUploadsCopier(
+            target_series, target_archive=target_series.main_archive)
+        copied_pu = copier.copyUpload(original_upload).packageupload
+        self.assertEqual(PackageUploadStatus.UNAPPROVED, copied_pu.status)
+
+    def test_copyUpload_approves_uefi_from_same_archive(self):
         # Copies of UEFI custom uploads within the same archive are
         # automatically accepted, since they have already been signed.
         original_upload = self.makeUpload(
-            custom_type=PackageUploadCustomFormat.SIGNING)
+            custom_type=PackageUploadCustomFormat.UEFI)
         target_series = self.factory.makeDistroSeries()
         copier = CustomUploadsCopier(target_series)
         copied_pu = copier.copyUpload(original_upload).packageupload
         self.assertEqual(PackageUploadStatus.ACCEPTED, copied_pu.status)
 
-    def test_copyUpload_approves_signing_to_ppa(self):
+    def test_copyUpload_approves_uefi_to_ppa(self):
         # Copies of UEFI custom uploads to a PPA are automatically accepted,
         # since PPAs have much more limited upload permissions than the main
         # archive, and in any case PPAs do not have an upload approval
         # workflow.
         original_upload = self.makeUpload(
-            custom_type=PackageUploadCustomFormat.SIGNING)
+            custom_type=PackageUploadCustomFormat.UEFI)
         target_series = self.factory.makeDistroSeries()
         target_archive = self.factory.makeArchive(
             distribution=target_series.distribution)