Merge ~cjwatson/launchpad:generate-copy-archive-signing-keys into launchpad:master

[Colin Watson]

Extend ppa-generate-keys to generate keys for copy archives too

It's currently awkward to run builds that use the output of test rebuilds, since they're only signed if we go through some manual steps to arrange that. Since the copy archive publisher has access to the signing service, having it generate signing keys is straightforward enough, so let's just do that.

This requires a separate invocation with the `--copy-archives` option (mainly since copy archives are published on a different machine so the signing keys need to be authorized for use by that machine).

Once we have this running automatically on production, we should change `Archive.can_be_published` to have a guard for copy archives similar to the one it has for PPAs: it should only publish production copy archives that have had a signing key generated for them.

[Ioana Lasc (ilasc)]

review: Approve

diff --git a/lib/lp/archivepublisher/archivegpgsigningkey.py b/lib/lp/archivepublisher/archivegpgsigningkey.py
index c32bd9a..c1ada20 100644
--- a/lib/lp/archivepublisher/archivegpgsigningkey.py
+++ b/lib/lp/archivepublisher/archivegpgsigningkey.py
@@ -251,7 +251,9 @@ class ArchiveGPGSigningKey(SignableArchive):
         # Always generate signing keys for the default PPA, even if it
         # was not specifically requested. The default PPA signing key
         # is then propagated to the context named-ppa.
-        default_ppa = self.archive.owner.archive
+        default_ppa = (
+            self.archive.owner.archive if self.archive.is_ppa
+            else self.archive)
         if self.archive != default_ppa:
             def propagate_key(_):
                 self.archive.signing_key_owner = default_ppa.signing_key_owner
@@ -274,8 +276,15 @@ class ArchiveGPGSigningKey(SignableArchive):
                 propagate_key(None)
                 return
 
-        key_displayname = (
-            "Launchpad PPA for %s" % self.archive.owner.displayname)
+        # XXX cjwatson 2021-12-17: If we need key generation for other
+        # archive purposes (PRIMARY/PARTNER) then we should extend this, and
+        # perhaps push it down to a property of the archive.
+        if self.archive.is_copy:
+            key_displayname = (
+                "Launchpad copy archive %s" % self.archive.reference)
+        else:
+            key_displayname = (
+                "Launchpad PPA for %s" % self.archive.owner.displayname)
         if getFeatureFlag(PUBLISHER_GPG_USES_SIGNING_SERVICE):
             try:
                 signing_key = getUtility(ISigningKeySet).generate(
diff --git a/lib/lp/archivepublisher/tests/archive-signing.txt b/lib/lp/archivepublisher/tests/archive-signing.txt
index f142a01..ced9aa5 100644
--- a/lib/lp/archivepublisher/tests/archive-signing.txt
+++ b/lib/lp/archivepublisher/tests/archive-signing.txt
@@ -31,7 +31,7 @@ We will set up and use the test-keyserver.
 Querying 'pending signing key' PPAs
 -----------------------------------
 
-`IArchiveSet.getPPAsPendingSigningKey` allows call-sites to query for
+`IArchiveSet.getArchivesPendingSigningKey` allows call-sites to query for
 PPA pending signing key generation.
 
     >>> from lp.registry.interfaces.person import IPersonSet
@@ -40,14 +40,14 @@ PPA pending signing key generation.
 Only PPAs with at least one source publication are considered.
 
     >>> archive_set = getUtility(IArchiveSet)
-    >>> for ppa in archive_set.getPPAsPendingSigningKey():
+    >>> for ppa in archive_set.getArchivesPendingSigningKey():
     ...     print(ppa.displayname)
     PPA for Celso Providelo
     PPA for Mark Shuttleworth
 
 The PPA for 'No Privileges' user exists, is enabled and has no
 signing, but it also does not contain any source publications, that's
-why it's skipped in the getPPAsPendingSigningKey() results.
+why it's skipped in the getArchivesPendingSigningKey() results.
 
     >>> cprov = getUtility(IPersonSet).getByName('cprov')
     >>> no_priv = getUtility(IPersonSet).getByName('no-priv')
@@ -71,7 +71,7 @@ we copy an arbitrary source into it.
     >>> copied_sources = a_source.copyTo(
     ...     a_source.distroseries, a_source.pocket, no_priv.archive)
 
-    >>> for ppa in archive_set.getPPAsPendingSigningKey():
+    >>> for ppa in archive_set.getArchivesPendingSigningKey():
     ...     print(ppa.displayname)
     PPA for Celso Providelo
     PPA for Mark Shuttleworth
@@ -81,7 +81,7 @@ Disabled PPAs are excluded from the 'PendingSigningKey' pool:
 
     >>> no_priv.archive.disable()
 
-    >>> for ppa in archive_set.getPPAsPendingSigningKey():
+    >>> for ppa in archive_set.getArchivesPendingSigningKey():
     ...     print(ppa.displayname)
     PPA for Celso Providelo
     PPA for Mark Shuttleworth
@@ -111,10 +111,34 @@ And use it as the Mark's PPA signing key.
 
 It will exclude Mark's PPA from the 'PendingSigningKey' pool as well.
 
-    >>> for ppa in archive_set.getPPAsPendingSigningKey():
+    >>> for ppa in archive_set.getArchivesPendingSigningKey():
     ...     print(ppa.displayname)
     PPA for Celso Providelo
 
+We can also query for copy archives.
+
+    >>> from lp.soyuz.enums import ArchivePurpose
+    >>> rebuild_archive = factory.makeArchive(
+    ...     distribution=cprov.archive.distribution, name='test-rebuild',
+    ...     displayname='Test rebuild', purpose=ArchivePurpose.COPY)
+    >>> _ = a_source.copyTo(
+    ...     a_source.distroseries, a_source.pocket, rebuild_archive)
+    >>> for archive in archive_set.getArchivesPendingSigningKey(
+    ...         purpose=ArchivePurpose.COPY):
+    ...     print(archive.displayname)
+    Test rebuild
+
+Set up a signing key for the new test rebuild archive, and after that it no
+longer shows up as pending signing key generation.
+
+    >>> rebuild_archive.signing_key_owner = a_key.owner
+    >>> rebuild_archive.signing_key_fingerprint = a_key.fingerprint
+    >>> for archive in archive_set.getArchivesPendingSigningKey(
+    ...         purpose=ArchivePurpose.COPY):
+    ...     print(archive.displayname)
+    >>> rebuild_archive.signing_key_owner = None
+    >>> rebuild_archive.signing_key_fingerprint = None
+
 
 Generating a PPA signing key
 ----------------------------
@@ -270,7 +294,6 @@ As documented in archive.txt, when a named-ppa is created it is
 already configured to used the same signing-key created for the
 default PPA. We will create a named-ppa for Celso.
 
-    >>> from lp.soyuz.enums import ArchivePurpose
     >>> named_ppa = getUtility(IArchiveSet).new(
     ...     owner=cprov, purpose=ArchivePurpose.PPA, name='boing')
 
@@ -359,6 +382,11 @@ named after the user, even if the default PPA name is something different.
     >>> print(named_ppa.signing_key.fingerprint)
     447DBF38C4F9C4ED752246B77D88913717B05A8F
 
+Keys generated for copy archives use a different naming scheme.
+
+    >>> IArchiveGPGSigningKey(rebuild_archive).generateSigningKey()
+    Generating: Launchpad copy archive ubuntu/test-rebuild
+
 Restore the original functionality of GPGHandler.
 
     >>> naked_gpghandler.generateKey = real_key_generator
diff --git a/lib/lp/soyuz/interfaces/archive.py b/lib/lp/soyuz/interfaces/archive.py
index d01ebdf..a3de895 100644
--- a/lib/lp/soyuz/interfaces/archive.py
+++ b/lib/lp/soyuz/interfaces/archive.py
@@ -2416,10 +2416,12 @@ class IArchiveSet(Interface):
     def getPPADistributionsForUser(user):
         """Return the `Distribution`s of all PPAs for the given user."""
 
-    def getPPAsPendingSigningKey():
-        """Return all PPAs pending signing key generation.
+    def getArchivesPendingSigningKey(purpose=ArchivePurpose.PPA):
+        """Return all archives with `purpose` pending signing key generation.
 
         The result is ordered by archive creation date.
+
+        :param purpose: Only return archives with this `ArchivePurpose`.
         """
 
     def getLatestPPASourcePublicationsForDistribution(distribution):
diff --git a/lib/lp/soyuz/model/archive.py b/lib/lp/soyuz/model/archive.py
index ae95cf9..50ed7aa 100644
--- a/lib/lp/soyuz/model/archive.py
+++ b/lib/lp/soyuz/model/archive.py
@@ -2819,7 +2819,7 @@ class ArchiveSet:
             self._getPPAsForUserClause(user))
         return result.config(distinct=True)
 
-    def getPPAsPendingSigningKey(self):
+    def getArchivesPendingSigningKey(self, purpose=ArchivePurpose.PPA):
         """See `IArchiveSet`."""
         origin = (
             Archive,
@@ -2828,7 +2828,7 @@ class ArchiveSet:
         results = IStore(Archive).using(*origin).find(
             Archive,
             Archive.signing_key_fingerprint == None,
-            Archive.purpose == ArchivePurpose.PPA, Archive._enabled == True)
+            Archive.purpose == purpose, Archive._enabled == True)
         results.order_by(Archive.date_created)
         return results.config(distinct=True)
 
diff --git a/lib/lp/soyuz/scripts/ppakeygenerator.py b/lib/lp/soyuz/scripts/ppakeygenerator.py
index 88e1d84..1b650f1 100644
--- a/lib/lp/soyuz/scripts/ppakeygenerator.py
+++ b/lib/lp/soyuz/scripts/ppakeygenerator.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2019 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2021 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 __all__ = [
@@ -14,6 +14,7 @@ from lp.services.scripts.base import (
     LaunchpadCronScript,
     LaunchpadScriptFailure,
     )
+from lp.soyuz.enums import ArchivePurpose
 from lp.soyuz.interfaces.archive import IArchiveSet
 
 
@@ -26,6 +27,9 @@ class PPAKeyGenerator(LaunchpadCronScript):
         self.parser.add_option(
             "-A", "--archive",
             help="The reference of the archive whose key should be generated.")
+        self.parser.add_option(
+            "--copy-archives", action="store_true", default=False,
+            help="Run only over COPY archives.")
 
     def generateKey(self, archive):
         """Generate a signing key for the given archive."""
@@ -38,9 +42,9 @@ class PPAKeyGenerator(LaunchpadCronScript):
 
     def main(self):
         """Generate signing keys for the selected PPAs."""
+        archive_set = getUtility(IArchiveSet)
         if self.options.archive is not None:
-            archive = getUtility(IArchiveSet).getByReference(
-                self.options.archive)
+            archive = archive_set.getByReference(self.options.archive)
             if archive is None:
                 raise LaunchpadScriptFailure(
                     "No archive named '%s' could be found."
@@ -51,9 +55,11 @@ class PPAKeyGenerator(LaunchpadCronScript):
                     % (archive.reference, archive.displayname,
                        archive.signing_key_fingerprint))
             archives = [archive]
+        elif self.options.copy_archives:
+            archives = list(archive_set.getArchivesPendingSigningKey(
+                purpose=ArchivePurpose.COPY))
         else:
-            archive_set = getUtility(IArchiveSet)
-            archives = list(archive_set.getPPAsPendingSigningKey())
+            archives = list(archive_set.getArchivesPendingSigningKey())
 
         for archive in archives:
             self.generateKey(archive)
diff --git a/lib/lp/soyuz/scripts/tests/test_ppakeygenerator.py b/lib/lp/soyuz/scripts/tests/test_ppakeygenerator.py
index 182f64a..0ad8a7d 100644
--- a/lib/lp/soyuz/scripts/tests/test_ppakeygenerator.py
+++ b/lib/lp/soyuz/scripts/tests/test_ppakeygenerator.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2019 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2021 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """`PPAKeyGenerator` script class tests."""
@@ -10,14 +10,15 @@ from lp.registry.interfaces.gpg import IGPGKeySet
 from lp.registry.interfaces.person import IPersonSet
 from lp.services.propertycache import get_property_cache
 from lp.services.scripts.base import LaunchpadScriptFailure
+from lp.soyuz.enums import ArchivePurpose
 from lp.soyuz.interfaces.archive import IArchiveSet
 from lp.soyuz.scripts.ppakeygenerator import PPAKeyGenerator
-from lp.testing import TestCase
+from lp.testing import TestCaseWithFactory
 from lp.testing.faketransaction import FakeTransaction
 from lp.testing.layers import LaunchpadZopelessLayer
 
 
-class TestPPAKeyGenerator(TestCase):
+class TestPPAKeyGenerator(TestCaseWithFactory):
     layer = LaunchpadZopelessLayer
 
     def _fixArchiveForKeyGeneration(self, archive):
@@ -29,7 +30,8 @@ class TestPPAKeyGenerator(TestCase):
         ubuntutest = getUtility(IDistributionSet).getByName('ubuntutest')
         archive.distribution = ubuntutest
 
-    def _getKeyGenerator(self, archive_reference=None, txn=None):
+    def _getKeyGenerator(self, archive_reference=None, copy_archives=False,
+                         txn=None):
         """Return a `PPAKeyGenerator` instance.
 
         Monkey-patch the script object with a fake transaction manager
@@ -40,6 +42,8 @@ class TestPPAKeyGenerator(TestCase):
 
         if archive_reference is not None:
             test_args.extend(['-A', archive_reference])
+        if copy_archives:
+            test_args.append('--copy-archives')
 
         key_generator = PPAKeyGenerator(
             name='ppa-generate-keys', test_args=test_args)
@@ -109,7 +113,7 @@ class TestPPAKeyGenerator(TestCase):
         The 'signing_key' for all 'pending-signing-key' PPAs are generated
         and the transaction is committed once for each PPA.
         """
-        archives = list(getUtility(IArchiveSet).getPPAsPendingSigningKey())
+        archives = list(getUtility(IArchiveSet).getArchivesPendingSigningKey())
 
         for archive in archives:
             self._fixArchiveForKeyGeneration(archive)
@@ -123,3 +127,33 @@ class TestPPAKeyGenerator(TestCase):
             self.assertIsNotNone(archive.signing_key_fingerprint)
 
         self.assertEqual(txn.commit_count, len(archives))
+
+    def testGenerateKeyForAllCopyArchives(self):
+        """Signing key generation for all PPAs.
+
+        The 'signing_key' for all 'pending-signing-key' PPAs are generated
+        and the transaction is committed once for each PPA.
+        """
+        for _ in range(3):
+            rebuild = self.factory.makeArchive(
+                distribution=getUtility(IDistributionSet).getByName(
+                    'ubuntutest'),
+                purpose=ArchivePurpose.COPY)
+            self.factory.makeSourcePackagePublishingHistory(archive=rebuild)
+
+        archives = list(getUtility(IArchiveSet).getArchivesPendingSigningKey(
+            purpose=ArchivePurpose.COPY))
+        self.assertNotEqual([], archives)
+
+        for archive in archives:
+            self._fixArchiveForKeyGeneration(archive)
+            self.assertIsNone(archive.signing_key_fingerprint)
+
+        txn = FakeTransaction()
+        key_generator = self._getKeyGenerator(copy_archives=True, txn=txn)
+        key_generator.main()
+
+        for archive in archives:
+            self.assertIsNotNone(archive.signing_key_fingerprint)
+
+        self.assertEqual(txn.commit_count, len(archives))