Merge lp:~cjwatson/launchpad/publish-distro-careful-release into lp:launchpad

[Colin Watson]

Add publish-distro options allowing us to just rewrite Release files in all PPAs.

Add publish-distro options allowing us to just rewrite Release files in all PPAs. This will let us deal with situations like bug 1558331 where we need to upgrade signatures on everything, without having to incur the cost of a full republication.

I don't have figures on how long a careful-Release run over all PPAs will take, but step D is usually quick compared to the rest of it.

[William Grant (wgrant)]

review: Approve (code)

=== modified file 'lib/lp/archivepublisher/publishing.py'
--- lib/lp/archivepublisher/publishing.py	2016-03-11 11:45:56 +0000
+++ lib/lp/archivepublisher/publishing.py	2016-03-17 17:14:24 +0000
@@ -574,6 +574,20 @@
                                        (distroseries.name, pocket.name))
                         continue
                     self.checkDirtySuiteBeforePublishing(distroseries, pocket)
+                else:
+                    if not self.isAllowed(distroseries, pocket):
+                        continue
+                    # If we were asked for careful Release file generation
+                    # but not careful indexing, then we may not have the raw
+                    # material needed to generate Release files for all
+                    # suites.  Only force those suites that already have
+                    # Release files.
+                    release_path = os.path.join(
+                        self._config.distsroot, distroseries.getSuite(pocket),
+                        "Release")
+                    if file_exists(release_path):
+                        self.release_files_needed.add(
+                            (distroseries.name, pocket))
                 self._writeSuite(distroseries, pocket)
 
     def _allIndexFiles(self, distroseries):
 
=== modified file 'lib/lp/archivepublisher/scripts/publishdistro.py'
--- lib/lp/archivepublisher/scripts/publishdistro.py	2015-09-23 13:19:25 +0000
+++ lib/lp/archivepublisher/scripts/publishdistro.py	2016-03-17 17:14:24 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2015 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2016 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Publisher script class."""
@@ -67,6 +67,18 @@
             help="Make index generation (e.g. apt-ftparchive) careful.")
 
         self.parser.add_option(
+            "--careful-release", action="store_true", dest="careful_release",
+            default=False,
+            help="Make the Release file generation process careful.")
+
+        self.parser.add_option(
+            "--include-non-pending", action="store_true",
+            dest="include_non_pending", default=False,
+            help=(
+                "When publishing PPAs, also include those that do not have "
+                "pending publications."))
+
+        self.parser.add_option(
             '-s', '--suite', metavar='SUITE', dest='suite', action='append',
             type='string', default=[], help='The suite to publish')
 
@@ -153,6 +165,7 @@
             ('Publishing', self.options.careful_publishing),
             ('Domination', self.options.careful_domination),
             (indexing_engine, self.options.careful_apt),
+            ('Release', self.options.careful_release),
             ]
         for name, option in log_items:
             self.logOption(name, self.describeCare(option))
@@ -207,7 +220,8 @@
 
     def getPPAs(self, distribution):
         """Find private package archives for the selected distribution."""
-        if self.isCareful(self.options.careful_publishing):
+        if (self.isCareful(self.options.careful_publishing) or
+                self.options.include_non_pending):
             return distribution.getAllPPAs()
         else:
             return distribution.getPendingPublicationPPAs()
@@ -274,7 +288,8 @@
             publisher.C_writeIndexes(careful_indexing)
         self.txn.commit()
 
-        publisher.D_writeReleaseFiles(careful_indexing)
+        publisher.D_writeReleaseFiles(self.isCareful(
+            self.options.careful_apt or self.options.careful_release))
         # The caller will commit this last step.
 
         publisher.createSeriesAliases()
 
=== modified file 'lib/lp/archivepublisher/tests/test_publishdistro.py'
--- lib/lp/archivepublisher/tests/test_publishdistro.py	2016-03-14 19:40:14 +0000
+++ lib/lp/archivepublisher/tests/test_publishdistro.py	2016-03-17 17:14:24 +0000
@@ -11,6 +11,10 @@
 import subprocess
 import sys
 
+from testtools.matchers import (
+    Not,
+    PathExists,
+    )
 from zope.component import getUtility
 from zope.security.proxy import removeSecurityProxy
 
@@ -394,6 +398,44 @@
             self.config.distsroot)
         self.assertNotExists(index_path)
 
+    def testCarefulRelease(self):
+        """publish-distro can be asked to just rewrite Release files."""
+        archive = self.factory.makeArchive(distribution=self.ubuntutest)
+        pub_source = self.getPubSource(filecontent='foo', archive=archive)
+
+        self.setUpRequireSigningKeys()
+        tac = KeyServerTac()
+        tac.setUp()
+        self.addCleanup(tac.tearDown)
+        key_path = os.path.join(gpgkeysdir, '[e-mail]')
+        IArchiveSigningKey(archive).setSigningKey(key_path)
+
+        self.layer.txn.commit()
+
+        self.runPublishDistro(['--ppa'])
+
+        pub_source.sync()
+        self.assertEqual(PackagePublishingStatus.PUBLISHED, pub_source.status)
+
+        dists_path = getPubConfig(archive).distsroot
+        hoary_inrelease_path = os.path.join(
+            dists_path, 'hoary-test', 'InRelease')
+        breezy_inrelease_path = os.path.join(
+            dists_path, 'breezy-autotest', 'InRelease')
+        self.assertThat(hoary_inrelease_path, Not(PathExists()))
+        os.unlink(breezy_inrelease_path)
+
+        self.runPublishDistro(['--ppa', '--careful-release'])
+        self.assertThat(hoary_inrelease_path, Not(PathExists()))
+        self.assertThat(breezy_inrelease_path, Not(PathExists()))
+
+        self.runPublishDistro(
+            ['--ppa', '--careful-release', '--include-non-pending'])
+        # hoary-test never had indexes created, so is untouched.
+        self.assertThat(hoary_inrelease_path, Not(PathExists()))
+        # breezy-autotest has its Release files rewritten.
+        self.assertThat(breezy_inrelease_path, PathExists())
+
 
 class TestPublishDistroWithGPGService(TestPublishDistro):
     """A copy of the TestPublishDistro tests, but with the gpgservice feature
@@ -703,7 +745,7 @@
 
     def test_getPPAs_gets_pending_distro_PPAs_if_careful(self):
         # In careful mode, getPPAs includes PPAs for the distribution
-        # that are pending pulication.
+        # that are pending publication.
         distro = self.makeDistro()
         script = self.makeScript(distro, ['--careful'])
         ppa = self.factory.makeArchive(distro, purpose=ArchivePurpose.PPA)
@@ -712,15 +754,23 @@
 
     def test_getPPAs_gets_nonpending_distro_PPAs_if_careful(self):
         # In careful mode, getPPAs includes PPAs for the distribution
-        # that are not pending pulication.
+        # that are not pending publication.
         distro = self.makeDistro()
         script = self.makeScript(distro, ['--careful'])
         ppa = self.factory.makeArchive(distro, purpose=ArchivePurpose.PPA)
         self.assertContentEqual([ppa], script.getPPAs(distro))
 
+    def test_getPPAs_gets_nonpending_distro_PPAs_if_requested(self):
+        # In --include-non-pending mode, getPPAs includes PPAs for the
+        # distribution that are not pending publication.
+        distro = self.makeDistro()
+        script = self.makeScript(distro, ['--include-non-pending'])
+        ppa = self.factory.makeArchive(distro, purpose=ArchivePurpose.PPA)
+        self.assertContentEqual([ppa], script.getPPAs(distro))
+
     def test_getPPAs_gets_pending_distro_PPAs_if_not_careful(self):
         # In non-careful mode, getPPAs includes PPAs that are pending
-        # pulication.
+        # publication.
         distro = self.makeDistro()
         script = self.makeScript(distro)
         ppa = self.factory.makeArchive(distro, purpose=ArchivePurpose.PPA)
@@ -729,7 +779,7 @@
 
     def test_getPPAs_ignores_nonpending_distro_PPAs_if_not_careful(self):
         # In non-careful mode, getPPAs does not include PPAs that are
-        # not pending pulication.
+        # not pending publication.
         distro = self.makeDistro()
         script = self.makeScript(distro)
         self.factory.makeArchive(distro, purpose=ArchivePurpose.PPA)