Merge ~cjwatson/launchpad:initialize-distroseries-proposed into launchpad:master

[Colin Watson]

Fix source/target pocket handling in InitializeDistroSeries

Copy pockets other than RELEASE when using the cloner as well as when
using the copier; and copy packages from PROPOSED to PROPOSED rather
than to RELEASE, since (unlike UPDATES and SECURITY) the PROPOSED pocket
is modifiable in unreleased series.

LP: #1824966

[William Grant (wgrant)]

review: Approve (code)

diff --git a/lib/lp/soyuz/scripts/initialize_distroseries.py b/lib/lp/soyuz/scripts/initialize_distroseries.py
index 6a9753e..d266a84 100644
--- a/lib/lp/soyuz/scripts/initialize_distroseries.py
+++ b/lib/lp/soyuz/scripts/initialize_distroseries.py
@@ -9,6 +9,7 @@ __all__ = [
     'InitializeDistroSeries',
     ]
 
+from collections import OrderedDict
 from operator import methodcaller
 
 import transaction
@@ -54,13 +55,15 @@ class InitializationError(Exception):
     """Raised when there is an exception during the initialization process."""
 
 
-# Pockets to consider when initializing the derived series from its parent(s).
-INIT_POCKETS = [
-    PackagePublishingPocket.RELEASE,
-    PackagePublishingPocket.SECURITY,
-    PackagePublishingPocket.UPDATES,
-    PackagePublishingPocket.PROPOSED,
-    ]
+# Pockets to consider when initializing the derived series from its
+# parent(s), mapped to the pockets that they should be cloned/copied to.  We
+# need this because some pockets are unmodifiable in unreleased series.
+INIT_POCKETS = OrderedDict([
+    (PackagePublishingPocket.RELEASE, PackagePublishingPocket.RELEASE),
+    (PackagePublishingPocket.SECURITY, PackagePublishingPocket.RELEASE),
+    (PackagePublishingPocket.UPDATES, PackagePublishingPocket.RELEASE),
+    (PackagePublishingPocket.PROPOSED, PackagePublishingPocket.PROPOSED),
+    ])
 
 
 class InitializeDistroSeries:
@@ -233,7 +236,7 @@ class InitializeDistroSeries:
 
         arch_tags = self.arches if len(self.arches) != 0 else None
         pending_builds = parent.getBuildRecords(
-            BuildStatus.NEEDSBUILD, pocket=INIT_POCKETS,
+            BuildStatus.NEEDSBUILD, pocket=list(INIT_POCKETS),
             arch_tag=arch_tags, name=spns)
 
         if not pending_builds.is_empty():
@@ -263,7 +266,7 @@ class InitializeDistroSeries:
         # all sources.
 
         items = getUtility(IPackageUploadSet).getBuildsForSources(
-            parent, statuses, INIT_POCKETS, spns)
+            parent, statuses, list(INIT_POCKETS), spns)
         if not items.is_empty():
             raise InitializationError(
                 "The parent series has sources waiting in its upload "
@@ -548,40 +551,43 @@ class InitializeDistroSeries:
                     assert target_archive is not None, (
                         "Target archive doesn't exist?")
                 if self._use_cloner(target_archive, archive):
-                    origin = PackageLocation(
-                        archive, parent.distribution, parent,
-                        PackagePublishingPocket.RELEASE)
-                    destination = PackageLocation(
-                        target_archive, self.distroseries.distribution,
-                        self.distroseries, PackagePublishingPocket.RELEASE)
-                    processors = None
-                    if self.rebuild:
-                        processors = [
-                            das[1].processor for das in distroarchseries_list]
-                        distroarchseries_list = ()
-                    getUtility(IPackageCloner).clonePackages(
-                        origin, destination, distroarchseries_list,
-                        processors, spns)
+                    for source_pocket, target_pocket in INIT_POCKETS.items():
+                        origin = PackageLocation(
+                            archive, parent.distribution, parent,
+                            source_pocket)
+                        destination = PackageLocation(
+                            target_archive, self.distroseries.distribution,
+                            self.distroseries, target_pocket)
+                        processors = None
+                        if self.rebuild:
+                            processors = [
+                                das[1].processor
+                                for das in distroarchseries_list]
+                            distroarchseries_list = ()
+                        getUtility(IPackageCloner).clonePackages(
+                            origin, destination, distroarchseries_list,
+                            processors, spns)
                 else:
-                    # There is only one available pocket in an unreleased
-                    # series.
-                    target_pocket = PackagePublishingPocket.RELEASE
-                    sources = archive.getPublishedSources(
-                        distroseries=parent, pocket=INIT_POCKETS,
-                        status=(PackagePublishingStatus.PENDING,
-                                PackagePublishingStatus.PUBLISHED),
-                        name=spns)
                     # XXX: rvb 2011-06-23 bug=801112: do_copy is atomic (all
                     # or none of the sources will be copied). This might
                     # lead to a partially initialised series if there is a
                     # single conflict in the destination series.
+                    sources_published = []
                     try:
-                        sources_published = do_copy(
-                            sources, target_archive, self.distroseries,
-                            target_pocket, include_binaries=not self.rebuild,
-                            check_permissions=False, strict_binaries=False,
-                            close_bugs=False, create_dsd_job=False,
-                            person=None)
+                        for source_pocket, target_pocket in (
+                                INIT_POCKETS.items()):
+                            sources = archive.getPublishedSources(
+                                distroseries=parent, pocket=source_pocket,
+                                status=(PackagePublishingStatus.PENDING,
+                                        PackagePublishingStatus.PUBLISHED),
+                                name=spns)
+                            sources_published.extend(do_copy(
+                                sources, target_archive, self.distroseries,
+                                target_pocket,
+                                include_binaries=not self.rebuild,
+                                check_permissions=False, strict_binaries=False,
+                                close_bugs=False, create_dsd_job=False,
+                                person=None))
                         if self.rebuild:
                             rebuilds = []
                             for pubrec in sources_published:
diff --git a/lib/lp/soyuz/scripts/tests/test_initialize_distroseries.py b/lib/lp/soyuz/scripts/tests/test_initialize_distroseries.py
index bd2cf15..25d2cb3 100644
--- a/lib/lp/soyuz/scripts/tests/test_initialize_distroseries.py
+++ b/lib/lp/soyuz/scripts/tests/test_initialize_distroseries.py
@@ -374,22 +374,74 @@ class TestInitializeDistroSeries(InitializationHelperTestCase):
         # No exception should be raised.
         ids.check()
 
-    def test_success_with_updates_packages(self):
-        # Initialization copies all the package from the UPDATES pocket.
+    def test_success_with_updates_packages_cloner(self):
+        # Initialization using the cloner copies all the packages from the
+        # UPDATES pocket.
+        self.parent, self.parent_das = self.setupParent(
+            pocket=PackagePublishingPocket.UPDATES)
+        self.factory.makeSourcePackagePublishingHistory(
+            distroseries=self.parent)
+        child = self._fullInitialize(
+            [self.parent], previous_series=self.parent,
+            distribution=self.parent.distribution)
+        self.assertDistroSeriesInitializedCorrectly(
+            child, self.parent, self.parent_das)
+
+    def test_success_with_updates_packages_copier(self):
+        # Initialization using the copier copies all the packages from the
+        # UPDATES pocket.
         self.parent, self.parent_das = self.setupParent(
             pocket=PackagePublishingPocket.UPDATES)
         child = self._fullInitialize([self.parent])
         self.assertDistroSeriesInitializedCorrectly(
             child, self.parent, self.parent_das)
 
-    def test_success_with_security_packages(self):
-        # Initialization copies all the package from the SECURITY pocket.
+    def test_success_with_security_packages_cloner(self):
+        # Initialization using the cloner copies all the packages from the
+        # SECURITY pocket.
+        self.parent, self.parent_das = self.setupParent(
+            pocket=PackagePublishingPocket.SECURITY)
+        self.factory.makeSourcePackagePublishingHistory(
+            distroseries=self.parent)
+        child = self._fullInitialize(
+            [self.parent], previous_series=self.parent,
+            distribution=self.parent.distribution)
+        self.assertDistroSeriesInitializedCorrectly(
+            child, self.parent, self.parent_das)
+
+    def test_success_with_security_packages_copier(self):
+        # Initialization using the copier copies all the packages from the
+        # SECURITY pocket.
         self.parent, self.parent_das = self.setupParent(
             pocket=PackagePublishingPocket.SECURITY)
         child = self._fullInitialize([self.parent])
         self.assertDistroSeriesInitializedCorrectly(
             child, self.parent, self.parent_das)
 
+    def test_success_with_proposed_packages_cloner(self):
+        # Initialization using the cloner copies all the packages from the
+        # PROPOSED pocket.
+        self.parent, self.parent_das = self.setupParent(
+            pocket=PackagePublishingPocket.PROPOSED)
+        self.factory.makeSourcePackagePublishingHistory(
+            distroseries=self.parent)
+        child = self._fullInitialize(
+            [self.parent], previous_series=self.parent,
+            distribution=self.parent.distribution)
+        self.assertDistroSeriesInitializedCorrectly(
+            child, self.parent, self.parent_das,
+            child_pocket=PackagePublishingPocket.PROPOSED)
+
+    def test_success_with_proposed_packages_copier(self):
+        # Initialization using the copier copies all the packages from the
+        # PROPOSED pocket.
+        self.parent, self.parent_das = self.setupParent(
+            pocket=PackagePublishingPocket.PROPOSED)
+        child = self._fullInitialize([self.parent])
+        self.assertDistroSeriesInitializedCorrectly(
+            child, self.parent, self.parent_das,
+            child_pocket=PackagePublishingPocket.PROPOSED)
+
     def test_do_not_copy_superseded_sources(self):
         # Make sure we don't copy superseded sources from the parent,
         # we only want (pending, published).
@@ -578,8 +630,9 @@ class TestInitializeDistroSeries(InitializationHelperTestCase):
              "queues that match your selection."),
             ids.check)
 
-    def assertDistroSeriesInitializedCorrectly(self, child, parent,
-                                               parent_das):
+    def assertDistroSeriesInitializedCorrectly(
+            self, child, parent, parent_das,
+            child_pocket=PackagePublishingPocket.RELEASE):
         # Check that 'udev' has been copied correctly.
         parent_udev_pubs = parent.main_archive.getPublishedSources(
             u'udev', distroseries=parent)
@@ -587,12 +640,16 @@ class TestInitializeDistroSeries(InitializationHelperTestCase):
             u'udev', distroseries=child)
         self.assertEqual(
             parent_udev_pubs.count(), child_udev_pubs.count())
+        self.assertEqual(
+            {child_pocket}, set(pub.pocket for pub in child_udev_pubs))
         parent_arch_udev_pubs = parent.main_archive.getAllPublishedBinaries(
             distroarchseries=parent[parent_das.architecturetag], name=u'udev')
         child_arch_udev_pubs = child.main_archive.getAllPublishedBinaries(
             distroarchseries=child[parent_das.architecturetag], name=u'udev')
         self.assertEqual(
             parent_arch_udev_pubs.count(), child_arch_udev_pubs.count())
+        self.assertEqual(
+            {child_pocket}, set(pub.pocket for pub in child_arch_udev_pubs))
         # And the binary package, and linked source package look fine too.
         udev_bin = child_arch_udev_pubs[0].binarypackagerelease
         self.assertEqual(udev_bin.title, u'udev-0.1-1')