Merge lp:~michael.nelson/launchpad/copy-binaries-timeout into lp:launchpad

[Michael Nelson]

[Michael Nelson (michael.nelson)]

= Summary =

This branch addresses some of the causes of bug 447138.

Five options for improving the performance when copying a large number
of binaries are identified at:
https://bugs.edge.launchpad.net/soyuz/+bug/447138/comments/1

This branch addresses points (1) and (4).

== Proposed fix ==

Create IPublishingSet.copyBinariesTo() so that we only need to lookup
the 'main' component once (point 1 above) and only need one query to
obtain all the newly created BPPHs.

== Pre-implementation notes ==

See comment #1 on bug above.

== Implementation details ==

== Tests ==

bin/test -vv -t test_copypackage -t doc/pubishing.txt

== Demo and Q/A ==

I'll start Q/Aing this on dogfood now. In production we can ask the arch
admins to try including binaries when copying linux kernel package.

= Launchpad lint =

Checking for conflicts. and issues in doctests and templates.
Running jslint, xmllint, pyflakes, and pylint.
Using normal rules.

Linting changed files:
  lib/lp/soyuz/scripts/packagecopier.py
  lib/lp/soyuz/model/publishing.py
  lib/lp/soyuz/interfaces/publishing.py

== Pylint notices ==

lib/lp/soyuz/scripts/packagecopier.py
    27: [F0401] Unable to import 'lazr.delegates' (No module named
delegates)

lib/lp/soyuz/interfaces/publishing.py
    33: [F0401] Unable to import 'lazr.enum' (No module named enum)
    40: [F0401] Unable to import 'lazr.restful.fields' (No module named
restful)
    41: [F0401] Unable to import 'lazr.restful.declarations' (No module
named restful)

--
Michael

[Muharem Hrnjadovic (al-maisan)]

Looks good, needs more testing/corroboration on the Soyuz dogfood system prior to landing though.

review: Approve

=== modified file 'lib/lp/soyuz/interfaces/publishing.py'
--- lib/lp/soyuz/interfaces/publishing.py	2009-10-05 18:29:12 +0000
+++ lib/lp/soyuz/interfaces/publishing.py	2009-10-29 12:30:28 +0000
@@ -856,6 +856,23 @@
 class IPublishingSet(Interface):
     """Auxiliary methods for dealing with sets of publications."""
 
+    def copyBinariesTo(binaries, distroseries, pocket, archive):
+        """Copy multiple binaries to a given destination.
+
+        Processing multiple binaries in a batch allows certain
+        performance optimisations such as looking up the main
+        component once only, and getting all the BPPH records
+        with one query.
+
+        :param binaries: A list of binaries to copy.
+        :param distroseries: The target distroseries.
+        :param pocket: The target pocket.
+        :param archive: The target archive.
+
+        :return: A result set of the created binary package
+            publishing histories.
+        """
+
     def newBinaryPublication(archive, binarypackagerelease, distroarchseries,
                              component, section, priority, pocket):
         """Create a new `BinaryPackagePublishingHistory`.

=== modified file 'lib/lp/soyuz/model/publishing.py'
--- lib/lp/soyuz/model/publishing.py	2009-10-06 11:16:03 +0000
+++ lib/lp/soyuz/model/publishing.py	2009-10-29 12:30:28 +0000
@@ -1049,32 +1049,9 @@
 
     def copyTo(self, distroseries, pocket, archive):
         """See `BinaryPackagePublishingHistory`."""
-        current = self.secure_record
-
-        if current.binarypackagerelease.architecturespecific:
-            try:
-                target_architecture = distroseries[
-                    current.distroarchseries.architecturetag]
-            except NotFoundError:
-                return []
-            destination_architectures = [target_architecture]
-        else:
-            destination_architectures = distroseries.architectures
-
-        copies = []
-        for architecture in destination_architectures:
-            copy = getUtility(IPublishingSet).newBinaryPublication(
-                archive,
-                self.binarypackagerelease,
-                architecture,
-                current.component,
-                current.section,
-                current.priority,
-                pocket
-                )
-            copies.append(copy)
-
-        return copies
+
+        return getUtility(IPublishingSet).copyBinariesTo(
+            [self], distroseries, pocket, archive)
 
     def getAncestry(self, archive=None, distroseries=None, pocket=None,
                     status=None):
@@ -1120,6 +1097,72 @@
 
     implements(IPublishingSet)
 
+    def copyBinariesTo(self, binaries, distroseries, pocket, archive):
+        """See `IPublishingSet`."""
+
+        # If the target archive is a ppa then we will need to override
+        # the component for each copy - so lookup the main component
+        # here once.
+        override_component = None
+        if archive.is_ppa:
+            override_component = getUtility(IComponentSet)['main']
+
+        secure_copies = []
+
+        for binary in binaries:
+            binarypackagerelease = binary.binarypackagerelease
+            target_component = override_component or binary.component
+
+            if binarypackagerelease.architecturespecific:
+                # If the binary is architecture specific and the target
+                # distroseries does not include the architecture then we
+                # skip the binary and continue.
+                try:
+                    # For safety, we use the architecture the binary was
+                    # built, and not the one it is published, coping with
+                    # single arch-indep publications for architectures that
+                    # do not exist in the destination series.
+                    # See #387589 for more information.
+                    target_architecture = distroseries[
+                        binarypackagerelease.build.arch_tag]
+                except NotFoundError:
+                    continue
+                destination_architectures = [target_architecture]
+            else:
+                destination_architectures = distroseries.architectures
+
+            for distroarchseries in destination_architectures:
+
+                # We only copy the binary if it doesn't already exist
+                # in the destination.
+                binary_in_destination = archive.getAllPublishedBinaries(
+                    name=binarypackagerelease.name, exact_match=True,
+                    version=binarypackagerelease.version,
+                    status=active_publishing_status, pocket=pocket,
+                    distroarchseries=distroarchseries)
+
+                if binary_in_destination.count() == 0:
+                    pub = SecureBinaryPackagePublishingHistory(
+                        archive=archive,
+                        binarypackagerelease=binarypackagerelease,
+                        distroarchseries=distroarchseries,
+                        component=target_component,
+                        section=binary.section,
+                        priority=binary.priority,
+                        status=PackagePublishingStatus.PENDING,
+                        datecreated=UTC_NOW,
+                        pocket=pocket,
+                        embargo=False)
+                    secure_copies.append(pub)
+
+        # One day, this will not be necessary when we have time to kill
+        # the Secure* records.
+        copy_ids = [secure_copy.id for secure_copy in secure_copies]
+
+        store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
+        return store.find(BinaryPackagePublishingHistory,
+            BinaryPackagePublishingHistory.id.is_in(copy_ids))
+
     def newBinaryPublication(self, archive, binarypackagerelease,
                              distroarchseries, component, section, priority,
                              pocket):

=== modified file 'lib/lp/soyuz/scripts/packagecopier.py'
--- lib/lp/soyuz/scripts/packagecopier.py	2009-08-24 20:09:38 +0000
+++ lib/lp/soyuz/scripts/packagecopier.py	2009-10-29 12:30:28 +0000
@@ -22,7 +22,6 @@
 
 from zope.component import getUtility
 
-from canonical.launchpad.interfaces.launchpad import NotFoundError
 from canonical.launchpad.interfaces.librarian import ILibraryFileAliasSet
 from canonical.librarian.utils import copy_and_close
 from lazr.delegates import delegates
@@ -33,8 +32,8 @@
 from lp.soyuz.interfaces.build import (
     BuildStatus, BuildSetStatus)
 from lp.soyuz.interfaces.publishing import (
-    IBinaryPackagePublishingHistory, ISourcePackagePublishingHistory,
-    active_publishing_status)
+    IBinaryPackagePublishingHistory, IPublishingSet,
+    ISourcePackagePublishingHistory, active_publishing_status)
 from lp.soyuz.interfaces.queue import (
     IPackageUpload, IPackageUploadSet)
 from lp.soyuz.scripts.ftpmasterbase import (
@@ -520,28 +519,10 @@
     # unique publication per binary package releases (i.e. excludes
     # irrelevant arch-indep publications) and IBPPH.copy is prepared
     # to expand arch-indep publications.
-    # For safety, we use the architecture the binary was built, and
-    # not the one it is published, coping with single arch-indep
-    # publications for architectures that do not exist in the
-    # destination series. See #387589 for more information.
-    for binary in source.getBuiltBinaries():
-        binarypackagerelease = binary.binarypackagerelease
-        try:
-            target_distroarchseries = series[
-                binarypackagerelease.build.arch_tag]
-        except NotFoundError:
-            # It is not an error if the destination series doesn't
-            # support all the architectures originally built. We
-            # simply do not copy the binary and life goes on.
-            continue
-        binary_in_destination = archive.getAllPublishedBinaries(
-            name=binarypackagerelease.name, exact_match=True,
-            version=binarypackagerelease.version,
-            status=active_publishing_status, pocket=pocket,
-            distroarchseries=target_distroarchseries)
-        if binary_in_destination.count() == 0:
-            binary_copy = binary.copyTo(series, pocket, archive)
-            copies.extend(binary_copy)
+    binary_copies = getUtility(IPublishingSet).copyBinariesTo(
+        source.getBuiltBinaries(), series, pocket, archive)
+
+    copies.extend(binary_copies)
 
     # Always ensure the needed builds exist in the copy destination
     # after copying the binaries.