Merge lp:~stevenk/launchpad/set-spph-packageupload into lp:launchpad

[Steve Kowalik]

Write a garbo job to try and set SourcePackagePublishingHistory.packageupload for existing publications.

Write a garbo job to try and set SourcePackagePublishingHistory.packageupload for existing publications. Since every publication isn't going to have a packageupload, we make sure to sort by id and use memcache to keep track of where we are up to. Since a publication may have multiple packageuploads related to a source (due to delayed copies or other close madness), we make sure the archive, distroseries, pocket, component and section all match.

[William Grant (wgrant)]

Needs an index like sourcepackagepublishinghistory(id) WHERE packageupload IS NULL, but otherwise fine.

review: Approve (code)

=== modified file 'lib/lp/scripts/garbo.py'
--- lib/lp/scripts/garbo.py	2012-05-24 05:43:49 +0000
+++ lib/lp/scripts/garbo.py	2012-06-04 02:49:19 +0000
@@ -26,8 +26,13 @@
 import iso8601
 from psycopg2 import IntegrityError
 import pytz
-from storm.expr import In
+from storm.expr import (
+    And,
+    In,
+    Select,
+    )
 from storm.locals import (
+    ClassAlias,
     Max,
     Min,
     SQL,
@@ -98,6 +103,12 @@
     MAIN_STORE,
     MASTER_FLAVOR,
     )
+from lp.soyuz.model.publishing import SourcePackagePublishingHistory
+from lp.soyuz.model.queue import (
+    PackageUpload,
+    PackageUploadSource,
+    )
+from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
 from lp.translations.interfaces.potemplate import IPOTemplateSet
 from lp.translations.model.potmsgset import POTMsgSet
 from lp.translations.model.potranslation import POTranslation
@@ -1137,6 +1148,71 @@
         transaction.commit()
 
 
+class PopulateSourcePackagePublishingHistoryPackageUpload(TunableLoop):
+
+    maximum_chunk_size = 5000
+
+    def __init__(self, log, abort_time=None):
+        super(
+            PopulateSourcePackagePublishingHistoryPackageUpload,
+            self).__init__(log, abort_time)
+        self.store = IMasterStore(SourcePackagePublishingHistory)
+        self.memcache_key = '%s:populate-spph-pu' % config.instance_name
+        watermark = getUtility(IMemcacheClient).get(self.memcache_key)
+        self.start_at = watermark or 0
+
+    def findSPPHs(self):
+        return self.store.find(
+            SourcePackagePublishingHistory.id,
+            SourcePackagePublishingHistory.packageuploadID == None,
+            SourcePackagePublishingHistory.id >= self.start_at).order_by(
+                SourcePackagePublishingHistory.id)
+
+    def isDone(self):
+        return self.findSPPHs().is_empty()
+
+    def __call__(self, chunk_size):
+        ids = [spph_id for spph_id in self.findSPPHs()[:chunk_size]]
+        matching_spph = ClassAlias(SourcePackagePublishingHistory)
+        packageuploads = self.store.find(
+            (SourcePackagePublishingHistory, PackageUpload),
+            SourcePackagePublishingHistory.sourcepackagereleaseID ==
+                PackageUploadSource.sourcepackagereleaseID,
+            SourcePackagePublishingHistory.sourcepackagereleaseID ==
+                SourcePackageRelease.id,
+            PackageUploadSource.packageuploadID == PackageUpload.id,
+            # If the changesfile is not None, it is an original upload
+            # (IE. not a delayed copy, or package copy job.)
+            PackageUpload.changesfile != None,
+            # Make sure we grab the minimum SPPH, just in case the
+            # publication has had overrides changed.
+            SourcePackagePublishingHistory.id == Select(
+                Min(matching_spph.id), tables=matching_spph, where=And(
+                    SourcePackagePublishingHistory.sourcepackagereleaseID ==
+                        matching_spph.sourcepackagereleaseID,
+                    SourcePackagePublishingHistory.archiveID ==
+                        matching_spph.archiveID)),
+            SourcePackagePublishingHistory.archiveID ==
+                PackageUpload.archiveID,
+            SourcePackagePublishingHistory.distroseriesID ==
+                PackageUpload.distroseriesID,
+            SourcePackagePublishingHistory.pocket == PackageUpload.pocket,
+            SourcePackagePublishingHistory.componentID ==
+                SourcePackageRelease.componentID,
+            SourcePackagePublishingHistory.sectionID ==
+                SourcePackageRelease.sectionID,
+            SourcePackagePublishingHistory.id.is_in(ids))
+        for (spph, pu) in packageuploads:
+            if pu is not None:
+                spph.packageupload = pu
+        self.start_at = ids[-1] + 1
+        result = getUtility(IMemcacheClient).set(
+            self.memcache_key, self.start_at)
+        if not result:
+            self.log.warning('Failed to set start_at in memcache.')
+        transaction.commit()
+
+
 class BaseDatabaseGarbageCollector(LaunchpadCronScript):
     """Abstract base class to run a collection of TunableLoops."""
     script_name = None  # Script name for locking and database user. Override.
@@ -1392,6 +1468,7 @@
         DuplicateSessionPruner,
         BugHeatUpdater,
         BugTaskFlattener,
+        PopulateSourcePackagePublishingHistoryPackageUpload,
         ]
     experimental_tunable_loops = []
 

=== modified file 'lib/lp/scripts/tests/test_garbo.py'
--- lib/lp/scripts/tests/test_garbo.py	2012-05-09 01:35:41 +0000
+++ lib/lp/scripts/tests/test_garbo.py	2012-06-04 02:49:19 +0000
@@ -1149,6 +1149,61 @@
         self.runHourly()
         self.assertEqual((task.id,), get_flat())
 
+    def test_PopulateSPPHPU_no_pu(self):
+        # PopulateSourcePackagePublishingHistoryPackageUpload correctly
+        # handles a SPPH with no packageupload.
+        with dbuser('testadmin'):
+            spph = self.factory.makeSourcePackagePublishingHistory()
+        self.runHourly()
+        with dbuser('testadmin'):
+            self.assertIs(None, spph.packageupload)
+
+    def createSPPHwithPU(self):
+        distroseries = self.factory.makeDistroSeries()
+        spph = self.factory.makeSourcePackagePublishingHistory(
+            distroseries=distroseries, archive=distroseries.main_archive,
+            pocket=self.factory.getAnyPocket())
+        pu = self.factory.makePackageUpload(
+            distroseries=distroseries, archive=distroseries.main_archive,
+            pocket=self.factory.getAnyPocket())
+        pu.addSource(spph.sourcepackagerelease)
+        self.assertIs(None, spph.packageupload)
+        return (spph, pu)
+
+    def test_PopulateSourcePackagePublishingHistoryPackageUpload(self):
+        # PopulateSourcePackagePublishingHistoryPackageUpload handles a
+        # SPPH with a matching packageupload.
+        with dbuser('testadmin'):
+            (spph, pu) = self.createSPPHwithPU()
+        self.runHourly()
+        with dbuser('testadmin'):
+            self.assertEqual(pu, spph.packageupload)
+
+    def test_PopulateSPPHPU_no_changesfile(self):
+        # PopulateSourcePackagePublishingHistoryPackageUpload will not
+        # set a SPPH's packageupload to a PU that is not the original upload.
+        with dbuser('testadmin'):
+            (spph, pu) = self.createSPPHwithPU()
+            pu.changesfile = None
+        self.runHourly()
+        with dbuser('testadmin'):
+            self.assertIs(None, spph.packageupload)
+
+    def test_PopulateSPPHPU_multiple_publications(self):
+        # If there are multiple publications (due to overrides changing),
+        # PopulateSourcePackagePublishingHistoryPackageUpload will set the
+        # correct publication's packageupload.
+        with dbuser('testadmin'):
+            (spph, pu) = self.createSPPHwithPU()
+            cs = self.factory.makeComponentSelection(
+                distroseries=spph.distroseries)
+            overridden_spph = spph.changeOverride(cs.component)
+            self.assertIs(None, overridden_spph.packageupload)
+        self.runHourly()
+        with dbuser('testadmin'):
+            self.assertEqual(pu, spph.packageupload)
+            self.assertIs(None, overridden_spph.packageupload)
+
 
 class TestGarboTasks(TestCaseWithFactory):
     layer = LaunchpadZopelessLayer