Merge lp:~wgrant/launchpad/bugtaskflat-garbo into lp:launchpad

[William Grant]

Add a garbo job to populate BugTaskFlat.

BugTaskFlat will soon exist on production, but it needs to be populated. This is a garbo job to do that, pretty much the same as the bug legacy access mirrorer that I removed a couple of weeks back.

[Steve Kowalik (stevenk)]

Looks good.

review: Approve (code)

=== modified file 'lib/lp/scripts/garbo.py'
--- lib/lp/scripts/garbo.py	2012-03-26 22:47:47 +0000
+++ lib/lp/scripts/garbo.py	2012-04-02 07:16:34 +0000
@@ -44,6 +44,7 @@
 from lp.bugs.model.bug import Bug
 from lp.bugs.model.bugattachment import BugAttachment
 from lp.bugs.model.bugnotification import BugNotification
+from lp.bugs.model.bugtask import BugTask
 from lp.bugs.model.bugwatch import BugWatchActivity
 from lp.bugs.scripts.checkwatches.scheduler import (
     BugWatchScheduler,
@@ -81,6 +82,7 @@
 from lp.services.librarian.model import TimeLimitedToken
 from lp.services.log.logger import PrefixFilter
 from lp.services.looptuner import TunableLoop
+from lp.services.memcache.interfaces import IMemcacheClient
 from lp.services.oauth.model import OAuthNonce
 from lp.services.openid.model.openidconsumer import OpenIDConsumerNonce
 from lp.services.propertycache import cachedproperty
@@ -1091,6 +1093,40 @@
         self.offset += chunk_size
 
 
+class BugTaskFlattener(TunableLoop):
+    """A `TunableLoop` to populate BugTaskFlat for all bugtasks."""
+
+    maximum_chunk_size = 5000
+
+    def __init__(self, log, abort_time=None):
+        super(BugTaskFlattener, self).__init__(log, abort_time)
+        watermark = getUtility(IMemcacheClient).get(
+            '%s:bugtask-flattener' % config.instance_name)
+        self.start_at = watermark or 0
+
+    def findTaskIDs(self):
+        return IMasterStore(BugTask).find(
+            (BugTask.id,), BugTask.id >= self.start_at).order_by(BugTask.id)
+
+    def isDone(self):
+        return self.findTaskIDs().is_empty()
+
+    def __call__(self, chunk_size):
+        ids = [row[0] for row in self.findTaskIDs()[:chunk_size]]
+        list(IMasterStore(BugTask).using(BugTask).find(
+            SQL('bugtask_flatten(BugTask.id, false)'),
+            BugTask.id.is_in(ids)))
+
+        self.start_at = ids[-1] + 1
+        result = getUtility(IMemcacheClient).set(
+            '%s:bugtask-flattener' % config.instance_name,
+            self.start_at)
+        if not result:
+            self.log.warning('Failed to set start_at in memcache.')
+
+        transaction.commit()
+
+
 class BaseDatabaseGarbageCollector(LaunchpadCronScript):
     """Abstract base class to run a collection of TunableLoops."""
     script_name = None  # Script name for locking and database user. Override.
@@ -1342,6 +1378,7 @@
         UnusedSessionPruner,
         DuplicateSessionPruner,
         BugHeatUpdater,
+        BugTaskFlattener,
         ]
     experimental_tunable_loops = []
 
 
=== modified file 'lib/lp/scripts/tests/test_garbo.py'
--- lib/lp/scripts/tests/test_garbo.py	2012-03-26 22:47:47 +0000
+++ lib/lp/scripts/tests/test_garbo.py	2012-04-02 07:16:34 +0000
@@ -42,6 +42,7 @@
     BugNotification,
     BugNotificationRecipient,
     )
+from lp.bugs.model.bugtask import BugTask
 from lp.code.bzr import (
     BranchFormat,
     RepositoryFormat,
@@ -1104,6 +1105,35 @@
         self.assertEqual(whiteboard, spec.whiteboard)
         self.assertEqual(0, spec.work_items.count())
 
+    def test_BugTaskFlattener(self):
+        # Private bugs without corresponding data in the access policy
+        # schema get mirrored.
+        switch_dbuser('testadmin')
+        task = self.factory.makeBugTask()
+        # Remove the existing mirrored data.
+        IMasterStore(BugTask).execute(
+            'DELETE FROM BugTaskFlat WHERE bugtask = ?', (task.id,))
+        transaction.commit()
+        self.runHourly()
+        # Check that there's a record again, and delete it.
+        switch_dbuser('testadmin')
+        self.assertEqual(
+            (task.id,),
+            IMasterStore(BugTask).execute(
+                'SELECT bugtask FROM BugTaskFlat WHERE bugtask = ?',
+                (task.id,)).get_one())
+        IMasterStore(BugTask).execute(
+            'DELETE FROM BugTaskFlat WHERE bugtask = ?', (task.id,))
+        transaction.commit()
+        self.runHourly()
+        # A watermark is kept in memcache, so a second run doesn't
+        # consider the same task.
+        self.assertIs(
+            None,
+            IMasterStore(BugTask).execute(
+                'SELECT bugtask FROM BugTaskFlat WHERE bugtask = ?',
+                (task.id,)).get_one())
+
 
 class TestGarboTasks(TestCaseWithFactory):
     layer = LaunchpadZopelessLayer