Merge lp:~gmb/launchpad/garbo-hourly-nextcheck-calc-bug-545909 into lp:launchpad/db-devel

[Graham Binns]

Add a garbo-hourly job to schedule bug watches.

This branch adds the garbo-hourly job for scheduling bug watches.

[Gavin Panella (allenap)]

Review done in person. A few small tweaks needed, but otherwise jolly nice :)

review: Approve

=== modified file 'database/schema/security.cfg'
--- database/schema/security.cfg	2010-03-25 13:27:44 +0000
+++ database/schema/security.cfg	2010-03-25 21:02:38 +0000
@@ -1828,6 +1828,8 @@
 public.bugaffectsperson                 = SELECT
 public.bugnotification                  = SELECT, DELETE
 public.bugnotificationrecipientarchive  = SELECT
+public.bugwatch                         = SELECT, UPDATE
+public.bugwatchactivity                 = SELECT
 public.codeimportresult                 = SELECT, DELETE
 public.emailaddress                     = SELECT
 public.oauthnonce                       = SELECT, DELETE

=== modified file 'lib/canonical/launchpad/scripts/garbo.py'
--- lib/canonical/launchpad/scripts/garbo.py	2010-03-05 11:46:55 +0000
+++ lib/canonical/launchpad/scripts/garbo.py	2010-03-25 21:02:38 +0000
@@ -13,7 +13,6 @@
 import transaction
 from psycopg2 import IntegrityError
 from zope.component import getUtility
-from zope.interface import implements
 from storm.locals import In, SQL, Max, Min
 
 from canonical.config import config
@@ -27,12 +26,14 @@
 from canonical.launchpad.interfaces import IMasterStore
 from canonical.launchpad.interfaces.emailaddress import EmailAddressStatus
 from canonical.launchpad.interfaces.looptuner import ITunableLoop
-from canonical.launchpad.utilities.looptuner import DBLoopTuner
+from canonical.launchpad.utilities.looptuner import (
+    DBLoopTuner, TunableLoop)
 from canonical.launchpad.webapp.interfaces import (
     IStoreSelector, AUTH_STORE, MAIN_STORE, MASTER_FLAVOR)
 from lp.bugs.interfaces.bug import IBugSet
 from lp.bugs.interfaces.bugjob import ICalculateBugHeatJobSource
 from lp.bugs.model.bugnotification import BugNotification
+from lp.bugs.scripts.checkwatches.scheduler import BugWatchScheduler
 from lp.code.interfaces.revision import IRevisionSet
 from lp.code.model.branchjob import BranchJob
 from lp.code.model.codeimportresult import CodeImportResult
@@ -47,29 +48,6 @@
 ONE_DAY_IN_SECONDS = 24*60*60
 
 
-class TunableLoop:
-    implements(ITunableLoop)
-
-    goal_seconds = 4
-    minimum_chunk_size = 1
-    maximum_chunk_size = None # Override
-    cooldown_time = 0
-
-    def __init__(self, log, abort_time=None):
-        self.log = log
-        self.abort_time = abort_time
-
-    def run(self):
-        assert self.maximum_chunk_size is not None, (
-            "Did not override maximum_chunk_size.")
-        DBLoopTuner(
-            self, self.goal_seconds,
-            minimum_chunk_size = self.minimum_chunk_size,
-            maximum_chunk_size = self.maximum_chunk_size,
-            cooldown_time = self.cooldown_time,
-            abort_time = self.abort_time).run()
-
-
 class OAuthNoncePruner(TunableLoop):
     """An ITunableLoop to prune old OAuthNonce records.
 
@@ -839,6 +817,7 @@
         OpenIDConsumerAssociationPruner,
         RevisionCachePruner,
         BugHeatUpdater,
+        BugWatchScheduler,
         ]
     experimental_tunable_loops = []
 

=== modified file 'lib/canonical/launchpad/utilities/looptuner.py'
--- lib/canonical/launchpad/utilities/looptuner.py	2009-07-17 00:26:05 +0000
+++ lib/canonical/launchpad/utilities/looptuner.py	2010-03-25 21:02:38 +0000
@@ -3,7 +3,11 @@
 
 __metaclass__ = type
 
-__all__ = ['DBLoopTuner', 'LoopTuner']
+__all__ = [
+    'DBLoopTuner',
+    'LoopTuner',
+    'TunableLoop',
+    ]
 
 
 from datetime import timedelta
@@ -11,6 +15,7 @@
 
 import transaction
 from zope.component import getUtility
+from zope.interface import implements
 
 import canonical.launchpad.scripts
 from canonical.launchpad.webapp.interfaces import (
@@ -262,3 +267,26 @@
                 time.sleep(remaining_nap)
         return self._time()
 
+
+class TunableLoop:
+    """A base implementation of `ITunableLoop`."""
+    implements(ITunableLoop)
+
+    goal_seconds = 4
+    minimum_chunk_size = 1
+    maximum_chunk_size = None # Override
+    cooldown_time = 0
+
+    def __init__(self, log, abort_time=None):
+        self.log = log
+        self.abort_time = abort_time
+
+    def run(self):
+        assert self.maximum_chunk_size is not None, (
+            "Did not override maximum_chunk_size.")
+        DBLoopTuner(
+            self, self.goal_seconds,
+            minimum_chunk_size = self.minimum_chunk_size,
+            maximum_chunk_size = self.maximum_chunk_size,
+            cooldown_time = self.cooldown_time,
+            abort_time = self.abort_time).run()

=== modified file 'lib/lp/bugs/doc/externalbugtracker.txt'
--- lib/lp/bugs/doc/externalbugtracker.txt	2010-03-17 18:45:27 +0000
+++ lib/lp/bugs/doc/externalbugtracker.txt	2010-03-25 21:02:38 +0000
@@ -201,7 +201,7 @@
 
     >>> from lp.bugs.scripts import checkwatches
     >>> (bug_watch_updater._syncable_gnome_products ==
-    ...     checkwatches.SYNCABLE_GNOME_PRODUCTS)
+    ...     checkwatches.updater.SYNCABLE_GNOME_PRODUCTS)
     True
 
     >>> syncable_products = ['HeartOfGold']

=== added directory 'lib/lp/bugs/scripts/checkwatches'
=== added file 'lib/lp/bugs/scripts/checkwatches/__init__.py'
--- lib/lp/bugs/scripts/checkwatches/__init__.py	1970-01-01 00:00:00 +0000
+++ lib/lp/bugs/scripts/checkwatches/__init__.py	2010-03-25 21:02:38 +0000
@@ -0,0 +1,8 @@
+# Copyright 2010 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+"""Top-level __init__ for the checkwatches package."""
+
+# We do this to maintain backwards compatibility with tests.
+from lp.bugs.scripts.checkwatches.updater import *
+
+

=== added file 'lib/lp/bugs/scripts/checkwatches/scheduler.py'
--- lib/lp/bugs/scripts/checkwatches/scheduler.py	1970-01-01 00:00:00 +0000
+++ lib/lp/bugs/scripts/checkwatches/scheduler.py	2010-03-25 21:02:38 +0000
@@ -0,0 +1,91 @@
+# Copyright 2010 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+"""Code for the BugWatch scheduler."""
+
+__metaclass__ = type
+__all__ = [
+    'BugWatchScheduler',
+    ]
+
+import transaction
+
+from storm.expr import Not
+
+from canonical.database.sqlbase import sqlvalues
+from canonical.launchpad.utilities.looptuner import TunableLoop
+from canonical.launchpad.interfaces import IMasterStore
+
+from lp.bugs.model.bugwatch import BugWatch
+
+
+# The maximum additional delay in days that a watch may have placed upon
+# it.
+MAX_DELAY_DAYS = 6
+# The maximum number of BugWatchActivity entries we want to examine.
+MAX_SAMPLE_SIZE = 5
+
+
+def get_delay_coefficient(max_delay_days, max_sample_size):
+    return float(max_delay_days) / float(max_sample_size)
+
+
+class BugWatchScheduler(TunableLoop):
+    """An `ITunableLoop` for scheduling BugWatches."""
+
+    maximum_chunk_size = 1000
+
+    def __init__(self, log, abort_time=None, max_delay_days=None,
+                 max_sample_size=None):
+        super(BugWatchScheduler, self).__init__(log, abort_time)
+        self.transaction = transaction
+        self.store = IMasterStore(BugWatch)
+
+        if max_delay_days is None:
+            max_delay_days = MAX_DELAY_DAYS
+        if max_sample_size is None:
+            max_sample_size = MAX_SAMPLE_SIZE
+        self.max_sample_size = max_sample_size
+
+        self.delay_coefficient = get_delay_coefficient(
+            max_delay_days, max_sample_size)
+
+    def __call__(self, chunk_size):
+        """Run the loop."""
+        # XXX 2010-03-25 gmb bug=198767:
+        #     We cast chunk_size to an integer to ensure that we're not
+        #     trying to slice using floats or anything similarly
+        #     foolish. We shouldn't have to do this.
+        chunk_size = int(chunk_size)
+        query = """
+        UPDATE BugWatch
+            SET next_check =
+                COALESCE(
+                    lastchecked + interval '1 day',
+                    now() AT TIME ZONE 'UTC') +
+                (interval '1 day' * (%s * recent_failure_count))
+            FROM (
+                SELECT bug_watch.id,
+                    (SELECT COUNT(*)
+                        FROM (SELECT 1
+                            FROM bugwatchactivity
+                           WHERE bugwatchactivity.bug_watch = bug_watch.id
+                             AND bugwatchactivity.result IS NOT NULL
+                           ORDER BY bugwatchactivity.id DESC
+                           LIMIT %s) AS recent_failures
+                    ) AS recent_failure_count
+                FROM BugWatch AS bug_watch
+                WHERE bug_watch.next_check IS NULL
+                LIMIT %s
+            ) AS counts
+        WHERE BugWatch.id = counts.id
+        """ % sqlvalues(
+            self.delay_coefficient, self.max_sample_size, chunk_size)
+        self.transaction.begin()
+        result = self.store.execute(query)
+        self.log.debug("Scheduled %s watches" % result.rowcount)
+        self.transaction.commit()
+
+    def isDone(self):
+        """Return True when there are no more watches to schedule."""
+        return self.store.find(
+            BugWatch, BugWatch.next_check == None).is_empty()

=== renamed file 'lib/lp/bugs/scripts/checkwatches.py' => 'lib/lp/bugs/scripts/checkwatches/updater.py'
--- lib/lp/bugs/scripts/checkwatches.py	2010-03-25 16:21:10 +0000
+++ lib/lp/bugs/scripts/checkwatches/updater.py	2010-03-25 21:02:38 +0000
@@ -3,6 +3,16 @@
 """Classes and logic for the checkwatches cronscript."""
 
 __metaclass__ = type
+__all__ = [
+    'BaseScheduler',
+    'BugWatchUpdater',
+    'CheckWatchesCronScript',
+    'CheckWatchesErrorUtility',
+    'externalbugtracker',
+    'SerialScheduler',
+    'TooMuchTimeSkew',
+    'TwistedThreadScheduler',
+    ]
 
 from copy import copy
 from datetime import datetime, timedelta

=== modified file 'lib/lp/bugs/scripts/tests/test_checkwatches.py'
--- lib/lp/bugs/scripts/tests/test_checkwatches.py	2010-03-23 18:46:08 +0000
+++ lib/lp/bugs/scripts/tests/test_checkwatches.py	2010-03-25 21:02:38 +0000
@@ -73,8 +73,8 @@
         # We monkey-patch externalbugtracker.get_external_bugtracker()
         # so that it always returns what we want.
         self.original_get_external_bug_tracker = (
-            checkwatches.externalbugtracker.get_external_bugtracker)
-        checkwatches.externalbugtracker.get_external_bugtracker = (
+            checkwatches.updater.externalbugtracker.get_external_bugtracker)
+        checkwatches.updater.externalbugtracker.get_external_bugtracker = (
             always_BugzillaAPI_get_external_bugtracker)
 
         # Create an updater with a limited set of syncable gnome

=== modified file 'lib/lp/bugs/tests/test_bugwatch.py'
--- lib/lp/bugs/tests/test_bugwatch.py	2010-01-04 11:14:16 +0000
+++ lib/lp/bugs/tests/test_bugwatch.py	2010-03-25 21:02:38 +0000
@@ -5,20 +5,27 @@
 
 __metaclass__ = type
 
+import transaction
 import unittest
 
+from datetime import datetime, timedelta
+from pytz import utc
+
 from urlparse import urlunsplit
 
 from zope.component import getUtility
 
 from canonical.launchpad.ftests import login, ANONYMOUS
+from canonical.launchpad.scripts.logger import QuietFakeLogger
 from canonical.launchpad.webapp import urlsplit
 from canonical.testing import (
     DatabaseFunctionalLayer, LaunchpadFunctionalLayer, LaunchpadZopelessLayer)
 
 from lp.bugs.interfaces.bugtracker import BugTrackerType, IBugTrackerSet
 from lp.bugs.interfaces.bugwatch import (
-    IBugWatchSet, NoBugTrackerFound, UnrecognizedBugTrackerURL)
+    BugWatchActivityStatus, IBugWatchSet, NoBugTrackerFound,
+    UnrecognizedBugTrackerURL)
+from lp.bugs.scripts.checkwatches.scheduler import BugWatchScheduler
 from lp.registry.interfaces.person import IPersonSet
 
 from lp.testing import TestCaseWithFactory
@@ -372,5 +379,86 @@
             self.bug_watch.bugtasks, list)
 
 
+class TestBugWatchScheduler(TestCaseWithFactory):
+    """Tests for the BugWatchScheduler, which runs as part of garbo."""
+
+    layer = LaunchpadFunctionalLayer
+
+    def setUp(self):
+        super(TestBugWatchScheduler, self).setUp('[e-mail]')
+        # We'll make sure that all the other bug watches look like
+        # they've been scheduled so that only our watch gets scheduled.
+        for watch in getUtility(IBugWatchSet).search():
+            watch.next_check = datetime.now(utc)
+
+        self.bug_watch = self.factory.makeBugWatch()
+        transaction.commit()
+        self.scheduler = BugWatchScheduler(QuietFakeLogger())
+
+    def test_scheduler_schedules_unchecked_watches(self):
+        # The BugWatchScheduler will schedule a BugWatch that has never
+        # been checked to be checked immediately.
+        self.bug_watch.next_check = None
+        self.scheduler(1)
+
+        self.assertNotEqual(None, self.bug_watch.next_check)
+        self.assertTrue(
+            self.bug_watch.next_check <= datetime.now(utc))
+
+    def test_scheduler_schedules_working_watches(self):
+        # If a watch has been checked and has never failed its next
+        # check will be scheduled for 24 hours after its last check.
+        now = datetime.now(utc)
+        self.bug_watch.lastchecked = now
+        self.bug_watch.next_check = None
+        transaction.commit()
+        self.scheduler(1)
+
+        self.assertEqual(
+            now + timedelta(hours=24), self.bug_watch.next_check)
+
+    def test_scheduler_schedules_failing_watches(self):
+        # If a watch has failed once, it will be scheduled more than 24
+        # hours after its last check.
+        now = datetime.now(utc)
+        self.bug_watch.lastchecked = now
+
+        # The delay depends on the number of failures that the watch has
+        # had.
+        for failure_count in range(1, 6):
+            self.bug_watch.next_check = None
+            self.bug_watch.addActivity(
+                result=BugWatchActivityStatus.BUG_NOT_FOUND)
+            transaction.commit()
+            self.scheduler(1)
+
+            coefficient = self.scheduler.delay_coefficient * failure_count
+            self.assertEqual(
+                now + timedelta(days=1 + coefficient),
+                self.bug_watch.next_check)
+
+        # The scheduler only looks at the last 5 activity items, so even
+        # if there have been more failures the maximum delay will be 7
+        # days.
+        for count in range(10):
+            self.bug_watch.addActivity(
+                result=BugWatchActivityStatus.BUG_NOT_FOUND)
+        self.bug_watch.next_check = None
+        transaction.commit()
+        self.scheduler(1)
+        self.assertEqual(
+            now + timedelta(days=7), self.bug_watch.next_check)
+
+    def test_scheduler_doesnt_schedule_scheduled_watches(self):
+        # The scheduler will ignore watches whose next_check has been
+        # set.
+        next_check_date = datetime.now(utc) + timedelta(days=1)
+        self.bug_watch.next_check = next_check_date
+        transaction.commit()
+        self.scheduler(1)
+
+        self.assertEqual(next_check_date, self.bug_watch.next_check)
+
+
 def test_suite():
     return unittest.TestLoader().loadTestsFromName(__name__)