Merge lp:~rockstar/launchpad/branch-scan-job-script into lp:launchpad/db-devel

[Paul Hummer]

[Paul Hummer (rockstar)]

Hello there Tim sir-

  This branch adds the cron script that finds and runs all the BranchScanJobs,
and its accompanying tests. It also adds the creation of these BranchScanJobs
to IBranch.mirrorComplete, and has it accompanying tests as well.

Cheers,
Paul

[Tim Penhey (thumper)]

I'd like to see the Makefile updated for sync_branches to use this script.

review: Needs Fixing

[Tim Penhey (thumper)]

Land it!

review: Approve

=== modified file 'Makefile'
--- Makefile	2010-01-11 20:57:16 +0000
+++ Makefile	2010-01-14 23:48:14 +0000
@@ -218,7 +218,7 @@
 
 scan_branches:
 	# Scan branches from the filesystem into the database.
-	$(PY) cronscripts/branch-scanner.py
+	$(PY) cronscripts/scan_branches.py
 
 
 sync_branches: pull_branches scan_branches mpcreationjobs

=== added file 'cronscripts/scan_branches.py'
--- cronscripts/scan_branches.py	1970-01-01 00:00:00 +0000
+++ cronscripts/scan_branches.py	2010-01-14 23:48:14 +0000
@@ -0,0 +1,25 @@
+#!/usr/bin/python2.5
+#
+# Copyright 2010 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Scan branches for new revisions."""
+
+__metaclass__ = type
+
+import _pythonpath
+
+from lp.services.job.runner import JobCronScript
+from lp.code.interfaces.branchjob import IBranchScanJobSource
+
+
+class RunScanBranches(JobCronScript):
+    """Run BranchScanJob jobs."""
+
+    config_name = 'branchscanner'
+    source_interface = IBranchScanJobSource
+
+
+if __name__ == '__main__':
+    script = RunScanBranches()
+    script.lock_and_run()

=== modified file 'cronscripts/upgrade_branches.py'
--- cronscripts/upgrade_branches.py	2009-12-10 18:46:14 +0000
+++ cronscripts/upgrade_branches.py	2010-01-14 23:48:14 +0000
@@ -14,7 +14,7 @@
 
 
 class RunUpgradeBranches(JobCronScript):
-    """Run UpdatePreviewDiff jobs."""
+    """Run UpgradeBranchJob jobs."""
 
     config_name = 'upgrade_branches'
     source_interface = IBranchUpgradeJobSource

=== modified file 'lib/lp/code/configure.zcml'
--- lib/lp/code/configure.zcml	2010-01-11 04:22:01 +0000
+++ lib/lp/code/configure.zcml	2010-01-14 23:48:14 +0000
@@ -901,6 +901,15 @@
     <allow interface="lp.services.job.interfaces.job.IRunnableJob" />
   </class>
 
+  <securedutility
+      component="lp.code.model.branchjob.BranchScanJob"
+      provides="lp.code.interfaces.branchjob.IBranchScanJobSource">
+    <allow interface="lp.code.interfaces.branchjob.IBranchScanJobSource"/>
+  </securedutility>
+  <class class="lp.code.model.branchjob.BranchScanJob">
+    <allow interface="lp.services.job.interfaces.job.IRunnableJob" />
+  </class>
+
   <!-- Linked branches -->
   <adapter factory="lp.code.model.linkedbranch.ProductSeriesLinkedBranch" />
   <adapter factory="lp.code.model.linkedbranch.ProductLinkedBranch" />

=== modified file 'lib/lp/code/interfaces/branchjob.py'
--- lib/lp/code/interfaces/branchjob.py	2010-01-13 02:44:21 +0000
+++ lib/lp/code/interfaces/branchjob.py	2010-01-14 23:48:14 +0000
@@ -33,7 +33,7 @@
 
 from canonical.launchpad import _
 from lp.code.interfaces.branch import IBranch
-from lp.services.job.interfaces.job import IJob, IRunnableJob
+from lp.services.job.interfaces.job import IJob, IRunnableJob, IJobSource
 
 
 class IBranchJob(Interface):
@@ -76,14 +76,11 @@
         """
 
 
-class IBranchScanJob(Interface):
+class IBranchScanJob(IRunnableJob):
     """ A job to scan branches."""
 
-    def run():
-        """Scan a branch for new revisions."""
-
-
-class IBranchScanJobSource(Interface):
+
+class IBranchScanJobSource(IJobSource):
 
     def create(branch):
         """Scan a branch for new revisions.
@@ -91,18 +88,11 @@
         :param branch: The database branch to upgrade.
         """
 
-    def iterReady():
-        """Iterate through all IBranchScanJobs."""
-
-
 class IBranchUpgradeJob(IRunnableJob):
     """A job to upgrade branches with out-of-date formats."""
 
-    def run():
-        """Upgrade the branch to the format specified."""
-
-
-class IBranchUpgradeJobSource(Interface):
+
+class IBranchUpgradeJobSource(IJobSource):
 
     def create(branch):
         """Upgrade a branch to a more current format.
@@ -110,12 +100,6 @@
         :param branch: The database branch to upgrade.
         """
 
-    def iterReady():
-        """Iterate through all IBranchUpgradeJobs."""
-
-    def contextManager():
-        """Get a context for running this kind of job in."""
-
 
 class IRevisionMailJob(IRunnableJob):
     """A Job to send email a revision change in a branch."""

=== modified file 'lib/lp/code/model/branch.py'
--- lib/lp/code/model/branch.py	2010-01-12 21:37:34 +0000
+++ lib/lp/code/model/branch.py	2010-01-14 23:48:14 +0000
@@ -929,6 +929,8 @@
             self.next_mirror_time = (
                 datetime.now(pytz.timezone('UTC')) + increment)
         self.last_mirrored_id = last_revision_id
+        from lp.code.model.branchjob import BranchScanJob
+        BranchScanJob.create(self)
 
     def mirrorFailed(self, reason):
         """See `IBranch`."""

=== modified file 'lib/lp/code/model/branchjob.py'
--- lib/lp/code/model/branchjob.py	2010-01-13 05:37:02 +0000
+++ lib/lp/code/model/branchjob.py	2010-01-14 23:48:14 +0000
@@ -3,6 +3,7 @@
 
 __all__ = [
     'BranchJob',
+    'BranchScanJob',
     'BranchUpgradeJob',
     'RevisionsAddedJob',
     'RevisionMailJob',
@@ -275,6 +276,16 @@
         bzrsync = BzrSync(self.branch)
         bzrsync.syncBranchAndClose()
 
+    @staticmethod
+    @contextlib.contextmanager
+    def contextManager():
+        """See `IBranchScanJobSource`."""
+        errorlog.globalErrorUtility.configure('branchscanner')
+        server = get_multi_server()
+        server.setUp()
+        yield
+        server.tearDown()
+
 
 class BranchUpgradeJob(BranchJobDerived):
     """A Job that upgrades branches to the current stable format."""

=== modified file 'lib/lp/code/model/tests/test_branch.py'
--- lib/lp/code/model/tests/test_branch.py	2010-01-07 06:37:14 +0000
+++ lib/lp/code/model/tests/test_branch.py	2010-01-14 23:48:14 +0000
@@ -1433,6 +1433,18 @@
         branch.mirrorComplete(rev_id)
         self.assertEqual(True, branch.pending_writes)
 
+    def test_mirrorComplete_creates_scan_job(self):
+        # After a branch has been pulled, it should have created a
+        # BranchScanJob to complete the process.
+        branch = self.factory.makeAnyBranch()
+        branch.startMirroring()
+        rev_id = self.factory.getUniqueString('rev-id')
+        branch.mirrorComplete(rev_id)
+
+        store = Store.of(branch)
+        scan_jobs = store.find(BranchJob, job_type=BranchJobType.SCAN_BRANCH)
+        self.assertEqual(scan_jobs.count(), 1)
+
     def test_pulled_and_scanned(self):
         # If a branch has been pulled and scanned, then there are no pending
         # writes.

=== added file 'lib/lp/code/scripts/tests/test_scan_branches.py'
--- lib/lp/code/scripts/tests/test_scan_branches.py	1970-01-01 00:00:00 +0000
+++ lib/lp/code/scripts/tests/test_scan_branches.py	2010-01-14 23:48:14 +0000
@@ -0,0 +1,59 @@
+#! /usr/bin/python2.5
+#
+# Copyright 2010 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Test the scan_branches script."""
+
+
+import transaction
+
+from canonical.testing import ZopelessAppServerLayer
+from lp.testing import TestCaseWithFactory
+from canonical.launchpad.scripts.tests import run_script
+from lp.code.model.branchjob import BranchScanJob
+
+
+class TestScanBranches(TestCaseWithFactory):
+    """Test the scan_branches script."""
+
+    layer = ZopelessAppServerLayer
+
+    def make_branch_with_commits_and_scan_job(self, db_branch):
+        """Create a branch from a db_branch, make commits and a scan job."""
+        target, target_tree = self.create_branch_and_tree(
+            db_branch=db_branch)
+        target_tree.commit('First commit', rev_id='rev1')
+        target_tree.commit('Second commit', rev_id='rev2')
+        target_tree.commit('Third commit', rev_id='rev3')
+        job = BranchScanJob.create(db_branch)
+        transaction.commit()
+
+    def run_script_and_assert_success(self):
+        """Run the scan_branches script and assert it ran successfully."""
+        retcode, stdout, stderr = run_script(
+            'cronscripts/scan_branches.py', [],
+            expect_returncode=0)
+        self.assertEqual('', stdout)
+        self.assertIn(
+            'INFO    Ran 1 IBranchScanJobSource jobs.\n', stderr)
+
+    def test_scan_branch(self):
+        """Test that scan branches adds revisions to the database."""
+        self.useBzrBranches(real_server=True)
+
+        db_branch = self.factory.makeAnyBranch()
+        self.make_branch_with_commits_and_scan_job(db_branch)
+
+        self.run_script_and_assert_success()
+        self.assertEqual(db_branch.revision_count, 3)
+
+    def test_scan_branch_packagebranch(self):
+        """Test that scan_branches can scan package branches."""
+        self.useBzrBranches(real_server=True)
+
+        db_branch = self.factory.makePackageBranch()
+        self.make_branch_with_commits_and_scan_job(db_branch)
+
+        self.run_script_and_assert_success()
+        self.assertEqual(db_branch.revision_count, 3)

=== modified file 'lib/lp/services/job/interfaces/job.py'
--- lib/lp/services/job/interfaces/job.py	2010-01-11 22:27:16 +0000
+++ lib/lp/services/job/interfaces/job.py	2010-01-14 23:48:14 +0000
@@ -144,3 +144,13 @@
 
     def run():
         """Run this job."""
+
+
+class IJobSource(Interface):
+    """Interface for creating and getting jobs."""
+
+    def iterReady():
+        """Iterate through all jobs."""
+
+    def contextManager():
+        """Get a context for running this kind of job in."""