Merge lp:~jelmer/launchpad/cronjob into lp:launchpad

[Jelmer Vernooij]

[r=allenap][ui=none][no-qa] Add sync_packages cronjob and SyncPackageJob.run implementation.

This adds a cronjob for synchronizing packages between archives, and adds the body of the SyncPackageJob.run.

[Jelmer Vernooij (jelmer)]

This branch is lint clean.

tests: ./bin/test lp.soyuz.tests.test_syncpackagejob

[Gavin Panella (allenap)]

Looks good. I have several comments, but only [3] is likely to be of
importance, maybe [2] also.

[1]

+oops_prefix: IDSJ

Does this need to be configured for other environments too? Or is that
in production-configs?

[2]

+error_dir: /var/tmp/soyuz.test

Any idea how this pans out with parallel testing?

[3]

=== modified file 'database/schema/security.cfg'

This is targeted to devel. Because of the changes to security.cfg, you
might be better off landing this to db-devel, because you'll need to
prepare an extra script to grant these permissions on qastaging and
production otherwise.

[4]

=== added file 'cronscripts/sync_packages.py'

Scripts named with hyphens outnumber those with underscores 49 to 7.

Consider doing the right thing here ;)

Or tell me to f*** off :)

[5]

+ vars = job.getOopsVars()

vars is a builtin, so consider renaming this to avoid confusion.

[6]

Consider running utilities/format-new-and-modified-imports -r submit:
in your branch.

review: Approve

=== modified file 'configs/development/launchpad-lazr.conf'
--- configs/development/launchpad-lazr.conf	2010-10-25 20:42:59 +0000
+++ configs/development/launchpad-lazr.conf	2010-11-12 11:27:13 +0000
@@ -137,6 +137,10 @@
 oops_prefix: IDSJ
 error_dir: /var/tmp/soyuz.test
 
+[sync_packages]
+oops_prefix: SPJ
+error_dir: /var/tmp/soyuz.test
+
 [launchpad]
 enable_test_openid_provider: True
 openid_provider_vhost: testopenid

=== added file 'cronscripts/sync-packages.py'
--- cronscripts/sync-packages.py	1970-01-01 00:00:00 +0000
+++ cronscripts/sync-packages.py	2010-11-12 11:27:13 +0000
@@ -0,0 +1,25 @@
+#!/usr/bin/python -S
+#
+# Copyright 2010 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Synchronize packages."""
+
+__metaclass__ = type
+
+import _pythonpath
+
+from lp.services.job.runner import JobCronScript
+from lp.soyuz.interfaces.distributionjob import ISyncPackageJobSource
+
+
+class RunSyncPackageJob(JobCronScript):
+    """Run SyncPackageJob jobs."""
+
+    config_name = 'sync_packages'
+    source_interface = ISyncPackageJobSource
+
+
+if __name__ == '__main__':
+    script = RunSyncPackageJob()
+    script.lock_and_run()

=== modified file 'database/schema/security.cfg'
--- database/schema/security.cfg	2010-11-09 10:26:19 +0000
+++ database/schema/security.cfg	2010-11-12 11:27:13 +0000
@@ -968,6 +968,49 @@
 public.sourcepackagepublishinghistory           = SELECT, INSERT
 public.sourcepackagerelease                     = SELECT
 
+[sync_packages]
+type=user
+groups=script
+public.archive                                  = SELECT
+public.archivepermission                        = SELECT, INSERT
+public.binarypackagebuild                       = SELECT, INSERT
+public.binarypackagefile                        = SELECT, INSERT
+public.binarypackagename                        = SELECT
+public.binarypackagepublishinghistory           = SELECT, INSERT
+public.binarypackagerelease                     = SELECT
+public.buildfarmjob                             = SELECT, INSERT
+public.buildpackagejob                          = SELECT, INSERT, UPDATE, DELETE
+public.buildqueue                               = SELECT, INSERT, UPDATE
+public.component                                = SELECT
+public.componentselection                       = SELECT, INSERT
+public.distribution                             = SELECT
+public.distributionjob                          = SELECT
+public.distroarchseries                         = SELECT, INSERT
+public.distroseries                             = SELECT, UPDATE
+public.flatpackagesetinclusion                  = SELECT, INSERT
+public.gpgkey                                   = SELECT
+public.job                                      = SELECT, INSERT, UPDATE, DELETE
+public.libraryfilealias                         = SELECT, INSERT, UPDATE, DELETE
+public.libraryfilecontent                       = SELECT, INSERT
+public.packagebuild                             = SELECT, INSERT
+public.packageset                               = SELECT, INSERT
+public.packagesetgroup                          = SELECT, INSERT
+public.packagesetinclusion                      = SELECT, INSERT
+public.packagesetsources                        = SELECT, INSERT
+public.packageupload                            = SELECT
+public.packaging                                = SELECT, INSERT
+public.person                                   = SELECT
+public.pocketchroot                             = SELECT
+public.processor                                = SELECT
+public.processorfamily                          = SELECT
+public.section                                  = SELECT
+public.sectionselection                         = SELECT, INSERT
+public.sourcepackageformatselection             = SELECT, INSERT
+public.sourcepackagename                        = SELECT
+public.sourcepackagepublishinghistory           = SELECT, INSERT
+public.sourcepackagerelease                     = SELECT
+public.sourcepackagereleasefile                 = SELECT, INSERT, UPDATE
+
 [write]
 type=group
 # Full access except for tables that are exclusively updated by

=== modified file 'lib/canonical/config/schema-lazr.conf'
--- lib/canonical/config/schema-lazr.conf	2010-11-08 05:22:26 +0000
+++ lib/canonical/config/schema-lazr.conf	2010-11-12 11:27:13 +0000
@@ -940,6 +940,18 @@
 # See [error_reports].
 copy_to_zlog: false
 
+[sync_packages]
+dbuser: sync_packages
+
+# See [error_reports].
+error_dir: none
+
+# See [error_reports].
+oops_prefix: none
+
+# See [error_reports].
+copy_to_zlog: false
+
 [karmacacheupdater]
 # The database user which will be used by this process.
 # datatype: string

=== modified file 'lib/lp/registry/browser/distroseries.py'
--- lib/lp/registry/browser/distroseries.py	2010-10-06 18:53:53 +0000
+++ lib/lp/registry/browser/distroseries.py	2010-11-12 11:27:13 +0000
@@ -77,7 +77,8 @@
     )
 from lp.registry.interfaces.distroseries import IDistroSeries
 from lp.registry.interfaces.distroseriesdifference import (
-    IDistroSeriesDifferenceSource)
+    IDistroSeriesDifferenceSource,
+    )
 from lp.registry.interfaces.series import SeriesStatus
 from lp.services.features import getFeatureFlag
 from lp.services.propertycache import cachedproperty

=== modified file 'lib/lp/soyuz/model/syncpackagejob.py'
--- lib/lp/soyuz/model/syncpackagejob.py	2010-11-05 12:03:15 +0000
+++ lib/lp/soyuz/model/syncpackagejob.py	2010-11-12 11:27:13 +0000
@@ -17,7 +17,6 @@
     IMasterStore,
     IStore,
     )
-
 from lp.registry.interfaces.pocket import PackagePublishingPocket
 from lp.soyuz.interfaces.archive import IArchiveSet
 from lp.soyuz.interfaces.distributionjob import (
@@ -98,4 +97,8 @@
 
     def run(self):
         """See `IRunnableJob`."""
-        raise NotImplementedError(self.run)
+        self.target_archive.syncSource(
+            self.source_package_name, self.source_package_version,
+            self.source_archive, to_pocket=str(self.pocket),
+            to_series=self.distroseries.name,
+            include_binaries=self.include_binaries)

=== modified file 'lib/lp/soyuz/tests/test_syncpackagejob.py'
--- lib/lp/soyuz/tests/test_syncpackagejob.py	2010-11-04 13:37:23 +0000
+++ lib/lp/soyuz/tests/test_syncpackagejob.py	2010-11-12 11:27:13 +0000
@@ -1,14 +1,26 @@
 # Copyright 2010 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
+"""Tests for sync package jobs."""
+
+import os
+import subprocess
+import sys
+
+import transaction
 from zope.component import getUtility
+from zope.security.proxy import removeSecurityProxy
 
+from canonical.config import config
 from canonical.testing import LaunchpadZopelessLayer
+from lp.registry.errors import NoSuchSourcePackageName
+from lp.registry.interfaces.pocket import PackagePublishingPocket
 from lp.soyuz.interfaces.distributionjob import (
     ISyncPackageJob,
     ISyncPackageJobSource,
     )
-from lp.registry.interfaces.pocket import PackagePublishingPocket
+from lp.soyuz.interfaces.publishing import PackagePublishingStatus
+from lp.soyuz.tests.test_publishing import SoyuzTestPublisher
 from lp.testing import TestCaseWithFactory
 
 
@@ -45,3 +57,71 @@
                 PackagePublishingPocket.RELEASE,
                 "foo", "1.0-1", include_binaries=False)
         self.assertContentEqual([job], source.getActiveJobs(archive2))
+
+    def test_cronscript(self):
+        # The cron script runs without problems.
+        script = os.path.join(
+            config.root, 'cronscripts', 'sync-packages.py')
+        args = [sys.executable, script, '-v']
+        process = subprocess.Popen(
+            args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, stderr = process.communicate()
+        self.assertEqual(process.returncode, 0)
+
+    def test_run_unknown_package(self):
+        # A job properly records failure.
+        distroseries = self.factory.makeDistroSeries()
+        archive1 = self.factory.makeArchive(distroseries.distribution)
+        archive2 = self.factory.makeArchive(distroseries.distribution)
+        source = getUtility(ISyncPackageJobSource)
+        job = source.create(archive1, archive2, distroseries,
+                PackagePublishingPocket.RELEASE,
+                "foo", "1.0-1", include_binaries=False)
+        self.assertRaises(NoSuchSourcePackageName, job.run)
+
+    def test_run(self):
+        # A proper test run synchronizes packages.
+        publisher = SoyuzTestPublisher()
+        publisher.prepareBreezyAutotest()
+        distroseries = publisher.breezy_autotest
+
+        archive1 = self.factory.makeArchive(distroseries.distribution)
+        archive2 = self.factory.makeArchive(distroseries.distribution)
+
+        publisher.getPubBinaries(
+            distroseries=distroseries, binaryname="libc",
+            version="2.8-1",
+            status=PackagePublishingStatus.PUBLISHED,
+            archive=archive1)
+
+        source = getUtility(ISyncPackageJobSource)
+        job = source.create(archive1, archive2, distroseries,
+                PackagePublishingPocket.RELEASE,
+                "libc", "2.8-1", include_binaries=False)
+        # Make sure everything hits the database, switching db users
+        # aborts.
+        transaction.commit()
+        self.layer.switchDbUser('sync_packages')
+        job.run()
+
+        published_sources = archive2.getPublishedSources()
+        self.assertEquals(1, published_sources.count())
+        spr = published_sources[0].sourcepackagerelease
+        self.assertEquals("libc", spr.name)
+        self.assertEquals("2.8-1", spr.version)
+
+    def test_getOopsVars(self):
+        distroseries = self.factory.makeDistroSeries()
+        archive1 = self.factory.makeArchive(distroseries.distribution)
+        archive2 = self.factory.makeArchive(distroseries.distribution)
+        source = getUtility(ISyncPackageJobSource)
+        job = source.create(archive1, archive2, distroseries,
+                PackagePublishingPocket.RELEASE,
+                "foo", "1.0-1", include_binaries=False)
+        oops_vars = job.getOopsVars()
+        naked_job = removeSecurityProxy(job)
+        self.assertIn(
+            ('distribution_id', distroseries.distribution.id), oops_vars)
+        self.assertIn(('distroseries_id', distroseries.id), oops_vars)
+        self.assertIn(
+            ('distribution_job_id', naked_job.context.id), oops_vars)