Merge lp:~stevenk/launchpad/drop-psc into lp:launchpad

[Steve Kowalik]

[r=lifeless,wgrant][bug=764292] Drop PopulateSPRChangelogs to thank it for its work.

After running for one month, and processing the changelogs for as-near-as-makes-no-difference half a million SPRs, the PopulateSPRChangelogs class in the hourly garbo script has done its dash. So to thank it, we kick it out of the code base.

[Robert Collins (lifeless)]

You should wait till staging and qastaging are fully migrated too.
Otherwise bad things will happen as you qa stuff :).

[William Grant (wgrant)]

Looks good. No need to wait.

review: Approve (code*)

[Robert Collins (lifeless)]

shrug, sure.

review: Approve

=== modified file 'database/schema/security.cfg'
--- database/schema/security.cfg	2011-04-06 15:35:05 +0000
+++ database/schema/security.cfg	2011-04-15 05:03:31 +0000
@@ -2260,9 +2260,6 @@
 public.job                              = SELECT, INSERT, DELETE
 public.branchjob                        = SELECT, DELETE
 public.bugjob                           = SELECT, INSERT
-public.libraryfilealias                 = SELECT, INSERT
-public.libraryfilecontent               = SELECT, INSERT
-public.sourcepackagerelease             = SELECT, UPDATE
 
 [garbo_daily]
 type=user

=== modified file 'lib/lp/scripts/garbo.py'
--- lib/lp/scripts/garbo.py	2011-04-12 08:58:38 +0000
+++ lib/lp/scripts/garbo.py	2011-04-15 05:03:31 +0000
@@ -13,21 +13,14 @@
     datetime,
     timedelta,
     )
-from fixtures import TempDir
 import logging
 import multiprocessing
-import os
-import signal
-import subprocess
 import threading
 import time
 
 from psycopg2 import IntegrityError
 import pytz
-from storm.expr import LeftJoin
 from storm.locals import (
-    And,
-    Count,
     Max,
     Min,
     SQL,
@@ -51,7 +44,6 @@
 from canonical.launchpad.database.oauth import OAuthNonce
 from canonical.launchpad.database.openidconsumer import OpenIDConsumerNonce
 from canonical.launchpad.interfaces.emailaddress import EmailAddressStatus
-from canonical.launchpad.interfaces.librarian import ILibraryFileAliasSet
 from canonical.launchpad.interfaces.lpstorm import IMasterStore
 from canonical.launchpad.utilities.looptuner import TunableLoop
 from canonical.launchpad.webapp.interfaces import (
@@ -59,9 +51,6 @@
     MAIN_STORE,
     MASTER_FLAVOR,
     )
-from canonical.librarian.utils import copy_and_close
-from lp.archiveuploader.dscfile import findFile
-from lp.archiveuploader.nascentuploadfile import UploadError
 from lp.bugs.interfaces.bug import IBugSet
 from lp.bugs.model.bug import Bug
 from lp.bugs.model.bugattachment import BugAttachment
@@ -82,14 +71,11 @@
 from lp.registry.model.person import Person
 from lp.services.job.model.job import Job
 from lp.services.log.logger import PrefixFilter
-from lp.services.memcache.interfaces import IMemcacheClient
 from lp.services.scripts.base import (
     LaunchpadCronScript,
     SilentLaunchpadScriptFailure,
     )
 from lp.services.session.model import SessionData
-from lp.soyuz.model.files import SourcePackageReleaseFile
-from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
 from lp.translations.interfaces.potemplate import IPOTemplateSet
 from lp.translations.model.potranslation import POTranslation
 
@@ -97,10 +83,6 @@
 ONE_DAY_IN_SECONDS = 24*60*60
 
 
-def subprocess_setup():
-    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
-
-
 class BulkPruner(TunableLoop):
     """A abstract ITunableLoop base class for simple pruners.
 
@@ -839,124 +821,6 @@
         self.done = True
 
 
-class PopulateSPRChangelogs(TunableLoop):
-    maximum_chunk_size = 1
-
-    def __init__(self, log, abort_time=None):
-        super(PopulateSPRChangelogs, self).__init__(log, abort_time)
-        value = getUtility(IMemcacheClient).get('populate-spr-changelogs')
-        if not value:
-            self.start_at = 0
-        else:
-            self.start_at = value
-        self.finish_at = self.getCandidateSPRs(0).last()
-
-    def getCandidateSPRs(self, start_at):
-        return IMasterStore(SourcePackageRelease).using(
-            SourcePackageRelease,
-            # Find any SPRFs that have expired (LFA.content IS NULL).
-            LeftJoin(
-                SourcePackageReleaseFile,
-                SourcePackageReleaseFile.sourcepackagereleaseID ==
-                    SourcePackageRelease.id),
-            LeftJoin(
-                LibraryFileAlias,
-                And(LibraryFileAlias.id ==
-                    SourcePackageReleaseFile.libraryfileID,
-                    LibraryFileAlias.content == None)),
-            # And exclude any SPRs that have any expired SPRFs.
-            ).find(
-                SourcePackageRelease.id,
-                SourcePackageRelease.id >= start_at,
-                SourcePackageRelease.changelog == None,
-            ).group_by(SourcePackageRelease.id).having(
-                Count(LibraryFileAlias) == 0
-            ).order_by(SourcePackageRelease.id)
-
-    def isDone(self):
-        return self.start_at > self.finish_at
-
-    def __call__(self, chunk_size):
-        for sprid in self.getCandidateSPRs(self.start_at)[:chunk_size]:
-            spr = SourcePackageRelease.get(sprid)
-            with TempDir() as tmp_dir:
-                dsc_file = None
-
-                # Grab the files from the librarian into a temporary
-                # directory.
-                try:
-                    for sprf in spr.files:
-                        dest = os.path.join(
-                            tmp_dir.path, sprf.libraryfile.filename)
-                        dest_file = open(dest, 'w')
-                        sprf.libraryfile.open()
-                        copy_and_close(sprf.libraryfile, dest_file)
-                        if dest.endswith('.dsc'):
-                            dsc_file = dest
-                except LookupError:
-                    self.log.warning(
-                        'SPR %d (%s %s) has missing library files.' % (
-                            spr.id, spr.name, spr.version))
-                    continue
-
-                if dsc_file is None:
-                    self.log.warning(
-                        'SPR %d (%s %s) has no DSC.' % (
-                            spr.id, spr.name, spr.version))
-                    continue
-
-                # Extract the source package. Throw away stdout/stderr
-                # -- we only really care about the return code.
-                fnull = open('/dev/null', 'w')
-                ret = subprocess.call(
-                    ['dpkg-source', '-x', dsc_file, os.path.join(
-                        tmp_dir.path, 'extracted')],
-                        stdout=fnull, stderr=fnull,
-                        preexec_fn=subprocess_setup)
-                fnull.close()
-                if ret != 0:
-                    self.log.warning(
-                        'SPR %d (%s %s) failed to unpack: returned %d' % (
-                            spr.id, spr.name, spr.version, ret))
-                    continue
-
-                # We have an extracted source package. Let's get the
-                # changelog. findFile ensures that it's not too huge, and
-                # not a symlink.
-                try:
-                    changelog_path = findFile(
-                        tmp_dir.path, 'debian/changelog')
-                except UploadError, e:
-                    changelog_path = None
-                    self.log.warning(
-                        'SPR %d (%s %s) changelog could not be '
-                        'imported: %s' % (
-                            spr.id, spr.name, spr.version, e))
-                if changelog_path:
-                    # The LFA should be restricted only if there aren't any
-                    # public publications.
-                    restricted = not any(
-                        not a.private for a in spr.published_archives)
-                    spr.changelog = getUtility(ILibraryFileAliasSet).create(
-                        'changelog',
-                        os.stat(changelog_path).st_size,
-                        open(changelog_path, "r"),
-                        "text/x-debian-source-changelog",
-                        restricted=restricted)
-                    self.log.info('SPR %d (%s %s) changelog imported.' % (
-                        spr.id, spr.name, spr.version))
-                else:
-                    self.log.warning('SPR %d (%s %s) had no changelog.' % (
-                        spr.id, spr.name, spr.version))
-
-        self.start_at = spr.id + 1
-        result = getUtility(IMemcacheClient).set(
-            'populate-spr-changelogs', self.start_at)
-        if not result:
-            self.log.warning('Failed to set start_at in memcache.')
-        transaction.commit()
-
-
 class BaseDatabaseGarbageCollector(LaunchpadCronScript):
     """Abstract base class to run a collection of TunableLoops."""
     script_name = None # Script name for locking and database user. Override.
@@ -1122,7 +986,6 @@
         UnusedSessionPruner,
         DuplicateSessionPruner,
         BugHeatUpdater,
-        PopulateSPRChangelogs,
         ]
     experimental_tunable_loops = []
 

=== modified file 'lib/lp/scripts/tests/test_garbo.py'
--- lib/lp/scripts/tests/test_garbo.py	2011-04-08 14:10:32 +0000
+++ lib/lp/scripts/tests/test_garbo.py	2011-04-15 05:03:31 +0000
@@ -10,11 +10,8 @@
     datetime,
     timedelta,
     )
-from fixtures import TempDir
 import logging
-import os
 from StringIO import StringIO
-import subprocess
 import time
 
 from pytz import UTC
@@ -42,7 +39,6 @@
 from canonical.launchpad.database.oauth import OAuthNonce
 from canonical.launchpad.database.openidconsumer import OpenIDConsumerNonce
 from canonical.launchpad.interfaces.emailaddress import EmailAddressStatus
-from canonical.launchpad.interfaces.librarian import ILibraryFileAliasSet
 from canonical.launchpad.interfaces.lpstorm import IMasterStore
 from canonical.launchpad.scripts.tests import run_script
 from canonical.launchpad.webapp.interfaces import (
@@ -56,7 +52,6 @@
     LaunchpadZopelessLayer,
     ZopelessDatabaseLayer,
     )
-from lp.archiveuploader.dscfile import findFile
 from lp.bugs.model.bugnotification import (
     BugNotification,
     BugNotificationRecipient,
@@ -73,7 +68,6 @@
     )
 from lp.code.model.codeimportevent import CodeImportEvent
 from lp.code.model.codeimportresult import CodeImportResult
-from lp.registry.interfaces.distribution import IDistributionSet
 from lp.registry.interfaces.person import (
     IPersonSet,
     PersonCreationRationale,
@@ -93,8 +87,6 @@
     SessionData,
     SessionPkgData,
     )
-from lp.soyuz.enums import PackagePublishingStatus
-from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
 from lp.testing import (
     TestCase,
     TestCaseWithFactory,
@@ -114,13 +106,6 @@
 
     def test_hourly_script(self):
         """Ensure garbo-hourly.py actually runs."""
-        # Our sampledata doesn't contain anything that PopulateSPRChangelogs
-        # can process without errors, so it's easier to just set all of the
-        # changelogs to a random LFA. We can't just expire every LFA, since
-        # a bunch of SPRs have no SPRFs at all.
-        IMasterStore(SourcePackageRelease).find(SourcePackageRelease).set(
-            changelogID=1)
-        transaction.commit() # run_script() is a different process.
         rv, out, err = run_script(
             "cronscripts/garbo-hourly.py", ["-q"], expect_returncode=0)
         self.failIf(out.strip(), "Output to stdout: %s" % out)
@@ -889,60 +874,3 @@
             """ % sqlbase.quote(template.id)).get_one()
 
         self.assertEqual(1, count)
-
-    def upload_to_debian(self, restricted=False):
-        sid = getUtility(IDistributionSet)['debian']['sid']
-        spn = self.factory.makeSourcePackageName('9wm')
-        spr = self.factory.makeSourcePackageRelease(
-            sourcepackagename=spn, version='1.2-7', distroseries=sid)
-        archive = sid.main_archive
-        if restricted:
-            archive = self.factory.makeArchive(
-                distribution=sid.distribution, private=True)
-        self.factory.makeSourcePackagePublishingHistory(
-            sourcepackagerelease=spr, archive=archive,
-            status=PackagePublishingStatus.PUBLISHED)
-        for name in (
-            '9wm_1.2-7.diff.gz', '9wm_1.2.orig.tar.gz', '9wm_1.2-7.dsc'):
-            path = os.path.join(
-                'lib/lp/soyuz/scripts/tests/gina_test_archive/pool/main/9',
-                '9wm', name)
-            lfa = getUtility(ILibraryFileAliasSet).create(
-                name, os.stat(path).st_size, open(path, 'r'),
-                'application/octet-stream', restricted=restricted)
-            spr.addFile(lfa)
-        with TempDir() as tmp_dir:
-            fnull = open('/dev/null', 'w')
-            ret = subprocess.call(
-                ['dpkg-source', '-x', path, os.path.join(
-                    tmp_dir.path, 'extracted')],
-                    stdout=fnull, stderr=fnull)
-            fnull.close()
-            self.assertEqual(0, ret)
-            changelog_path = findFile(tmp_dir.path, 'debian/changelog')
-            changelog = open(changelog_path, 'r').read()
-        transaction.commit() # .runHourly() switches dbuser.
-        return (spr, changelog)
-
-    def test_populateSPRChangelogs(self):
-        # We set SPR.changelog for imported records from Debian.
-        LaunchpadZopelessLayer.switchDbUser('testadmin')
-        spr, changelog = self.upload_to_debian()
-        collector = self.runHourly()
-        log = self.log_buffer.getvalue()
-        self.assertTrue(
-            'SPR %d (9wm 1.2-7) changelog imported.' % spr.id in log)
-        self.assertFalse(spr.changelog == None)
-        self.assertFalse(spr.changelog.restricted)
-        self.assertEqual(changelog, spr.changelog.read())
-
-    def test_populateSPRChangelogs_restricted_sprf(self):
-        LaunchpadZopelessLayer.switchDbUser('testadmin')
-        spr, changelog = self.upload_to_debian(restricted=True)
-        collector = self.runHourly()
-        log = self.log_buffer.getvalue()
-        self.assertTrue(
-            'SPR %d (9wm 1.2-7) changelog imported.' % spr.id in log)
-        self.assertFalse(spr.changelog == None)
-        self.assertTrue(spr.changelog.restricted)
-        self.assertEqual(changelog, spr.changelog.read())