Launchpad itself

Merge lp:~stevenk/launchpad/drop-psc into lp:launchpad

  1. drop-psc
  2. Merge into devel
Proposed by Steve Kowalik
Status: Merged
Approved by: Robert Collins
Approved revision: no longer in the source branch.
Merged at revision: 12850
Proposed branch: lp:~stevenk/launchpad/drop-psc
Merge into: lp:launchpad
Diff against target: 340 lines (+0/-212)
3 files modified
database/schema/security.cfg (+0/-3)
lib/lp/scripts/garbo.py (+0/-137)
lib/lp/scripts/tests/test_garbo.py (+0/-72)
To merge this branch: bzr merge lp:~stevenk/launchpad/drop-psc
Reviewer Review Type Date Requested Status
Robert Collins (community) Approve
William Grant code* Approve
Review via email: mp+57807@code.launchpad.net

Commit message

[r=lifeless,wgrant][bug=764292] Drop PopulateSPRChangelogs to thank it for its work.

Description of the change

After running for one month, and processing the changelogs for as-near-as-makes-no-difference half a million SPRs, the PopulateSPRChangelogs class in the hourly garbo script has done its dash. So to thank it, we kick it out of the code base.

To post a comment you must log in.
Revision history for this message
Robert Collins (lifeless) wrote :

You should wait till staging and qastaging are fully migrated too.
Otherwise bad things will happen as you qa stuff :).

Revision history for this message
William Grant (wgrant) wrote :

Looks good. No need to wait.

review: Approve (code*)
Revision history for this message
Robert Collins (lifeless) wrote :

shrug, sure.

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'database/schema/security.cfg'
--- database/schema/security.cfg 2011-04-06 15:35:05 +0000
+++ database/schema/security.cfg 2011-04-15 05:03:31 +0000
@@ -2260,9 +2260,6 @@
2260public.job = SELECT, INSERT, DELETE2260public.job = SELECT, INSERT, DELETE
2261public.branchjob = SELECT, DELETE2261public.branchjob = SELECT, DELETE
2262public.bugjob = SELECT, INSERT2262public.bugjob = SELECT, INSERT
2263public.libraryfilealias = SELECT, INSERT
2264public.libraryfilecontent = SELECT, INSERT
2265public.sourcepackagerelease = SELECT, UPDATE
22662263
2267[garbo_daily]2264[garbo_daily]
2268type=user2265type=user
22692266
=== modified file 'lib/lp/scripts/garbo.py'
--- lib/lp/scripts/garbo.py 2011-04-12 08:58:38 +0000
+++ lib/lp/scripts/garbo.py 2011-04-15 05:03:31 +0000
@@ -13,21 +13,14 @@
13 datetime,13 datetime,
14 timedelta,14 timedelta,
15 )15 )
16from fixtures import TempDir
17import logging16import logging
18import multiprocessing17import multiprocessing
19import os
20import signal
21import subprocess
22import threading18import threading
23import time19import time
2420
25from psycopg2 import IntegrityError21from psycopg2 import IntegrityError
26import pytz22import pytz
27from storm.expr import LeftJoin
28from storm.locals import (23from storm.locals import (
29 And,
30 Count,
31 Max,24 Max,
32 Min,25 Min,
33 SQL,26 SQL,
@@ -51,7 +44,6 @@
51from canonical.launchpad.database.oauth import OAuthNonce44from canonical.launchpad.database.oauth import OAuthNonce
52from canonical.launchpad.database.openidconsumer import OpenIDConsumerNonce45from canonical.launchpad.database.openidconsumer import OpenIDConsumerNonce
53from canonical.launchpad.interfaces.emailaddress import EmailAddressStatus46from canonical.launchpad.interfaces.emailaddress import EmailAddressStatus
54from canonical.launchpad.interfaces.librarian import ILibraryFileAliasSet
55from canonical.launchpad.interfaces.lpstorm import IMasterStore47from canonical.launchpad.interfaces.lpstorm import IMasterStore
56from canonical.launchpad.utilities.looptuner import TunableLoop48from canonical.launchpad.utilities.looptuner import TunableLoop
57from canonical.launchpad.webapp.interfaces import (49from canonical.launchpad.webapp.interfaces import (
@@ -59,9 +51,6 @@
59 MAIN_STORE,51 MAIN_STORE,
60 MASTER_FLAVOR,52 MASTER_FLAVOR,
61 )53 )
62from canonical.librarian.utils import copy_and_close
63from lp.archiveuploader.dscfile import findFile
64from lp.archiveuploader.nascentuploadfile import UploadError
65from lp.bugs.interfaces.bug import IBugSet54from lp.bugs.interfaces.bug import IBugSet
66from lp.bugs.model.bug import Bug55from lp.bugs.model.bug import Bug
67from lp.bugs.model.bugattachment import BugAttachment56from lp.bugs.model.bugattachment import BugAttachment
@@ -82,14 +71,11 @@
82from lp.registry.model.person import Person71from lp.registry.model.person import Person
83from lp.services.job.model.job import Job72from lp.services.job.model.job import Job
84from lp.services.log.logger import PrefixFilter73from lp.services.log.logger import PrefixFilter
85from lp.services.memcache.interfaces import IMemcacheClient
86from lp.services.scripts.base import (74from lp.services.scripts.base import (
87 LaunchpadCronScript,75 LaunchpadCronScript,
88 SilentLaunchpadScriptFailure,76 SilentLaunchpadScriptFailure,
89 )77 )
90from lp.services.session.model import SessionData78from lp.services.session.model import SessionData
91from lp.soyuz.model.files import SourcePackageReleaseFile
92from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
93from lp.translations.interfaces.potemplate import IPOTemplateSet79from lp.translations.interfaces.potemplate import IPOTemplateSet
94from lp.translations.model.potranslation import POTranslation80from lp.translations.model.potranslation import POTranslation
9581
@@ -97,10 +83,6 @@
97ONE_DAY_IN_SECONDS = 24*60*6083ONE_DAY_IN_SECONDS = 24*60*60
9884
9985
100def subprocess_setup():
101 signal.signal(signal.SIGPIPE, signal.SIG_DFL)
102
103
104class BulkPruner(TunableLoop):86class BulkPruner(TunableLoop):
105 """A abstract ITunableLoop base class for simple pruners.87 """A abstract ITunableLoop base class for simple pruners.
10688
@@ -839,124 +821,6 @@
839 self.done = True821 self.done = True
840822
841823
842class PopulateSPRChangelogs(TunableLoop):
843 maximum_chunk_size = 1
844
845 def __init__(self, log, abort_time=None):
846 super(PopulateSPRChangelogs, self).__init__(log, abort_time)
847 value = getUtility(IMemcacheClient).get('populate-spr-changelogs')
848 if not value:
849 self.start_at = 0
850 else:
851 self.start_at = value
852 self.finish_at = self.getCandidateSPRs(0).last()
853
854 def getCandidateSPRs(self, start_at):
855 return IMasterStore(SourcePackageRelease).using(
856 SourcePackageRelease,
857 # Find any SPRFs that have expired (LFA.content IS NULL).
858 LeftJoin(
859 SourcePackageReleaseFile,
860 SourcePackageReleaseFile.sourcepackagereleaseID ==
861 SourcePackageRelease.id),
862 LeftJoin(
863 LibraryFileAlias,
864 And(LibraryFileAlias.id ==
865 SourcePackageReleaseFile.libraryfileID,
866 LibraryFileAlias.content == None)),
867 # And exclude any SPRs that have any expired SPRFs.
868 ).find(
869 SourcePackageRelease.id,
870 SourcePackageRelease.id >= start_at,
871 SourcePackageRelease.changelog == None,
872 ).group_by(SourcePackageRelease.id).having(
873 Count(LibraryFileAlias) == 0
874 ).order_by(SourcePackageRelease.id)
875
876 def isDone(self):
877 return self.start_at > self.finish_at
878
879 def __call__(self, chunk_size):
880 for sprid in self.getCandidateSPRs(self.start_at)[:chunk_size]:
881 spr = SourcePackageRelease.get(sprid)
882 with TempDir() as tmp_dir:
883 dsc_file = None
884
885 # Grab the files from the librarian into a temporary
886 # directory.
887 try:
888 for sprf in spr.files:
889 dest = os.path.join(
890 tmp_dir.path, sprf.libraryfile.filename)
891 dest_file = open(dest, 'w')
892 sprf.libraryfile.open()
893 copy_and_close(sprf.libraryfile, dest_file)
894 if dest.endswith('.dsc'):
895 dsc_file = dest
896 except LookupError:
897 self.log.warning(
898 'SPR %d (%s %s) has missing library files.' % (
899 spr.id, spr.name, spr.version))
900 continue
901
902 if dsc_file is None:
903 self.log.warning(
904 'SPR %d (%s %s) has no DSC.' % (
905 spr.id, spr.name, spr.version))
906 continue
907
908 # Extract the source package. Throw away stdout/stderr
909 # -- we only really care about the return code.
910 fnull = open('/dev/null', 'w')
911 ret = subprocess.call(
912 ['dpkg-source', '-x', dsc_file, os.path.join(
913 tmp_dir.path, 'extracted')],
914 stdout=fnull, stderr=fnull,
915 preexec_fn=subprocess_setup)
916 fnull.close()
917 if ret != 0:
918 self.log.warning(
919 'SPR %d (%s %s) failed to unpack: returned %d' % (
920 spr.id, spr.name, spr.version, ret))
921 continue
922
923 # We have an extracted source package. Let's get the
924 # changelog. findFile ensures that it's not too huge, and
925 # not a symlink.
926 try:
927 changelog_path = findFile(
928 tmp_dir.path, 'debian/changelog')
929 except UploadError, e:
930 changelog_path = None
931 self.log.warning(
932 'SPR %d (%s %s) changelog could not be '
933 'imported: %s' % (
934 spr.id, spr.name, spr.version, e))
935 if changelog_path:
936 # The LFA should be restricted only if there aren't any
937 # public publications.
938 restricted = not any(
939 not a.private for a in spr.published_archives)
940 spr.changelog = getUtility(ILibraryFileAliasSet).create(
941 'changelog',
942 os.stat(changelog_path).st_size,
943 open(changelog_path, "r"),
944 "text/x-debian-source-changelog",
945 restricted=restricted)
946 self.log.info('SPR %d (%s %s) changelog imported.' % (
947 spr.id, spr.name, spr.version))
948 else:
949 self.log.warning('SPR %d (%s %s) had no changelog.' % (
950 spr.id, spr.name, spr.version))
951
952 self.start_at = spr.id + 1
953 result = getUtility(IMemcacheClient).set(
954 'populate-spr-changelogs', self.start_at)
955 if not result:
956 self.log.warning('Failed to set start_at in memcache.')
957 transaction.commit()
958
959
960class BaseDatabaseGarbageCollector(LaunchpadCronScript):824class BaseDatabaseGarbageCollector(LaunchpadCronScript):
961 """Abstract base class to run a collection of TunableLoops."""825 """Abstract base class to run a collection of TunableLoops."""
962 script_name = None # Script name for locking and database user. Override.826 script_name = None # Script name for locking and database user. Override.
@@ -1122,7 +986,6 @@
1122 UnusedSessionPruner,986 UnusedSessionPruner,
1123 DuplicateSessionPruner,987 DuplicateSessionPruner,
1124 BugHeatUpdater,988 BugHeatUpdater,
1125 PopulateSPRChangelogs,
1126 ]989 ]
1127 experimental_tunable_loops = []990 experimental_tunable_loops = []
1128991
1129992
=== modified file 'lib/lp/scripts/tests/test_garbo.py'
--- lib/lp/scripts/tests/test_garbo.py 2011-04-08 14:10:32 +0000
+++ lib/lp/scripts/tests/test_garbo.py 2011-04-15 05:03:31 +0000
@@ -10,11 +10,8 @@
10 datetime,10 datetime,
11 timedelta,11 timedelta,
12 )12 )
13from fixtures import TempDir
14import logging13import logging
15import os
16from StringIO import StringIO14from StringIO import StringIO
17import subprocess
18import time15import time
1916
20from pytz import UTC17from pytz import UTC
@@ -42,7 +39,6 @@
42from canonical.launchpad.database.oauth import OAuthNonce39from canonical.launchpad.database.oauth import OAuthNonce
43from canonical.launchpad.database.openidconsumer import OpenIDConsumerNonce40from canonical.launchpad.database.openidconsumer import OpenIDConsumerNonce
44from canonical.launchpad.interfaces.emailaddress import EmailAddressStatus41from canonical.launchpad.interfaces.emailaddress import EmailAddressStatus
45from canonical.launchpad.interfaces.librarian import ILibraryFileAliasSet
46from canonical.launchpad.interfaces.lpstorm import IMasterStore42from canonical.launchpad.interfaces.lpstorm import IMasterStore
47from canonical.launchpad.scripts.tests import run_script43from canonical.launchpad.scripts.tests import run_script
48from canonical.launchpad.webapp.interfaces import (44from canonical.launchpad.webapp.interfaces import (
@@ -56,7 +52,6 @@
56 LaunchpadZopelessLayer,52 LaunchpadZopelessLayer,
57 ZopelessDatabaseLayer,53 ZopelessDatabaseLayer,
58 )54 )
59from lp.archiveuploader.dscfile import findFile
60from lp.bugs.model.bugnotification import (55from lp.bugs.model.bugnotification import (
61 BugNotification,56 BugNotification,
62 BugNotificationRecipient,57 BugNotificationRecipient,
@@ -73,7 +68,6 @@
73 )68 )
74from lp.code.model.codeimportevent import CodeImportEvent69from lp.code.model.codeimportevent import CodeImportEvent
75from lp.code.model.codeimportresult import CodeImportResult70from lp.code.model.codeimportresult import CodeImportResult
76from lp.registry.interfaces.distribution import IDistributionSet
77from lp.registry.interfaces.person import (71from lp.registry.interfaces.person import (
78 IPersonSet,72 IPersonSet,
79 PersonCreationRationale,73 PersonCreationRationale,
@@ -93,8 +87,6 @@
93 SessionData,87 SessionData,
94 SessionPkgData,88 SessionPkgData,
95 )89 )
96from lp.soyuz.enums import PackagePublishingStatus
97from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
98from lp.testing import (90from lp.testing import (
99 TestCase,91 TestCase,
100 TestCaseWithFactory,92 TestCaseWithFactory,
@@ -114,13 +106,6 @@
114106
115 def test_hourly_script(self):107 def test_hourly_script(self):
116 """Ensure garbo-hourly.py actually runs."""108 """Ensure garbo-hourly.py actually runs."""
117 # Our sampledata doesn't contain anything that PopulateSPRChangelogs
118 # can process without errors, so it's easier to just set all of the
119 # changelogs to a random LFA. We can't just expire every LFA, since
120 # a bunch of SPRs have no SPRFs at all.
121 IMasterStore(SourcePackageRelease).find(SourcePackageRelease).set(
122 changelogID=1)
123 transaction.commit() # run_script() is a different process.
124 rv, out, err = run_script(109 rv, out, err = run_script(
125 "cronscripts/garbo-hourly.py", ["-q"], expect_returncode=0)110 "cronscripts/garbo-hourly.py", ["-q"], expect_returncode=0)
126 self.failIf(out.strip(), "Output to stdout: %s" % out)111 self.failIf(out.strip(), "Output to stdout: %s" % out)
@@ -889,60 +874,3 @@
889 """ % sqlbase.quote(template.id)).get_one()874 """ % sqlbase.quote(template.id)).get_one()
890875
891 self.assertEqual(1, count)876 self.assertEqual(1, count)
892
893 def upload_to_debian(self, restricted=False):
894 sid = getUtility(IDistributionSet)['debian']['sid']
895 spn = self.factory.makeSourcePackageName('9wm')
896 spr = self.factory.makeSourcePackageRelease(
897 sourcepackagename=spn, version='1.2-7', distroseries=sid)
898 archive = sid.main_archive
899 if restricted:
900 archive = self.factory.makeArchive(
901 distribution=sid.distribution, private=True)
902 self.factory.makeSourcePackagePublishingHistory(
903 sourcepackagerelease=spr, archive=archive,
904 status=PackagePublishingStatus.PUBLISHED)
905 for name in (
906 '9wm_1.2-7.diff.gz', '9wm_1.2.orig.tar.gz', '9wm_1.2-7.dsc'):
907 path = os.path.join(
908 'lib/lp/soyuz/scripts/tests/gina_test_archive/pool/main/9',
909 '9wm', name)
910 lfa = getUtility(ILibraryFileAliasSet).create(
911 name, os.stat(path).st_size, open(path, 'r'),
912 'application/octet-stream', restricted=restricted)
913 spr.addFile(lfa)
914 with TempDir() as tmp_dir:
915 fnull = open('/dev/null', 'w')
916 ret = subprocess.call(
917 ['dpkg-source', '-x', path, os.path.join(
918 tmp_dir.path, 'extracted')],
919 stdout=fnull, stderr=fnull)
920 fnull.close()
921 self.assertEqual(0, ret)
922 changelog_path = findFile(tmp_dir.path, 'debian/changelog')
923 changelog = open(changelog_path, 'r').read()
924 transaction.commit() # .runHourly() switches dbuser.
925 return (spr, changelog)
926
927 def test_populateSPRChangelogs(self):
928 # We set SPR.changelog for imported records from Debian.
929 LaunchpadZopelessLayer.switchDbUser('testadmin')
930 spr, changelog = self.upload_to_debian()
931 collector = self.runHourly()
932 log = self.log_buffer.getvalue()
933 self.assertTrue(
934 'SPR %d (9wm 1.2-7) changelog imported.' % spr.id in log)
935 self.assertFalse(spr.changelog == None)
936 self.assertFalse(spr.changelog.restricted)
937 self.assertEqual(changelog, spr.changelog.read())
938
939 def test_populateSPRChangelogs_restricted_sprf(self):
940 LaunchpadZopelessLayer.switchDbUser('testadmin')
941 spr, changelog = self.upload_to_debian(restricted=True)
942 collector = self.runHourly()
943 log = self.log_buffer.getvalue()
944 self.assertTrue(
945 'SPR %d (9wm 1.2-7) changelog imported.' % spr.id in log)
946 self.assertFalse(spr.changelog == None)
947 self.assertTrue(spr.changelog.restricted)
948 self.assertEqual(changelog, spr.changelog.read())