-----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 Tom Haddon wrote: > Tom Haddon has proposed merging lp:~mthaddon/launchpad/buildjob-nagios-check-perms into lp:launchpad. > > Requested reviews: > Canonical Launchpad Engineering (launchpad) > > > Permissions needed by a nagios check for whether there are build jobs that have started but don't have a builder assigned - obsolete once bug 499421 is fixed > > I need to ask some questions because this isn't in our standard format for review requests: 1. Was there a pre-implementation call? 2. Did you run "make lint", and was the output clean? > === modified file 'cronscripts/librarian-gc.py' > --- cronscripts/librarian-gc.py 2009-11-25 08:53:00 +0000 > +++ cronscripts/librarian-gc.py 2009-12-23 11:42:26 +0000 > @@ -51,6 +51,12 @@ > help="Skip removing files on disk with no database references" > " or flagged for deletion." > ) > + self.parser.add_option( > + '', "--skip-expiry", action="store_true", default=False, > + dest="skip_expiry", > + help="Skip expiring aliases with an expiry date in the past." > + ) How does this relate to the permissions issue? > + > > def main(self): > librariangc.log = self.logger > @@ -66,6 +72,8 @@ > > # Note that each of these next steps will issue commit commands > # as appropriate to make this script transaction friendly > + if not self.options.skip_expiry: > + librariangc.expire_aliases(conn) > if not self.options.skip_content: > librariangc.delete_unreferenced_content(conn) # first sweep > if not self.options.skip_blobs: > > === modified file 'database/schema/security.cfg' > --- database/schema/security.cfg 2009-12-22 09:37:00 +0000 > +++ database/schema/security.cfg 2009-12-23 11:42:26 +0000 > @@ -1838,6 +1838,11 @@ > > [nagios] > type=user > +public.archive = SELECT > +public.build = SELECT > +public.buildqueue = SELECT > +public.buildpackagejob = SELECT > +public.job = SELECT > public.libraryfilecontent = SELECT > public.openidrpconfig = SELECT > public.branch = SELECT > > === modified file 'lib/canonical/librarian/ftests/test_gc.py' > --- lib/canonical/librarian/ftests/test_gc.py 2009-11-25 08:53:00 +0000 > +++ lib/canonical/librarian/ftests/test_gc.py 2009-12-23 11:42:26 +0000 > @@ -55,6 +55,22 @@ > self.client = LibrarianClient() > librariangc.log = MockLogger() > > + # A value we use in a number of tests. This represents the > + # stay of execution hard coded into the garbage collector. > + # We don't destroy any data unless it has been waiting to be > + # destroyed for longer than this period. We pick a value > + # that is close enough to the stay of execution so that > + # forgetting timezone information will break things, but > + # far enough so that how long it takes the test to run > + # is not an issue. 'stay_of_excution - 1 hour' fits these > + # criteria. > + self.recent_past = ( > + datetime.utcnow().replace(tzinfo=utc) > + - timedelta(days=6, hours=23)) > + # A time beyond the stay of execution. > + self.ancient_past = ( > + datetime.utcnow().replace(tzinfo=utc) - timedelta(days=30)) > + > self.f1_id, self.f2_id = self._makeDupes() > > self.layer.switchDbUser(config.librarian_gc.dbuser) > @@ -66,12 +82,6 @@ > path = librariangc.get_file_path(self.f1_id) > self.failUnless(os.path.exists(path), "Librarian uploads failed") > > - # A value we use in a number of tests > - self.recent_past = ( > - datetime.utcnow().replace(tzinfo=utc) > - - timedelta(days=6, hours=23) > - ) > - > # Make sure that every file the database knows about exists on disk. > # We manually remove them for tests that need to cope with missing > # library items. > @@ -124,14 +134,12 @@ > > # Set the last accessed time into the past so they will be garbage > # collected > - past = datetime.utcnow() - timedelta(days=30) > - past = past.replace(tzinfo=utc) > - f1.last_accessed = past > - f2.last_accessed = past > - f1.date_created = past > - f2.date_created = past > - f1.content.datecreated = past > - f2.content.datecreated = past > + f1.last_accessed = self.ancient_past > + f2.last_accessed = self.ancient_past > + f1.date_created = self.ancient_past > + f2.date_created = self.ancient_past > + f1.content.datecreated = self.ancient_past > + f2.content.datecreated = self.ancient_past > > del f1, f2 > > @@ -219,8 +227,7 @@ > # Flag one of our LibraryFileAliases with an expiry date in the past > self.ztm.begin() > f1 = LibraryFileAlias.get(self.f1_id) > - past = datetime.utcnow().replace(tzinfo=utc) - timedelta(days=30) > - f1.expires = past > + f1.expires = self.ancient_past > del f1 > self.ztm.commit() > > @@ -261,6 +268,52 @@ > # Our recently expired LibraryFileAlias is still available. > LibraryFileAlias.get(self.f1_id) > > + def test_deleteWellExpiredAliases(self): > + # LibraryFileAlias records that are expired are unlinked from their > + # content. > + > + # Flag one of our LibraryFileAliases with an expiry date in the past > + self.ztm.begin() > + f1 = LibraryFileAlias.get(self.f1_id) > + f1.expires = self.ancient_past > + del f1 > + self.ztm.commit() > + > + # Unlink expired LibraryFileAliases. > + librariangc.expire_aliases(self.con) > + > + self.ztm.begin() > + # Make sure the well expired f1 is still there, but has no content. > + f1 = LibraryFileAlias.get(self.f1_id) > + self.assert_(f1.content is None) > + # f2 should still have content, as it isn't flagged for expiry. > + f2 = LibraryFileAlias.get(self.f2_id) > + self.assert_(f2.content is not None) > + > + def test_ignoreRecentlyExpiredAliases(self): > + # LibraryFileAlias records that have expired recently are not > + # garbage collected. > + > + # Flag one of our LibraryFileAliases with an expiry date in the > + # recent past. > + self.ztm.begin() > + f1 = LibraryFileAlias.get(self.f1_id) > + f1.expires = self.recent_past # Within stay of execution. > + del f1 > + self.ztm.commit() > + > + # Unlink expired LibraryFileAliases. > + librariangc.expire_aliases(self.con) > + > + self.ztm.begin() > + # Make sure f1 is still there and has content. This ensures that > + # our stay of execution is still working. > + f1 = LibraryFileAlias.get(self.f1_id) > + self.assert_(f1.content is not None) > + # f2 should still have content, as it isn't flagged for expiry. > + f2 = LibraryFileAlias.get(self.f2_id) > + self.assert_(f2.content is not None) > + > def test_DeleteUnreferencedContent(self): > # Merge the duplicates. This creates an > # unreferenced LibraryFileContent > How do these changes relate to the permission changes? > === modified file 'lib/canonical/librarian/librariangc.py' > --- lib/canonical/librarian/librariangc.py 2009-11-25 13:47:07 +0000 > +++ lib/canonical/librarian/librariangc.py 2009-12-23 11:42:26 +0000 > @@ -182,6 +182,55 @@ > con.commit() > > > +class ExpireAliases: > + """Expire expired LibraryFileAlias records. > + > + This simply involves setting the LibraryFileAlias.content to NULL. > + Unreferenced LibraryFileContent records are cleaned up elsewhere. > + """ > + implements(ITunableLoop) > + > + def __init__(self, con): > + self.con = con > + self.total_expired = 0 > + self._done = False > + > + def isDone(self): > + if self._done: > + log.info( > + "Expired %d LibraryFileAlias records." % self.total_expired) > + return True > + else: > + return False > + > + def __call__(self, chunksize): > + chunksize = int(chunksize) > + cur = self.con.cursor() > + cur.execute(""" > + UPDATE LibraryFileAlias > + SET content=NULL > + WHERE id IN ( > + SELECT id FROM LibraryFileAlias > + WHERE > + content IS NOT NULL > + AND expires < CURRENT_TIMESTAMP AT TIME ZONE 'UTC' > + - interval '1 week' > + LIMIT %d) > + """ % chunksize) > + self.total_expired += cur.rowcount > + if cur.rowcount == 0: > + self._done = True > + else: > + log.debug("Expired %d LibraryFileAlias records." % cur.rowcount) > + self.con.commit() > + > + > +def expire_aliases(con): > + """Invoke ExpireLibraryFileAliases.""" > + loop_tuner = DBLoopTuner(ExpireAliases(con), 5, log=log) > + loop_tuner.run() > + > + > class UnreferencedLibraryFileAliasPruner: > """Delete unreferenced LibraryFileAliases. > > How does this change relate to the permission changes? > === renamed directory 'lib/lp/archiveuploader/tests/data/suite/debug_1.0' => 'lib/lp/archiveuploader/tests/data/suite/debug_1.0-1' > === added file 'lib/lp/archiveuploader/tests/data/suite/debug_1.0-1/debug-bin-dbgsym_1.0-1_i386.ddeb' > Binary files lib/lp/archiveuploader/tests/data/suite/debug_1.0-1/debug-bin-dbgsym_1.0-1_i386.ddeb 1970-01-01 00:00:00 +0000 and lib/lp/archiveuploader/tests/data/suite/debug_1.0-1/debug-bin-dbgsym_1.0-1_i386.ddeb 2009-12-23 11:42:26 +0000 differ > === renamed file 'lib/lp/archiveuploader/tests/data/suite/debug_1.0/debug-bin_1.0_i386.deb' => 'lib/lp/archiveuploader/tests/data/suite/debug_1.0-1/debug-bin_1.0-1_i386.deb' > Binary files lib/lp/archiveuploader/tests/data/suite/debug_1.0/debug-bin_1.0_i386.deb 2009-04-15 13:45:42 +0000 and lib/lp/archiveuploader/tests/data/suite/debug_1.0-1/debug-bin_1.0-1_i386.deb 2009-12-23 11:42:26 +0000 differ > === renamed file 'lib/lp/archiveuploader/tests/data/suite/debug_1.0/debug_1.0.dsc' => 'lib/lp/archiveuploader/tests/data/suite/debug_1.0-1/debug_1.0-1.dsc' > --- lib/lp/archiveuploader/tests/data/suite/debug_1.0/debug_1.0.dsc 2009-04-15 13:45:42 +0000 > +++ lib/lp/archiveuploader/tests/data/suite/debug_1.0-1/debug_1.0-1.dsc 2009-12-23 11:42:26 +0000 > @@ -1,13 +1,13 @@ > Format: 1.0 > Source: debug > -Binary: debug-bin, debug-dbg > +Binary: debug-bin, debug-bin-dbgsym > Architecture: any > -Version: 1.0 > +Version: 1.0-1 > Maintainer: Launchpad team