Merge lp:~cjwatson/launchpad/livefs-keep-base-images into lp:launchpad

Proposed by Colin Watson on 2019-06-12
Status: Merged
Merged at revision: 19006
Proposed branch: lp:~cjwatson/launchpad/livefs-keep-base-images
Merge into: lp:launchpad
Prerequisite: lp:~cjwatson/launchpad/livefs-keep-binary-files-interval
Diff against target: 121 lines (+55/-12)
2 files modified
lib/lp/scripts/garbo.py (+15/-4)
lib/lp/scripts/tests/test_garbo.py (+40/-8)
To merge this branch: bzr merge lp:~cjwatson/launchpad/livefs-keep-base-images
Reviewer Review Type Date Requested Status
William Grant code 2019-06-12 Approve on 2019-07-02
Review via email: mp+368714@code.launchpad.net

Commit message

Exclude files that are set as base images from LiveFSFile pruning.

To post a comment you must log in.
William Grant (wgrant) :
review: Approve (code)
18986. By Colin Watson on 2019-07-02

Expand LiveFSFilePruner docstring a bit.

Colin Watson (cjwatson) :

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'lib/lp/scripts/garbo.py'
--- lib/lp/scripts/garbo.py 2019-07-02 16:21:44 +0000
+++ lib/lp/scripts/garbo.py 2019-07-02 16:21:44 +0000
@@ -1567,10 +1567,17 @@
1567 """A BulkPruner to remove old `LiveFSFile`s.1567 """A BulkPruner to remove old `LiveFSFile`s.
15681568
1569 We remove binary files attached to `LiveFSBuild`s that are more than1569 We remove binary files attached to `LiveFSBuild`s that are more than
1570 `LiveFS.keep_binary_files_interval` old; these files are very large and1570 `LiveFS.keep_binary_files_interval` old and that are not set as base
1571 are only useful for builds in progress. Text files are typically small1571 images for a `DistroArchSeries`; these files are very large and are only
1572 (<1MiB) and useful for retrospective analysis, so we preserve those1572 useful for builds in progress.
1573 indefinitely.1573
1574 DAS base images are excluded because
1575 `DistroArchSeries.setChrootFromBuild` takes a `LiveFSBuild` and we want
1576 to have the option of reverting to a previous base image shortly after
1577 upgrading to a newer one.
1578
1579 Text files are typically small (<1MiB) and useful for retrospective
1580 analysis, so we preserve those indefinitely.
1574 """1581 """
1575 target_table_class = LiveFSFile1582 target_table_class = LiveFSFile
1576 # Note that a NULL keep_binary_files_interval disables pruning, due to1583 # Note that a NULL keep_binary_files_interval disables pruning, due to
@@ -1586,6 +1593,10 @@
1586 CURRENT_TIMESTAMP AT TIME ZONE 'UTC'1593 CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
1587 - LiveFS.keep_binary_files_interval1594 - LiveFS.keep_binary_files_interval
1588 AND LibraryFileAlias.mimetype != 'text/plain'1595 AND LibraryFileAlias.mimetype != 'text/plain'
1596 EXCEPT
1597 SELECT LiveFSFile.id
1598 FROM LiveFSFile, PocketChroot
1599 WHERE LiveFSFile.libraryfile = PocketChroot.chroot
1589 """1600 """
15901601
15911602
15921603
=== modified file 'lib/lp/scripts/tests/test_garbo.py'
--- lib/lp/scripts/tests/test_garbo.py 2019-07-02 16:21:44 +0000
+++ lib/lp/scripts/tests/test_garbo.py 2019-07-02 16:21:44 +0000
@@ -1518,33 +1518,40 @@
15181518
1519 def _test_LiveFSFilePruner(self, content_type, interval,1519 def _test_LiveFSFilePruner(self, content_type, interval,
1520 keep_binary_files_days=_default,1520 keep_binary_files_days=_default,
1521 expected_count=0):1521 base_image=False, expected_count=0,
1522 **livefsbuild_kwargs):
1522 # Garbo should (or should not, if `expected_count=1`) remove LiveFS1523 # Garbo should (or should not, if `expected_count=1`) remove LiveFS
1523 # files of MIME type `content_type` that finished more than1524 # files of MIME type `content_type` that finished more than
1524 # `interval` days ago. If `keep_binary_files_days` is given, set1525 # `interval` days ago. If `keep_binary_files_days` is given, set
1525 # that on the test LiveFS.1526 # that on the test LiveFS. If `base_image` is True, install the
1527 # test LiveFS file as a base image for its DAS.
1526 now = datetime.now(UTC)1528 now = datetime.now(UTC)
1527 switch_dbuser('testadmin')1529 switch_dbuser('testadmin')
1528 self.useFixture(FeatureFixture({LIVEFS_FEATURE_FLAG: u'on'}))1530 self.useFixture(FeatureFixture({LIVEFS_FEATURE_FLAG: u'on'}))
1529 store = IMasterStore(LiveFSFile)1531 store = IMasterStore(LiveFSFile)
1532 initial_count = store.find(LiveFSFile).count()
15301533
1531 livefs_kwargs = {}1534 livefsbuild_kwargs = dict(livefsbuild_kwargs)
1532 if keep_binary_files_days is not _default:1535 if keep_binary_files_days is not _default:
1533 livefs_kwargs['keep_binary_files_days'] = keep_binary_files_days1536 livefsbuild_kwargs['keep_binary_files_days'] = (
1537 keep_binary_files_days)
1534 db_build = self.factory.makeLiveFSBuild(1538 db_build = self.factory.makeLiveFSBuild(
1535 date_created=now - timedelta(days=interval, minutes=15),1539 date_created=now - timedelta(days=interval, minutes=15),
1536 status=BuildStatus.FULLYBUILT, duration=timedelta(minutes=10),1540 status=BuildStatus.FULLYBUILT, duration=timedelta(minutes=10),
1537 **livefs_kwargs)1541 **livefsbuild_kwargs)
1538 db_lfa = self.factory.makeLibraryFileAlias(content_type=content_type)1542 db_lfa = self.factory.makeLibraryFileAlias(content_type=content_type)
1539 db_file = self.factory.makeLiveFSFile(1543 db_file = self.factory.makeLiveFSFile(
1540 livefsbuild=db_build, libraryfile=db_lfa)1544 livefsbuild=db_build, libraryfile=db_lfa)
1541 Store.of(db_file).flush()1545 if base_image:
1542 self.assertEqual(1, store.find(LiveFSFile).count())1546 db_build.distro_arch_series.setChrootFromBuild(
1547 db_build, db_file.libraryfile.filename)
1548 store.flush()
15431549
1544 self.runDaily()1550 self.runDaily()
15451551
1546 switch_dbuser('testadmin')1552 switch_dbuser('testadmin')
1547 self.assertEqual(expected_count, store.find(LiveFSFile).count())1553 self.assertEqual(
1554 initial_count + expected_count, store.find(LiveFSFile).count())
15481555
1549 def test_LiveFSFilePruner_old_binary_files(self):1556 def test_LiveFSFilePruner_old_binary_files(self):
1550 # By default, LiveFS binary files attached to builds over a day old1557 # By default, LiveFS binary files attached to builds over a day old
@@ -1580,6 +1587,31 @@
1580 'application/octet-stream', 100, keep_binary_files_days=None,1587 'application/octet-stream', 100, keep_binary_files_days=None,
1581 expected_count=1)1588 expected_count=1)
15821589
1590 def test_LiveFSFilePruner_base_image(self):
1591 # An old LiveFS binary file is not pruned if it is a base image.
1592 self._test_LiveFSFilePruner(
1593 'application/octet-stream', 100, base_image=True, expected_count=1)
1594
1595 def test_LiveFSFilePruner_other_base_image(self):
1596 # An old LiveFS binary file is pruned even if some other base image
1597 # exists.
1598 switch_dbuser('testadmin')
1599 self.useFixture(FeatureFixture({LIVEFS_FEATURE_FLAG: u'on'}))
1600 store = IMasterStore(LiveFSFile)
1601 other_build = self.factory.makeLiveFSBuild(
1602 status=BuildStatus.FULLYBUILT, duration=timedelta(minutes=10))
1603 other_lfa = self.factory.makeLibraryFileAlias(
1604 content_type='application/octet-stream')
1605 other_file = self.factory.makeLiveFSFile(
1606 livefsbuild=other_build, libraryfile=other_lfa)
1607 other_build.distro_arch_series.setChrootFromBuild(
1608 other_build, other_file.libraryfile.filename)
1609 store.flush()
1610 self._test_LiveFSFilePruner(
1611 'application/octet-stream', 100,
1612 distroarchseries=other_build.distro_arch_series)
1613 self.assertContentEqual([other_file], store.find(LiveFSFile))
1614
1583 def _test_SnapFilePruner(self, filename, job_status, interval,1615 def _test_SnapFilePruner(self, filename, job_status, interval,
1584 expected_count=0):1616 expected_count=0):
1585 # Garbo should (or should not, if `expected_count=1`) remove snap1617 # Garbo should (or should not, if `expected_count=1`) remove snap