Merge lp:~cjwatson/launchpad/livefs-keep-base-images into lp:launchpad

Proposed by Colin Watson
Status: Merged
Merged at revision: 19006
Proposed branch: lp:~cjwatson/launchpad/livefs-keep-base-images
Merge into: lp:launchpad
Prerequisite: lp:~cjwatson/launchpad/livefs-keep-binary-files-interval
Diff against target: 121 lines (+55/-12)
2 files modified
lib/lp/scripts/garbo.py (+15/-4)
lib/lp/scripts/tests/test_garbo.py (+40/-8)
To merge this branch: bzr merge lp:~cjwatson/launchpad/livefs-keep-base-images
Reviewer Review Type Date Requested Status
William Grant code Approve
Review via email: mp+368714@code.launchpad.net

Commit message

Exclude files that are set as base images from LiveFSFile pruning.

To post a comment you must log in.
Revision history for this message
William Grant (wgrant) :
review: Approve (code)
Revision history for this message
Colin Watson (cjwatson) :

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/scripts/garbo.py'
2--- lib/lp/scripts/garbo.py 2019-07-02 16:21:44 +0000
3+++ lib/lp/scripts/garbo.py 2019-07-02 16:21:44 +0000
4@@ -1567,10 +1567,17 @@
5 """A BulkPruner to remove old `LiveFSFile`s.
6
7 We remove binary files attached to `LiveFSBuild`s that are more than
8- `LiveFS.keep_binary_files_interval` old; these files are very large and
9- are only useful for builds in progress. Text files are typically small
10- (<1MiB) and useful for retrospective analysis, so we preserve those
11- indefinitely.
12+ `LiveFS.keep_binary_files_interval` old and that are not set as base
13+ images for a `DistroArchSeries`; these files are very large and are only
14+ useful for builds in progress.
15+
16+ DAS base images are excluded because
17+ `DistroArchSeries.setChrootFromBuild` takes a `LiveFSBuild` and we want
18+ to have the option of reverting to a previous base image shortly after
19+ upgrading to a newer one.
20+
21+ Text files are typically small (<1MiB) and useful for retrospective
22+ analysis, so we preserve those indefinitely.
23 """
24 target_table_class = LiveFSFile
25 # Note that a NULL keep_binary_files_interval disables pruning, due to
26@@ -1586,6 +1593,10 @@
27 CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
28 - LiveFS.keep_binary_files_interval
29 AND LibraryFileAlias.mimetype != 'text/plain'
30+ EXCEPT
31+ SELECT LiveFSFile.id
32+ FROM LiveFSFile, PocketChroot
33+ WHERE LiveFSFile.libraryfile = PocketChroot.chroot
34 """
35
36
37
38=== modified file 'lib/lp/scripts/tests/test_garbo.py'
39--- lib/lp/scripts/tests/test_garbo.py 2019-07-02 16:21:44 +0000
40+++ lib/lp/scripts/tests/test_garbo.py 2019-07-02 16:21:44 +0000
41@@ -1518,33 +1518,40 @@
42
43 def _test_LiveFSFilePruner(self, content_type, interval,
44 keep_binary_files_days=_default,
45- expected_count=0):
46+ base_image=False, expected_count=0,
47+ **livefsbuild_kwargs):
48 # Garbo should (or should not, if `expected_count=1`) remove LiveFS
49 # files of MIME type `content_type` that finished more than
50 # `interval` days ago. If `keep_binary_files_days` is given, set
51- # that on the test LiveFS.
52+ # that on the test LiveFS. If `base_image` is True, install the
53+ # test LiveFS file as a base image for its DAS.
54 now = datetime.now(UTC)
55 switch_dbuser('testadmin')
56 self.useFixture(FeatureFixture({LIVEFS_FEATURE_FLAG: u'on'}))
57 store = IMasterStore(LiveFSFile)
58+ initial_count = store.find(LiveFSFile).count()
59
60- livefs_kwargs = {}
61+ livefsbuild_kwargs = dict(livefsbuild_kwargs)
62 if keep_binary_files_days is not _default:
63- livefs_kwargs['keep_binary_files_days'] = keep_binary_files_days
64+ livefsbuild_kwargs['keep_binary_files_days'] = (
65+ keep_binary_files_days)
66 db_build = self.factory.makeLiveFSBuild(
67 date_created=now - timedelta(days=interval, minutes=15),
68 status=BuildStatus.FULLYBUILT, duration=timedelta(minutes=10),
69- **livefs_kwargs)
70+ **livefsbuild_kwargs)
71 db_lfa = self.factory.makeLibraryFileAlias(content_type=content_type)
72 db_file = self.factory.makeLiveFSFile(
73 livefsbuild=db_build, libraryfile=db_lfa)
74- Store.of(db_file).flush()
75- self.assertEqual(1, store.find(LiveFSFile).count())
76+ if base_image:
77+ db_build.distro_arch_series.setChrootFromBuild(
78+ db_build, db_file.libraryfile.filename)
79+ store.flush()
80
81 self.runDaily()
82
83 switch_dbuser('testadmin')
84- self.assertEqual(expected_count, store.find(LiveFSFile).count())
85+ self.assertEqual(
86+ initial_count + expected_count, store.find(LiveFSFile).count())
87
88 def test_LiveFSFilePruner_old_binary_files(self):
89 # By default, LiveFS binary files attached to builds over a day old
90@@ -1580,6 +1587,31 @@
91 'application/octet-stream', 100, keep_binary_files_days=None,
92 expected_count=1)
93
94+ def test_LiveFSFilePruner_base_image(self):
95+ # An old LiveFS binary file is not pruned if it is a base image.
96+ self._test_LiveFSFilePruner(
97+ 'application/octet-stream', 100, base_image=True, expected_count=1)
98+
99+ def test_LiveFSFilePruner_other_base_image(self):
100+ # An old LiveFS binary file is pruned even if some other base image
101+ # exists.
102+ switch_dbuser('testadmin')
103+ self.useFixture(FeatureFixture({LIVEFS_FEATURE_FLAG: u'on'}))
104+ store = IMasterStore(LiveFSFile)
105+ other_build = self.factory.makeLiveFSBuild(
106+ status=BuildStatus.FULLYBUILT, duration=timedelta(minutes=10))
107+ other_lfa = self.factory.makeLibraryFileAlias(
108+ content_type='application/octet-stream')
109+ other_file = self.factory.makeLiveFSFile(
110+ livefsbuild=other_build, libraryfile=other_lfa)
111+ other_build.distro_arch_series.setChrootFromBuild(
112+ other_build, other_file.libraryfile.filename)
113+ store.flush()
114+ self._test_LiveFSFilePruner(
115+ 'application/octet-stream', 100,
116+ distroarchseries=other_build.distro_arch_series)
117+ self.assertContentEqual([other_file], store.find(LiveFSFile))
118+
119 def _test_SnapFilePruner(self, filename, job_status, interval,
120 expected_count=0):
121 # Garbo should (or should not, if `expected_count=1`) remove snap