Merge lp:~cjwatson/launchpad/livefs-keep-binary-files-interval into lp:launchpad

Proposed by Colin Watson on 2019-06-12
Status: Merged
Merged at revision: 19005
Proposed branch: lp:~cjwatson/launchpad/livefs-keep-binary-files-interval
Merge into: lp:launchpad
Diff against target: 322 lines (+116/-20)
7 files modified
lib/lp/scripts/garbo.py (+10/-6)
lib/lp/scripts/tests/test_garbo.py (+34/-4)
lib/lp/soyuz/browser/livefs.py (+8/-2)
lib/lp/soyuz/interfaces/livefs.py (+8/-1)
lib/lp/soyuz/model/livefs.py (+31/-4)
lib/lp/soyuz/tests/test_livefs.py (+21/-1)
lib/lp/testing/factory.py (+4/-2)
To merge this branch: bzr merge lp:~cjwatson/launchpad/livefs-keep-binary-files-interval
Reviewer Review Type Date Requested Status
William Grant code 2019-06-12 Approve on 2019-06-12
Review via email: mp+368703@code.launchpad.net

Commit message

Allow configuring the binary file retention period of a LiveFS.

Description of the change

This makes it more practical to deal with live filesystems that are expected to produce base images for use by Launchpad itself. Setting the base image of course ensures that the corresponding LFA stays around, but we may very well want to set a base image from a live filesystem build that's more than a day old.

https://code.launchpad.net/~cjwatson/launchpad/db-livefs-keep-binary-files-interval/+merge/368702 is the corresponding DB change.

To post a comment you must log in.
18983. By Colin Watson on 2019-06-12

Disable pruning if LiveFS.keep_binary_files_interval is null.

William Grant (wgrant) :
review: Approve (code)
18984. By Colin Watson on 2019-06-12

Update comment.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/scripts/garbo.py'
2--- lib/lp/scripts/garbo.py 2019-01-30 11:23:33 +0000
3+++ lib/lp/scripts/garbo.py 2019-06-12 12:20:38 +0000
4@@ -1566,21 +1566,25 @@
5 class LiveFSFilePruner(BulkPruner):
6 """A BulkPruner to remove old `LiveFSFile`s.
7
8- We remove binary files attached to `LiveFSBuild`s that are more than a
9- day old; these files are very large and are only useful for builds in
10- progress. Text files are typically small (<1MiB) and useful for
11- retrospective analysis, so we preserve those indefinitely.
12+ We remove binary files attached to `LiveFSBuild`s that are more than
13+ `LiveFS.keep_binary_files_interval` old; these files are very large and
14+ are only useful for builds in progress. Text files are typically small
15+ (<1MiB) and useful for retrospective analysis, so we preserve those
16+ indefinitely.
17 """
18 target_table_class = LiveFSFile
19+ # Note that a NULL keep_binary_files_interval disables pruning, due to
20+ # SQL NULL propagation.
21 ids_to_prune_query = """
22 SELECT DISTINCT LiveFSFile.id
23- FROM LiveFSFile, LiveFSBuild, LibraryFileAlias
24+ FROM LiveFSFile, LiveFSBuild, LiveFS, LibraryFileAlias
25 WHERE
26 LiveFSFile.livefsbuild = LiveFSBuild.id
27+ AND LiveFSBuild.livefs = LiveFS.id
28 AND LiveFSFile.libraryfile = LibraryFileAlias.id
29 AND LiveFSBuild.date_finished <
30 CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
31- - CAST('1 day' AS interval)
32+ - LiveFS.keep_binary_files_interval
33 AND LibraryFileAlias.mimetype != 'text/plain'
34 """
35
36
37=== modified file 'lib/lp/scripts/tests/test_garbo.py'
38--- lib/lp/scripts/tests/test_garbo.py 2019-01-30 11:23:33 +0000
39+++ lib/lp/scripts/tests/test_garbo.py 2019-06-12 12:20:38 +0000
40@@ -154,6 +154,9 @@
41 )
42
43
44+_default = object()
45+
46+
47 class TestGarboScript(TestCase):
48 layer = LaunchpadScriptLayer
49
50@@ -1513,18 +1516,25 @@
51 'PopulateLatestPersonSourcePackageReleaseCache')
52 self.assertEqual(spph_2.id, job_data['last_spph_id'])
53
54- def _test_LiveFSFilePruner(self, content_type, interval, expected_count=0):
55+ def _test_LiveFSFilePruner(self, content_type, interval,
56+ keep_binary_files_days=_default,
57+ expected_count=0):
58 # Garbo should (or should not, if `expected_count=1`) remove LiveFS
59 # files of MIME type `content_type` that finished more than
60- # `interval` days ago.
61+ # `interval` days ago. If `keep_binary_files_days` is given, set
62+ # that on the test LiveFS.
63 now = datetime.now(UTC)
64 switch_dbuser('testadmin')
65 self.useFixture(FeatureFixture({LIVEFS_FEATURE_FLAG: u'on'}))
66 store = IMasterStore(LiveFSFile)
67
68+ livefs_kwargs = {}
69+ if keep_binary_files_days is not _default:
70+ livefs_kwargs['keep_binary_files_days'] = keep_binary_files_days
71 db_build = self.factory.makeLiveFSBuild(
72 date_created=now - timedelta(days=interval, minutes=15),
73- status=BuildStatus.FULLYBUILT, duration=timedelta(minutes=10))
74+ status=BuildStatus.FULLYBUILT, duration=timedelta(minutes=10),
75+ **livefs_kwargs)
76 db_lfa = self.factory.makeLibraryFileAlias(content_type=content_type)
77 db_file = self.factory.makeLiveFSFile(
78 livefsbuild=db_build, libraryfile=db_lfa)
79@@ -1537,7 +1547,8 @@
80 self.assertEqual(expected_count, store.find(LiveFSFile).count())
81
82 def test_LiveFSFilePruner_old_binary_files(self):
83- # LiveFS binary files attached to builds over a day old are pruned.
84+ # By default, LiveFS binary files attached to builds over a day old
85+ # are pruned.
86 self._test_LiveFSFilePruner('application/octet-stream', 1)
87
88 def test_LiveFSFilePruner_old_text_files(self):
89@@ -1550,6 +1561,25 @@
90 self._test_LiveFSFilePruner(
91 'application/octet-stream', 0, expected_count=1)
92
93+ def test_LiveFSFilePruner_custom_interval_old_binary_files(self):
94+ # If a custom retention interval is set, LiveFS binary files
95+ # attached to builds over that interval old are pruned.
96+ self._test_LiveFSFilePruner(
97+ 'application/octet-stream', 7, keep_binary_files_days=7)
98+
99+ def test_LiveFSFilePruner_custom_interval_recent_binary_files(self):
100+ # If a custom retention interval is set, LiveFS binary files
101+ # attached to builds less than that interval old are pruned.
102+ self._test_LiveFSFilePruner(
103+ 'application/octet-stream', 6, keep_binary_files_days=7,
104+ expected_count=1)
105+
106+ def test_LiveFSFilePruner_null_interval_disables_pruning(self):
107+ # A null retention interval disables pruning.
108+ self._test_LiveFSFilePruner(
109+ 'application/octet-stream', 100, keep_binary_files_days=None,
110+ expected_count=1)
111+
112 def _test_SnapFilePruner(self, filename, job_status, interval,
113 expected_count=0):
114 # Garbo should (or should not, if `expected_count=1`) remove snap
115
116=== modified file 'lib/lp/soyuz/browser/livefs.py'
117--- lib/lp/soyuz/browser/livefs.py 2018-07-16 00:51:23 +0000
118+++ lib/lp/soyuz/browser/livefs.py 2019-06-12 12:20:38 +0000
119@@ -1,4 +1,4 @@
120-# Copyright 2014-2018 Canonical Ltd. This software is licensed under the
121+# Copyright 2014-2019 Canonical Ltd. This software is licensed under the
122 # GNU Affero General Public License version 3 (see the file LICENSE).
123
124 """LiveFS views."""
125@@ -181,6 +181,7 @@
126 'name',
127 'require_virtualized',
128 'relative_build_score',
129+ 'keep_binary_files_days',
130 ])
131 distro_series = Choice(
132 vocabulary='BuildableDistroSeries', title=u'Distribution series')
133@@ -282,13 +283,18 @@
134
135 label = title
136
137- field_names = ['require_virtualized', 'relative_build_score']
138+ field_names = [
139+ 'require_virtualized',
140+ 'relative_build_score',
141+ 'keep_binary_files_days',
142+ ]
143
144 @property
145 def initial_values(self):
146 return {
147 'require_virtualized': self.context.require_virtualized,
148 'relative_build_score': self.context.relative_build_score,
149+ 'keep_binary_files_days': self.context.keep_binary_files_days,
150 }
151
152
153
154=== modified file 'lib/lp/soyuz/interfaces/livefs.py'
155--- lib/lp/soyuz/interfaces/livefs.py 2017-07-18 16:22:03 +0000
156+++ lib/lp/soyuz/interfaces/livefs.py 2019-06-12 12:20:38 +0000
157@@ -1,4 +1,4 @@
158-# Copyright 2014-2017 Canonical Ltd. This software is licensed under the
159+# Copyright 2014-2019 Canonical Ltd. This software is licensed under the
160 # GNU Affero General Public License version 3 (see the file LICENSE).
161
162 """Live filesystem interfaces."""
163@@ -262,6 +262,13 @@
164 "A delta to apply to all build scores for the live filesystem. "
165 "Builds with a higher score will build sooner.")))
166
167+ keep_binary_files_days = exported(Int(
168+ title=_("Binary file retention period"),
169+ required=False, readonly=False,
170+ description=_(
171+ "Keep binary files attached to builds of this live filesystem "
172+ "for at least this many days. If unset, disable pruning.")))
173+
174
175 class ILiveFSAdminAttributes(Interface):
176 """`ILiveFS` attributes that can be edited by admins.
177
178=== modified file 'lib/lp/soyuz/model/livefs.py'
179--- lib/lp/soyuz/model/livefs.py 2017-07-18 16:22:03 +0000
180+++ lib/lp/soyuz/model/livefs.py 2019-06-12 12:20:38 +0000
181@@ -1,4 +1,4 @@
182-# Copyright 2014-2017 Canonical Ltd. This software is licensed under the
183+# Copyright 2014-2019 Canonical Ltd. This software is licensed under the
184 # GNU Affero General Public License version 3 (see the file LICENSE).
185
186 __metaclass__ = type
187@@ -6,6 +6,9 @@
188 'LiveFS',
189 ]
190
191+from datetime import timedelta
192+import math
193+
194 import pytz
195 from storm.locals import (
196 Bool,
197@@ -17,6 +20,7 @@
198 Reference,
199 Store,
200 Storm,
201+ TimeDelta,
202 Unicode,
203 )
204 from zope.component import getUtility
205@@ -113,8 +117,12 @@
206
207 relative_build_score = Int(name='relative_build_score', allow_none=False)
208
209+ keep_binary_files_interval = TimeDelta(
210+ name='keep_binary_files_interval', allow_none=True)
211+
212 def __init__(self, registrant, owner, distro_series, name,
213- metadata, require_virtualized, date_created):
214+ metadata, require_virtualized, keep_binary_files_days,
215+ date_created):
216 """Construct a `LiveFS`."""
217 if not getFeatureFlag(LIVEFS_FEATURE_FLAG):
218 raise LiveFSFeatureDisabled
219@@ -128,6 +136,24 @@
220 self.relative_build_score = 0
221 self.date_created = date_created
222 self.date_last_modified = date_created
223+ self.keep_binary_files_days = keep_binary_files_days
224+
225+ @property
226+ def keep_binary_files_days(self):
227+ """See `ILiveFS`."""
228+ # Rounding up preserves the "at least this many days" part of the
229+ # contract, and makes the interface simpler.
230+ if self.keep_binary_files_interval is not None:
231+ return int(math.ceil(
232+ self.keep_binary_files_interval.total_seconds() / 86400))
233+ else:
234+ return None
235+
236+ @keep_binary_files_days.setter
237+ def keep_binary_files_days(self, days):
238+ """See `ILiveFS`."""
239+ self.keep_binary_files_interval = (
240+ timedelta(days=days) if days is not None else None)
241
242 def requestBuild(self, requester, archive, distro_arch_series, pocket,
243 unique_key=None, metadata_override=None, version=None):
244@@ -230,7 +256,8 @@
245 """See `ILiveFSSet`."""
246
247 def new(self, registrant, owner, distro_series, name, metadata,
248- require_virtualized=True, date_created=DEFAULT):
249+ require_virtualized=True, keep_binary_files_days=1,
250+ date_created=DEFAULT):
251 """See `ILiveFSSet`."""
252 if not registrant.inTeam(owner):
253 if owner.is_team:
254@@ -248,7 +275,7 @@
255 store = IMasterStore(LiveFS)
256 livefs = LiveFS(
257 registrant, owner, distro_series, name, metadata,
258- require_virtualized, date_created)
259+ require_virtualized, keep_binary_files_days, date_created)
260 store.add(livefs)
261
262 return livefs
263
264=== modified file 'lib/lp/soyuz/tests/test_livefs.py'
265--- lib/lp/soyuz/tests/test_livefs.py 2019-01-25 11:47:20 +0000
266+++ lib/lp/soyuz/tests/test_livefs.py 2019-06-12 12:20:38 +0000
267@@ -1,4 +1,4 @@
268-# Copyright 2014-2018 Canonical Ltd. This software is licensed under the
269+# Copyright 2014-2019 Canonical Ltd. This software is licensed under the
270 # GNU Affero General Public License version 3 (see the file LICENSE).
271
272 """Test live filesystems."""
273@@ -122,6 +122,26 @@
274 with celebrity_logged_in("buildd_admin"):
275 livefs.relative_build_score = 100
276
277+ def test_keep_binary_files_days(self):
278+ # Buildd admins can change the binary file retention period of a
279+ # LiveFS, but ordinary users cannot.
280+ livefs = self.factory.makeLiveFS()
281+ self.assertEqual(1, livefs.keep_binary_files_days)
282+ with person_logged_in(livefs.owner):
283+ self.assertRaises(
284+ Unauthorized, setattr, livefs, "keep_binary_files_days", 2)
285+ with celebrity_logged_in("buildd_admin"):
286+ livefs.keep_binary_files_days = 2
287+ self.assertEqual(2, livefs.keep_binary_files_days)
288+ self.assertEqual(
289+ timedelta(days=2),
290+ removeSecurityProxy(livefs).keep_binary_files_interval)
291+ with celebrity_logged_in("buildd_admin"):
292+ livefs.keep_binary_files_days = None
293+ self.assertIsNone(livefs.keep_binary_files_days)
294+ self.assertIsNone(
295+ removeSecurityProxy(livefs).keep_binary_files_interval)
296+
297 def test_requestBuild(self):
298 # requestBuild creates a new LiveFSBuild.
299 livefs = self.factory.makeLiveFS()
300
301=== modified file 'lib/lp/testing/factory.py'
302--- lib/lp/testing/factory.py 2019-05-06 14:22:34 +0000
303+++ lib/lp/testing/factory.py 2019-06-12 12:20:38 +0000
304@@ -4628,7 +4628,7 @@
305
306 def makeLiveFS(self, registrant=None, owner=None, distroseries=None,
307 name=None, metadata=None, require_virtualized=True,
308- date_created=DEFAULT):
309+ keep_binary_files_days=1, date_created=DEFAULT):
310 """Make a new LiveFS."""
311 if registrant is None:
312 registrant = self.makePerson()
313@@ -4642,7 +4642,9 @@
314 metadata = {}
315 livefs = getUtility(ILiveFSSet).new(
316 registrant, owner, distroseries, name, metadata,
317- require_virtualized=require_virtualized, date_created=date_created)
318+ require_virtualized=require_virtualized,
319+ keep_binary_files_days=keep_binary_files_days,
320+ date_created=date_created)
321 IStore(livefs).flush()
322 return livefs
323