Merge lp:~cjwatson/launchpad/build-fast-cleanup into lp:launchpad
- build-fast-cleanup
- Merge into devel
Proposed by
Colin Watson
Status: | Merged |
---|---|
Merged at revision: | 18644 |
Proposed branch: | lp:~cjwatson/launchpad/build-fast-cleanup |
Merge into: | lp:launchpad |
Diff against target: |
1130 lines (+253/-163) 11 files modified
lib/lp/buildmaster/interfaces/buildfarmjobbehaviour.py (+29/-2) lib/lp/buildmaster/model/buildfarmjobbehaviour.py (+30/-1) lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py (+14/-0) lib/lp/code/model/recipebuilder.py (+22/-21) lib/lp/code/model/tests/test_recipebuilder.py (+35/-36) lib/lp/snappy/model/snapbuildbehaviour.py (+5/-12) lib/lp/snappy/tests/test_snapbuildbehaviour.py (+53/-41) lib/lp/soyuz/model/binarypackagebuildbehaviour.py (+9/-18) lib/lp/soyuz/model/livefsbuildbehaviour.py (+8/-12) lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py (+13/-4) lib/lp/soyuz/tests/test_livefsbuildbehaviour.py (+35/-16) |
To merge this branch: | bzr merge lp:~cjwatson/launchpad/build-fast-cleanup |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
William Grant | code | Approve | |
Review via email: mp+344958@code.launchpad.net |
Commit message
Send fast_cleanup: True to virtualised builds, since they can safely skip the final cleanup steps.
Description of the change
In the process, I refactored the various common bits of building the extra build arguments dict into BuildFarmJobBeh
https:/
To post a comment you must log in.
Revision history for this message
William Grant (wgrant) : | # |
review:
Approve
(code)
Revision history for this message
Colin Watson (cjwatson) : | # |
Revision history for this message
Colin Watson (cjwatson) : | # |
Revision history for this message
William Grant (wgrant) : | # |
review:
Approve
(code)
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'lib/lp/buildmaster/interfaces/buildfarmjobbehaviour.py' |
2 | --- lib/lp/buildmaster/interfaces/buildfarmjobbehaviour.py 2016-03-31 14:58:46 +0000 |
3 | +++ lib/lp/buildmaster/interfaces/buildfarmjobbehaviour.py 2018-05-09 17:22:34 +0000 |
4 | @@ -1,4 +1,4 @@ |
5 | -# Copyright 2009-2014 Canonical Ltd. This software is licensed under the |
6 | +# Copyright 2009-2018 Canonical Ltd. This software is licensed under the |
7 | # GNU Affero General Public License version 3 (see the file LICENSE). |
8 | |
9 | """Interface for build farm job behaviours.""" |
10 | @@ -9,14 +9,41 @@ |
11 | 'IBuildFarmJobBehaviour', |
12 | ] |
13 | |
14 | -from zope.interface import Interface |
15 | +from zope.interface import ( |
16 | + Attribute, |
17 | + Interface, |
18 | + ) |
19 | |
20 | |
21 | class IBuildFarmJobBehaviour(Interface): |
22 | |
23 | + builder_type = Attribute( |
24 | + "The name of the builder type to use for this build, corresponding " |
25 | + "to a launchpad-buildd build manager tag.") |
26 | + |
27 | + distro_arch_series = Attribute("The `DistroArchSeries` to build against.") |
28 | + |
29 | def setBuilder(builder, slave): |
30 | """Sets the associated builder and slave for this instance.""" |
31 | |
32 | + def determineFilesToSend(): |
33 | + """Work out which files to send to the builder. |
34 | + |
35 | + :return: A dict mapping filenames to dicts as follows:: |
36 | + 'sha1': SHA-1 of file content |
37 | + 'url': URL from which the builder can fetch content |
38 | + 'username' (optional): username to authenticate as |
39 | + 'password' (optional): password to authenticate with |
40 | + """ |
41 | + |
42 | + def extraBuildArgs(logger=None): |
43 | + """Return extra arguments required by the builder for this build. |
44 | + |
45 | + :param logger: An optional logger. |
46 | + :return: A dict of builder arguments, or a Deferred resulting in the |
47 | + same. |
48 | + """ |
49 | + |
50 | def composeBuildRequest(logger): |
51 | """Compose parameters for a slave build request. |
52 | |
53 | |
54 | === modified file 'lib/lp/buildmaster/model/buildfarmjobbehaviour.py' |
55 | --- lib/lp/buildmaster/model/buildfarmjobbehaviour.py 2016-10-12 14:02:19 +0000 |
56 | +++ lib/lp/buildmaster/model/buildfarmjobbehaviour.py 2018-05-09 17:22:34 +0000 |
57 | @@ -1,4 +1,4 @@ |
58 | -# Copyright 2009-2016 Canonical Ltd. This software is licensed under the |
59 | +# Copyright 2009-2018 Canonical Ltd. This software is licensed under the |
60 | # GNU Affero General Public License version 3 (see the file LICENSE). |
61 | |
62 | """Base and idle BuildFarmJobBehaviour classes.""" |
63 | @@ -32,6 +32,7 @@ |
64 | from lp.services.librarian.interfaces import ILibraryFileAliasSet |
65 | from lp.services.librarian.utils import copy_and_close |
66 | from lp.services.utils import sanitise_urls |
67 | +from lp.services.webapp import canonical_url |
68 | |
69 | |
70 | SLAVE_LOG_FILENAME = 'buildlog' |
71 | @@ -48,11 +49,39 @@ |
72 | self.build = build |
73 | self._builder = None |
74 | |
75 | + @property |
76 | + def distro_arch_series(self): |
77 | + if self.build is not None: |
78 | + return self.build.distro_arch_series |
79 | + else: |
80 | + return None |
81 | + |
82 | def setBuilder(self, builder, slave): |
83 | """The builder should be set once and not changed.""" |
84 | self._builder = builder |
85 | self._slave = slave |
86 | |
87 | + def determineFilesToSend(self): |
88 | + """The default behaviour is to send no files.""" |
89 | + return {} |
90 | + |
91 | + def extraBuildArgs(self, logger=None): |
92 | + """The default behaviour is to send only common extra arguments.""" |
93 | + args = {} |
94 | + args["arch_tag"] = self.distro_arch_series.architecturetag |
95 | + args["archive_private"] = self.build.archive.private |
96 | + args["build_url"] = canonical_url(self.build) |
97 | + args["fast_cleanup"] = self._builder.virtualized |
98 | + args["series"] = self.distro_arch_series.distroseries.name |
99 | + return args |
100 | + |
101 | + @defer.inlineCallbacks |
102 | + def composeBuildRequest(self, logger): |
103 | + args = yield self.extraBuildArgs(logger=logger) |
104 | + defer.returnValue( |
105 | + (self.builder_type, self.distro_arch_series, |
106 | + self.determineFilesToSend(), args)) |
107 | + |
108 | def verifyBuildRequest(self, logger): |
109 | """The default behaviour is a no-op.""" |
110 | pass |
111 | |
112 | === modified file 'lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py' |
113 | --- lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py 2018-02-14 11:13:47 +0000 |
114 | +++ lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py 2018-05-09 17:22:34 +0000 |
115 | @@ -116,6 +116,20 @@ |
116 | '%s-%s' % (now.strftime("%Y%m%d-%H%M%S"), build_cookie), |
117 | upload_leaf) |
118 | |
119 | + def test_extraBuildArgs_virtualized(self): |
120 | + # If the builder is virtualized, extraBuildArgs sends |
121 | + # fast_cleanup: True. |
122 | + behaviour = self._makeBehaviour(self._makeBuild()) |
123 | + behaviour.setBuilder(self.factory.makeBuilder(virtualized=True), None) |
124 | + self.assertIs(True, behaviour.extraBuildArgs()["fast_cleanup"]) |
125 | + |
126 | + def test_extraBuildArgs_non_virtualized(self): |
127 | + # If the builder is non-virtualized, extraBuildArgs sends |
128 | + # fast_cleanup: False. |
129 | + behaviour = self._makeBehaviour(self._makeBuild()) |
130 | + behaviour.setBuilder(self.factory.makeBuilder(virtualized=False), None) |
131 | + self.assertIs(False, behaviour.extraBuildArgs()["fast_cleanup"]) |
132 | + |
133 | |
134 | class TestDispatchBuildToSlave(TestCase): |
135 | |
136 | |
137 | === modified file 'lib/lp/code/model/recipebuilder.py' |
138 | --- lib/lp/code/model/recipebuilder.py 2018-03-01 17:36:31 +0000 |
139 | +++ lib/lp/code/model/recipebuilder.py 2018-05-09 17:22:34 +0000 |
140 | @@ -1,4 +1,4 @@ |
141 | -# Copyright 2010-2017 Canonical Ltd. This software is licensed under the |
142 | +# Copyright 2010-2018 Canonical Ltd. This software is licensed under the |
143 | # GNU Affero General Public License version 3 (see the file LICENSE). |
144 | |
145 | """Code to build recipes on the buildfarm.""" |
146 | @@ -24,7 +24,7 @@ |
147 | ISourcePackageRecipeBuild, |
148 | ) |
149 | from lp.services.config import config |
150 | -from lp.services.webapp import canonical_url |
151 | +from lp.services.propertycache import cachedproperty |
152 | from lp.soyuz.adapters.archivedependencies import ( |
153 | get_primary_current_component, |
154 | get_sources_list_for_building, |
155 | @@ -36,22 +36,37 @@ |
156 | class RecipeBuildBehaviour(BuildFarmJobBehaviourBase): |
157 | """How to build a recipe on the build farm.""" |
158 | |
159 | + builder_type = "sourcepackagerecipe" |
160 | + |
161 | # The list of build status values for which email notifications are |
162 | # allowed to be sent. It is up to each callback as to whether it will |
163 | # consider sending a notification but it won't do so if the status is not |
164 | # in this list. |
165 | ALLOWED_STATUS_NOTIFICATIONS = ['PACKAGEFAIL', 'DEPFAIL', 'CHROOTFAIL'] |
166 | |
167 | + @cachedproperty |
168 | + def distro_arch_series(self): |
169 | + if self.build is not None and self._builder is not None: |
170 | + return self.build.distroseries.getDistroArchSeriesByProcessor( |
171 | + self._builder.processor) |
172 | + else: |
173 | + return None |
174 | + |
175 | @defer.inlineCallbacks |
176 | - def _extraBuildArgs(self, distroarchseries, logger=None): |
177 | + def extraBuildArgs(self, logger=None): |
178 | """ |
179 | Return the extra arguments required by the slave for the given build. |
180 | """ |
181 | + if self.distro_arch_series is None: |
182 | + raise CannotBuild( |
183 | + "Unable to find distroarchseries for %s in %s" % |
184 | + (self._builder.processor.name, |
185 | + self.build.distroseries.displayname)) |
186 | + |
187 | # Build extra arguments. |
188 | - args = {} |
189 | - args['series'] = self.build.distroseries.name |
190 | + args = yield super(RecipeBuildBehaviour, self).extraBuildArgs( |
191 | + logger=logger) |
192 | args['suite'] = self.build.distroseries.getSuite(self.build.pocket) |
193 | - args['arch_tag'] = distroarchseries.architecturetag |
194 | requester = self.build.requester |
195 | if requester.preferredemail is None: |
196 | # Use a constant, known, name and email. |
197 | @@ -71,11 +86,9 @@ |
198 | None).name |
199 | args['archives'], args['trusted_keys'] = ( |
200 | yield get_sources_list_for_building( |
201 | - self.build, distroarchseries, None, |
202 | + self.build, self.distro_arch_series, None, |
203 | tools_source=config.builddmaster.bzr_builder_sources_list, |
204 | logger=logger)) |
205 | - args['archive_private'] = self.build.archive.private |
206 | - args['build_url'] = canonical_url(self.build) |
207 | # XXX cjwatson 2017-07-26: This duplicates "series", which is common |
208 | # to all build types; this name for it is deprecated and should be |
209 | # removed once launchpad-buildd no longer requires it. |
210 | @@ -84,18 +97,6 @@ |
211 | args['git'] = True |
212 | defer.returnValue(args) |
213 | |
214 | - @defer.inlineCallbacks |
215 | - def composeBuildRequest(self, logger): |
216 | - das = self.build.distroseries.getDistroArchSeriesByProcessor( |
217 | - self._builder.processor) |
218 | - if das is None: |
219 | - raise CannotBuild( |
220 | - "Unable to find distroarchseries for %s in %s" % |
221 | - (self._builder.processor.name, |
222 | - self.build.distroseries.displayname)) |
223 | - args = yield self._extraBuildArgs(das, logger=logger) |
224 | - defer.returnValue(("sourcepackagerecipe", das, {}, args)) |
225 | - |
226 | def verifyBuildRequest(self, logger): |
227 | """Assert some pre-build checks. |
228 | |
229 | |
230 | === modified file 'lib/lp/code/model/tests/test_recipebuilder.py' |
231 | --- lib/lp/code/model/tests/test_recipebuilder.py 2018-03-01 17:36:31 +0000 |
232 | +++ lib/lp/code/model/tests/test_recipebuilder.py 2018-05-09 17:22:34 +0000 |
233 | @@ -1,4 +1,4 @@ |
234 | -# Copyright 2010-2017 Canonical Ltd. This software is licensed under the |
235 | +# Copyright 2010-2018 Canonical Ltd. This software is licensed under the |
236 | # GNU Affero General Public License version 3 (see the file LICENSE). |
237 | |
238 | """Test RecipeBuildBehaviour.""" |
239 | @@ -70,7 +70,7 @@ |
240 | layer = LaunchpadZopelessLayer |
241 | |
242 | def makeJob(self, recipe_registrant=None, recipe_owner=None, |
243 | - archive=None, git=False): |
244 | + archive=None, git=False, with_builder=False): |
245 | """Create a sample `ISourcePackageRecipeBuild`.""" |
246 | spn = self.factory.makeSourcePackageName("apackage") |
247 | if archive is None: |
248 | @@ -105,6 +105,10 @@ |
249 | sourcepackage=sourcepackage, archive=archive, |
250 | recipe=recipe, requester=recipe_owner, distroseries=distroseries) |
251 | job = IBuildFarmJobBehaviour(spb) |
252 | + if with_builder: |
253 | + builder = MockBuilder() |
254 | + builder.processor = processor |
255 | + job.setBuilder(builder, None) |
256 | return job |
257 | |
258 | |
259 | @@ -171,16 +175,16 @@ |
260 | |
261 | @defer.inlineCallbacks |
262 | def test_extraBuildArgs(self): |
263 | - # _extraBuildArgs will return a sane set of additional arguments |
264 | + # extraBuildArgs will return a sane set of additional arguments. |
265 | self._setBuilderConfig() |
266 | - job = self.makeJob() |
267 | + job = self.makeJob(with_builder=True) |
268 | distroarchseries = job.build.distroseries.architectures[0] |
269 | expected_archives, expected_trusted_keys = ( |
270 | yield get_sources_list_for_building( |
271 | job.build, distroarchseries, None)) |
272 | expected_archives.insert( |
273 | 0, "deb http://foo %s main" % job.build.distroseries.name) |
274 | - args = yield job._extraBuildArgs(distroarchseries) |
275 | + args = yield job.extraBuildArgs() |
276 | self.assertEqual({ |
277 | 'arch_tag': 'i386', |
278 | 'archive_private': False, |
279 | @@ -190,6 +194,7 @@ |
280 | 'author_name': 'Joe User', |
281 | 'build_url': canonical_url(job.build), |
282 | 'distroseries_name': job.build.distroseries.name, |
283 | + 'fast_cleanup': True, |
284 | 'ogrecomponent': 'universe', |
285 | 'recipe_text': |
286 | '# bzr-builder format 0.3 ' |
287 | @@ -207,11 +212,9 @@ |
288 | # build logs. |
289 | self._setBuilderConfig() |
290 | archive = self.factory.makeArchive(private=True) |
291 | - job = self.makeJob(archive=archive) |
292 | - distroarchseries = job.build.distroseries.architectures[0] |
293 | - extra_args = yield job._extraBuildArgs(distroarchseries) |
294 | - self.assertEqual( |
295 | - True, extra_args['archive_private']) |
296 | + job = self.makeJob(archive=archive, with_builder=True) |
297 | + extra_args = yield job.extraBuildArgs() |
298 | + self.assertEqual(True, extra_args['archive_private']) |
299 | |
300 | @defer.inlineCallbacks |
301 | def test_extraBuildArgs_team_owner_no_email(self): |
302 | @@ -224,9 +227,8 @@ |
303 | recipe_owner = self.factory.makeTeam( |
304 | name='vikings', members=[recipe_registrant]) |
305 | |
306 | - job = self.makeJob(recipe_registrant, recipe_owner) |
307 | - distroarchseries = job.build.distroseries.architectures[0] |
308 | - extra_args = yield job._extraBuildArgs(distroarchseries) |
309 | + job = self.makeJob(recipe_registrant, recipe_owner, with_builder=True) |
310 | + extra_args = yield job.extraBuildArgs() |
311 | self.assertEqual( |
312 | "Launchpad Package Builder", extra_args['author_name']) |
313 | self.assertEqual("noreply@launchpad.net", extra_args['author_email']) |
314 | @@ -241,9 +243,8 @@ |
315 | name='vikings', email='everyone@vikings.r.us', |
316 | members=[recipe_registrant]) |
317 | |
318 | - job = self.makeJob(recipe_registrant, recipe_owner) |
319 | - distroarchseries = job.build.distroseries.architectures[0] |
320 | - extra_args = yield job._extraBuildArgs(distroarchseries) |
321 | + job = self.makeJob(recipe_registrant, recipe_owner, with_builder=True) |
322 | + extra_args = yield job.extraBuildArgs() |
323 | self.assertEqual("Vikings", extra_args['author_name']) |
324 | self.assertEqual("everyone@vikings.r.us", extra_args['author_email']) |
325 | |
326 | @@ -254,28 +255,27 @@ |
327 | owner = self.factory.makePerson() |
328 | with person_logged_in(owner): |
329 | owner.deactivate(comment='deactivating') |
330 | - job = self.makeJob(owner) |
331 | - distroarchseries = job.build.distroseries.architectures[0] |
332 | - extra_args = yield job._extraBuildArgs(distroarchseries) |
333 | + job = self.makeJob(owner, with_builder=True) |
334 | + extra_args = yield job.extraBuildArgs() |
335 | self.assertEqual( |
336 | "Launchpad Package Builder", extra_args['author_name']) |
337 | self.assertEqual("noreply@launchpad.net", extra_args['author_email']) |
338 | |
339 | @defer.inlineCallbacks |
340 | def test_extraBuildArgs_withBadConfigForBzrBuilderPPA(self): |
341 | - # Ensure _extraBuildArgs doesn't blow up with a badly formatted |
342 | + # Ensure extraBuildArgs doesn't blow up with a badly formatted |
343 | # bzr_builder_sources_list in the config. |
344 | self.pushConfig( |
345 | "builddmaster", |
346 | bzr_builder_sources_list="deb http://foo %(series) main") |
347 | # (note the missing 's' in %(series) |
348 | - job = self.makeJob() |
349 | + job = self.makeJob(with_builder=True) |
350 | distroarchseries = job.build.distroseries.architectures[0] |
351 | expected_archives, expected_trusted_keys = ( |
352 | yield get_sources_list_for_building( |
353 | job.build, distroarchseries, None)) |
354 | logger = BufferLogger() |
355 | - extra_args = yield job._extraBuildArgs(distroarchseries, logger) |
356 | + extra_args = yield job.extraBuildArgs(logger) |
357 | self.assertEqual({ |
358 | 'arch_tag': 'i386', |
359 | 'archive_private': False, |
360 | @@ -285,6 +285,7 @@ |
361 | 'author_name': 'Joe User', |
362 | 'build_url': canonical_url(job.build), |
363 | 'distroseries_name': job.build.distroseries.name, |
364 | + 'fast_cleanup': True, |
365 | 'ogrecomponent': 'universe', |
366 | 'recipe_text': |
367 | '# bzr-builder format 0.3 ' |
368 | @@ -299,12 +300,12 @@ |
369 | logger.getLogBuffer()) |
370 | |
371 | @defer.inlineCallbacks |
372 | - def test_extraBuildArgs_withNoBZrBuilderConfigSet(self): |
373 | - # Ensure _extraBuildArgs doesn't blow up when |
374 | + def test_extraBuildArgs_withNoBzrBuilderConfigSet(self): |
375 | + # Ensure extraBuildArgs doesn't blow up when |
376 | # bzr_builder_sources_list isn't set. |
377 | - job = self.makeJob() |
378 | + job = self.makeJob(with_builder=True) |
379 | distroarchseries = job.build.distroseries.architectures[0] |
380 | - args = yield job._extraBuildArgs(distroarchseries) |
381 | + args = yield job.extraBuildArgs() |
382 | expected_archives, expected_trusted_keys = ( |
383 | yield get_sources_list_for_building( |
384 | job.build, distroarchseries, None)) |
385 | @@ -313,12 +314,12 @@ |
386 | |
387 | @defer.inlineCallbacks |
388 | def test_extraBuildArgs_git(self): |
389 | - job = self.makeJob(git=True) |
390 | + job = self.makeJob(git=True, with_builder=True) |
391 | distroarchseries = job.build.distroseries.architectures[0] |
392 | expected_archives, expected_trusted_keys = ( |
393 | yield get_sources_list_for_building( |
394 | job.build, distroarchseries, None)) |
395 | - extra_args = yield job._extraBuildArgs(distroarchseries) |
396 | + extra_args = yield job.extraBuildArgs() |
397 | self.assertEqual({ |
398 | 'arch_tag': 'i386', |
399 | 'archive_private': False, |
400 | @@ -328,6 +329,7 @@ |
401 | 'author_name': 'Joe User', |
402 | 'build_url': canonical_url(job.build), |
403 | 'distroseries_name': job.build.distroseries.name, |
404 | + 'fast_cleanup': True, |
405 | 'git': True, |
406 | 'ogrecomponent': 'universe', |
407 | 'recipe_text': |
408 | @@ -341,33 +343,30 @@ |
409 | |
410 | @defer.inlineCallbacks |
411 | def test_extraBuildArgs_archive_trusted_keys(self): |
412 | - # If the archive has a signing key, _extraBuildArgs sends it. |
413 | + # If the archive has a signing key, extraBuildArgs sends it. |
414 | yield self.useFixture(InProcessKeyServerFixture()).start() |
415 | archive = self.factory.makeArchive() |
416 | key_path = os.path.join(gpgkeysdir, "ppa-sample@canonical.com.sec") |
417 | yield IArchiveSigningKey(archive).setSigningKey( |
418 | key_path, async_keyserver=True) |
419 | - job = self.makeJob(archive=archive) |
420 | + job = self.makeJob(archive=archive, with_builder=True) |
421 | distroarchseries = job.build.distroseries.architectures[0] |
422 | self.factory.makeBinaryPackagePublishingHistory( |
423 | distroarchseries=distroarchseries, pocket=job.build.pocket, |
424 | archive=archive, status=PackagePublishingStatus.PUBLISHED) |
425 | - args = yield job._extraBuildArgs(distroarchseries) |
426 | + args = yield job.extraBuildArgs() |
427 | self.assertThat(args["trusted_keys"], MatchesListwise([ |
428 | Base64KeyMatches("0D57E99656BEFB0897606EE9A022DD1F5001B46D"), |
429 | ])) |
430 | |
431 | @defer.inlineCallbacks |
432 | def test_composeBuildRequest(self): |
433 | - job = self.makeJob() |
434 | + job = self.makeJob(with_builder=True) |
435 | test_publisher = SoyuzTestPublisher() |
436 | test_publisher.addFakeChroots(job.build.distroseries) |
437 | das = job.build.distroseries.nominatedarchindep |
438 | - builder = MockBuilder("bob-de-bouwer") |
439 | - builder.processor = das.processor |
440 | - job.setBuilder(builder, None) |
441 | build_request = yield job.composeBuildRequest(None) |
442 | - extra_args = yield job._extraBuildArgs(das) |
443 | + extra_args = yield job.extraBuildArgs() |
444 | self.assertEqual( |
445 | ('sourcepackagerecipe', das, {}, extra_args), build_request) |
446 | |
447 | |
448 | === modified file 'lib/lp/snappy/model/snapbuildbehaviour.py' |
449 | --- lib/lp/snappy/model/snapbuildbehaviour.py 2018-04-23 11:14:38 +0000 |
450 | +++ lib/lp/snappy/model/snapbuildbehaviour.py 2018-05-09 17:22:34 +0000 |
451 | @@ -30,7 +30,6 @@ |
452 | ) |
453 | from lp.registry.interfaces.series import SeriesStatus |
454 | from lp.services.config import config |
455 | -from lp.services.webapp import canonical_url |
456 | from lp.snappy.interfaces.snap import SnapBuildArchiveOwnerMismatch |
457 | from lp.snappy.interfaces.snapbuild import ISnapBuild |
458 | from lp.soyuz.adapters.archivedependencies import ( |
459 | @@ -44,6 +43,8 @@ |
460 | class SnapBuildBehaviour(BuildFarmJobBehaviourBase): |
461 | """Dispatches `SnapBuild` jobs to slaves.""" |
462 | |
463 | + builder_type = "snap" |
464 | + |
465 | def getLogFileName(self): |
466 | das = self.build.distro_arch_series |
467 | |
468 | @@ -79,12 +80,13 @@ |
469 | "Missing chroot for %s" % build.distro_arch_series.displayname) |
470 | |
471 | @defer.inlineCallbacks |
472 | - def _extraBuildArgs(self, logger=None): |
473 | + def extraBuildArgs(self, logger=None): |
474 | """ |
475 | Return the extra arguments required by the slave for the given build. |
476 | """ |
477 | build = self.build |
478 | - args = {} |
479 | + args = yield super(SnapBuildBehaviour, self).extraBuildArgs( |
480 | + logger=logger) |
481 | if config.snappy.builder_proxy_host and build.snap.allow_internet: |
482 | token = yield self._requestProxyToken() |
483 | args["proxy_url"] = ( |
484 | @@ -98,8 +100,6 @@ |
485 | endpoint=config.snappy.builder_proxy_auth_api_endpoint, |
486 | token=token['username'])) |
487 | args["name"] = build.snap.store_name or build.snap.name |
488 | - args["series"] = build.distro_series.name |
489 | - args["arch_tag"] = build.distro_arch_series.architecturetag |
490 | # XXX cjwatson 2015-08-03: Allow tools_source to be overridden at |
491 | # some more fine-grained level. |
492 | args["archives"], args["trusted_keys"] = ( |
493 | @@ -108,8 +108,6 @@ |
494 | tools_source=config.snappy.tools_source, |
495 | tools_fingerprint=config.snappy.tools_fingerprint, |
496 | logger=logger)) |
497 | - args["archive_private"] = build.archive.private |
498 | - args["build_url"] = canonical_url(build) |
499 | if build.channels is not None: |
500 | # We have to remove the security proxy that Zope applies to this |
501 | # dict, since otherwise we'll be unable to serialise it to |
502 | @@ -168,11 +166,6 @@ |
503 | token = json.loads(result) |
504 | defer.returnValue(token) |
505 | |
506 | - @defer.inlineCallbacks |
507 | - def composeBuildRequest(self, logger): |
508 | - args = yield self._extraBuildArgs(logger=logger) |
509 | - defer.returnValue(("snap", self.build.distro_arch_series, {}, args)) |
510 | - |
511 | def verifySuccessfulBuild(self): |
512 | """See `IBuildFarmJobBehaviour`.""" |
513 | # The implementation in BuildFarmJobBehaviourBase checks whether the |
514 | |
515 | === modified file 'lib/lp/snappy/tests/test_snapbuildbehaviour.py' |
516 | --- lib/lp/snappy/tests/test_snapbuildbehaviour.py 2018-04-23 11:14:38 +0000 |
517 | +++ lib/lp/snappy/tests/test_snapbuildbehaviour.py 2018-05-09 17:22:34 +0000 |
518 | @@ -80,7 +80,7 @@ |
519 | self.pushConfig("snappy", tools_source=None, tools_fingerprint=None) |
520 | |
521 | def makeJob(self, archive=None, pocket=PackagePublishingPocket.UPDATES, |
522 | - **kwargs): |
523 | + with_builder=False, **kwargs): |
524 | """Create a sample `ISnapBuildBehaviour`.""" |
525 | if archive is None: |
526 | distribution = self.factory.makeDistribution(name="distro") |
527 | @@ -95,7 +95,12 @@ |
528 | build = self.factory.makeSnapBuild( |
529 | archive=archive, distroarchseries=distroarchseries, pocket=pocket, |
530 | name="test-snap", **kwargs) |
531 | - return IBuildFarmJobBehaviour(build) |
532 | + job = IBuildFarmJobBehaviour(build) |
533 | + if with_builder: |
534 | + builder = MockBuilder() |
535 | + builder.processor = processor |
536 | + job.setBuilder(builder, None) |
537 | + return job |
538 | |
539 | |
540 | class TestSnapBuildBehaviour(TestSnapBuildBehaviourBase): |
541 | @@ -222,7 +227,7 @@ |
542 | |
543 | @defer.inlineCallbacks |
544 | def test_composeBuildRequest(self): |
545 | - job = self.makeJob() |
546 | + job = self.makeJob(with_builder=True) |
547 | lfa = self.factory.makeLibraryFileAlias(db_only=True) |
548 | job.build.distro_arch_series.addOrUpdateChroot(lfa) |
549 | build_request = yield job.composeBuildRequest(None) |
550 | @@ -233,17 +238,17 @@ |
551 | def test_requestProxyToken_unconfigured(self): |
552 | self.pushConfig("snappy", builder_proxy_auth_api_admin_secret=None) |
553 | branch = self.factory.makeBranch() |
554 | - job = self.makeJob(branch=branch) |
555 | + job = self.makeJob(branch=branch, with_builder=True) |
556 | expected_exception_msg = ( |
557 | "builder_proxy_auth_api_admin_secret is not configured.") |
558 | with ExpectedException(CannotBuild, expected_exception_msg): |
559 | - yield job._extraBuildArgs() |
560 | + yield job.extraBuildArgs() |
561 | |
562 | @defer.inlineCallbacks |
563 | def test_requestProxyToken(self): |
564 | branch = self.factory.makeBranch() |
565 | - job = self.makeJob(branch=branch) |
566 | - yield job._extraBuildArgs() |
567 | + job = self.makeJob(branch=branch, with_builder=True) |
568 | + yield job.extraBuildArgs() |
569 | self.assertThat(self.mock_proxy_api.calls, MatchesListwise([ |
570 | MatchesListwise([ |
571 | MatchesListwise([ |
572 | @@ -265,14 +270,14 @@ |
573 | |
574 | @defer.inlineCallbacks |
575 | def test_extraBuildArgs_bzr(self): |
576 | - # _extraBuildArgs returns appropriate arguments if asked to build a |
577 | + # extraBuildArgs returns appropriate arguments if asked to build a |
578 | # job for a Bazaar branch. |
579 | branch = self.factory.makeBranch() |
580 | - job = self.makeJob(branch=branch) |
581 | + job = self.makeJob(branch=branch, with_builder=True) |
582 | expected_archives, expected_trusted_keys = ( |
583 | yield get_sources_list_for_building( |
584 | job.build, job.build.distro_arch_series, None)) |
585 | - args = yield job._extraBuildArgs() |
586 | + args = yield job.extraBuildArgs() |
587 | self.assertEqual({ |
588 | "archive_private": False, |
589 | "archives": expected_archives, |
590 | @@ -280,6 +285,7 @@ |
591 | "branch": branch.bzr_identity, |
592 | "build_source_tarball": False, |
593 | "build_url": canonical_url(job.build), |
594 | + "fast_cleanup": True, |
595 | "name": "test-snap", |
596 | "proxy_url": self.proxy_url, |
597 | "revocation_endpoint": self.revocation_endpoint, |
598 | @@ -289,20 +295,21 @@ |
599 | |
600 | @defer.inlineCallbacks |
601 | def test_extraBuildArgs_git(self): |
602 | - # _extraBuildArgs returns appropriate arguments if asked to build a |
603 | + # extraBuildArgs returns appropriate arguments if asked to build a |
604 | # job for a Git branch. |
605 | [ref] = self.factory.makeGitRefs() |
606 | - job = self.makeJob(git_ref=ref) |
607 | + job = self.makeJob(git_ref=ref, with_builder=True) |
608 | expected_archives, expected_trusted_keys = ( |
609 | yield get_sources_list_for_building( |
610 | job.build, job.build.distro_arch_series, None)) |
611 | - args = yield job._extraBuildArgs() |
612 | + args = yield job.extraBuildArgs() |
613 | self.assertEqual({ |
614 | "archive_private": False, |
615 | "archives": expected_archives, |
616 | "arch_tag": "i386", |
617 | "build_source_tarball": False, |
618 | "build_url": canonical_url(job.build), |
619 | + "fast_cleanup": True, |
620 | "git_repository": ref.repository.git_https_url, |
621 | "git_path": ref.name, |
622 | "name": "test-snap", |
623 | @@ -314,21 +321,23 @@ |
624 | |
625 | @defer.inlineCallbacks |
626 | def test_extraBuildArgs_git_HEAD(self): |
627 | - # _extraBuildArgs returns appropriate arguments if asked to build a |
628 | + # extraBuildArgs returns appropriate arguments if asked to build a |
629 | # job for the default branch in a Launchpad-hosted Git repository. |
630 | [ref] = self.factory.makeGitRefs() |
631 | removeSecurityProxy(ref.repository)._default_branch = ref.path |
632 | - job = self.makeJob(git_ref=ref.repository.getRefByPath("HEAD")) |
633 | + job = self.makeJob( |
634 | + git_ref=ref.repository.getRefByPath("HEAD"), with_builder=True) |
635 | expected_archives, expected_trusted_keys = ( |
636 | yield get_sources_list_for_building( |
637 | job.build, job.build.distro_arch_series, None)) |
638 | - args = yield job._extraBuildArgs() |
639 | + args = yield job.extraBuildArgs() |
640 | self.assertEqual({ |
641 | "archive_private": False, |
642 | "archives": expected_archives, |
643 | "arch_tag": "i386", |
644 | "build_source_tarball": False, |
645 | "build_url": canonical_url(job.build), |
646 | + "fast_cleanup": True, |
647 | "git_repository": ref.repository.git_https_url, |
648 | "name": "test-snap", |
649 | "proxy_url": self.proxy_url, |
650 | @@ -339,22 +348,23 @@ |
651 | |
652 | @defer.inlineCallbacks |
653 | def test_extraBuildArgs_git_url(self): |
654 | - # _extraBuildArgs returns appropriate arguments if asked to build a |
655 | + # extraBuildArgs returns appropriate arguments if asked to build a |
656 | # job for a Git branch backed by a URL for an external repository. |
657 | url = "https://git.example.org/foo" |
658 | ref = self.factory.makeGitRefRemote( |
659 | repository_url=url, path="refs/heads/master") |
660 | - job = self.makeJob(git_ref=ref) |
661 | + job = self.makeJob(git_ref=ref, with_builder=True) |
662 | expected_archives, expected_trusted_keys = ( |
663 | yield get_sources_list_for_building( |
664 | job.build, job.build.distro_arch_series, None)) |
665 | - args = yield job._extraBuildArgs() |
666 | + args = yield job.extraBuildArgs() |
667 | self.assertEqual({ |
668 | "archive_private": False, |
669 | "archives": expected_archives, |
670 | "arch_tag": "i386", |
671 | "build_source_tarball": False, |
672 | "build_url": canonical_url(job.build), |
673 | + "fast_cleanup": True, |
674 | "git_repository": url, |
675 | "git_path": "master", |
676 | "name": "test-snap", |
677 | @@ -366,21 +376,22 @@ |
678 | |
679 | @defer.inlineCallbacks |
680 | def test_extraBuildArgs_git_url_HEAD(self): |
681 | - # _extraBuildArgs returns appropriate arguments if asked to build a |
682 | + # extraBuildArgs returns appropriate arguments if asked to build a |
683 | # job for the default branch in an external Git repository. |
684 | url = "https://git.example.org/foo" |
685 | ref = self.factory.makeGitRefRemote(repository_url=url, path="HEAD") |
686 | - job = self.makeJob(git_ref=ref) |
687 | + job = self.makeJob(git_ref=ref, with_builder=True) |
688 | expected_archives, expected_trusted_keys = ( |
689 | yield get_sources_list_for_building( |
690 | job.build, job.build.distro_arch_series, None)) |
691 | - args = yield job._extraBuildArgs() |
692 | + args = yield job.extraBuildArgs() |
693 | self.assertEqual({ |
694 | "archive_private": False, |
695 | "archives": expected_archives, |
696 | "arch_tag": "i386", |
697 | "build_source_tarball": False, |
698 | "build_url": canonical_url(job.build), |
699 | + "fast_cleanup": True, |
700 | "git_repository": url, |
701 | "name": "test-snap", |
702 | "proxy_url": self.proxy_url, |
703 | @@ -391,62 +402,61 @@ |
704 | |
705 | @defer.inlineCallbacks |
706 | def test_extraBuildArgs_prefers_store_name(self): |
707 | - # For the "name" argument, _extraBuildArgs prefers Snap.store_name |
708 | + # For the "name" argument, extraBuildArgs prefers Snap.store_name |
709 | # over Snap.name if the former is set. |
710 | - job = self.makeJob(store_name="something-else") |
711 | - args = yield job._extraBuildArgs() |
712 | + job = self.makeJob(store_name="something-else", with_builder=True) |
713 | + args = yield job.extraBuildArgs() |
714 | self.assertEqual("something-else", args["name"]) |
715 | |
716 | @defer.inlineCallbacks |
717 | def test_extraBuildArgs_archive_trusted_keys(self): |
718 | - # If the archive has a signing key, _extraBuildArgs sends it. |
719 | + # If the archive has a signing key, extraBuildArgs sends it. |
720 | yield self.useFixture(InProcessKeyServerFixture()).start() |
721 | archive = self.factory.makeArchive() |
722 | key_path = os.path.join(gpgkeysdir, "ppa-sample@canonical.com.sec") |
723 | yield IArchiveSigningKey(archive).setSigningKey( |
724 | key_path, async_keyserver=True) |
725 | - job = self.makeJob(archive=archive) |
726 | + job = self.makeJob(archive=archive, with_builder=True) |
727 | self.factory.makeBinaryPackagePublishingHistory( |
728 | distroarchseries=job.build.distro_arch_series, |
729 | pocket=job.build.pocket, archive=archive, |
730 | status=PackagePublishingStatus.PUBLISHED) |
731 | - args = yield job._extraBuildArgs() |
732 | + args = yield job.extraBuildArgs() |
733 | self.assertThat(args["trusted_keys"], MatchesListwise([ |
734 | Base64KeyMatches("0D57E99656BEFB0897606EE9A022DD1F5001B46D"), |
735 | ])) |
736 | |
737 | @defer.inlineCallbacks |
738 | def test_extraBuildArgs_channels(self): |
739 | - # If the build needs particular channels, _extraBuildArgs sends |
740 | - # them. |
741 | - job = self.makeJob(channels={"snapcraft": "edge"}) |
742 | + # If the build needs particular channels, extraBuildArgs sends them. |
743 | + job = self.makeJob(channels={"snapcraft": "edge"}, with_builder=True) |
744 | expected_archives, expected_trusted_keys = ( |
745 | yield get_sources_list_for_building( |
746 | job.build, job.build.distro_arch_series, None)) |
747 | - args = yield job._extraBuildArgs() |
748 | + args = yield job.extraBuildArgs() |
749 | self.assertFalse(isProxy(args["channels"])) |
750 | self.assertEqual({"snapcraft": "edge"}, args["channels"]) |
751 | |
752 | @defer.inlineCallbacks |
753 | def test_extraBuildArgs_disallow_internet(self): |
754 | # If external network access is not allowed for the snap, |
755 | - # _extraBuildArgs does not dispatch a proxy token. |
756 | - job = self.makeJob(allow_internet=False) |
757 | - args = yield job._extraBuildArgs() |
758 | + # extraBuildArgs does not dispatch a proxy token. |
759 | + job = self.makeJob(allow_internet=False, with_builder=True) |
760 | + args = yield job.extraBuildArgs() |
761 | self.assertNotIn("proxy_url", args) |
762 | self.assertNotIn("revocation_endpoint", args) |
763 | |
764 | @defer.inlineCallbacks |
765 | def test_extraBuildArgs_build_source_tarball(self): |
766 | - # If the snap requests building of a source tarball, _extraBuildArgs |
767 | + # If the snap requests building of a source tarball, extraBuildArgs |
768 | # sends the appropriate arguments. |
769 | - job = self.makeJob(build_source_tarball=True) |
770 | - args = yield job._extraBuildArgs() |
771 | + job = self.makeJob(build_source_tarball=True, with_builder=True) |
772 | + args = yield job.extraBuildArgs() |
773 | self.assertTrue(args["build_source_tarball"]) |
774 | |
775 | @defer.inlineCallbacks |
776 | def test_composeBuildRequest_proxy_url_set(self): |
777 | - job = self.makeJob() |
778 | + job = self.makeJob(with_builder=True) |
779 | build_request = yield job.composeBuildRequest(None) |
780 | proxy_url = ("http://{username}:{password}" |
781 | "@{host}:{port}".format( |
782 | @@ -462,7 +472,8 @@ |
783 | # composeBuildRequest raises CannotBuild. |
784 | branch = self.factory.makeBranch() |
785 | owner = self.factory.makePerson(name="snap-owner") |
786 | - job = self.makeJob(registrant=owner, owner=owner, branch=branch) |
787 | + job = self.makeJob( |
788 | + registrant=owner, owner=owner, branch=branch, with_builder=True) |
789 | branch.destroySelf(break_references=True) |
790 | self.assertIsNone(job.build.snap.branch) |
791 | self.assertIsNone(job.build.snap.git_repository) |
792 | @@ -478,7 +489,8 @@ |
793 | repository = self.factory.makeGitRepository() |
794 | [ref] = self.factory.makeGitRefs(repository=repository) |
795 | owner = self.factory.makePerson(name="snap-owner") |
796 | - job = self.makeJob(registrant=owner, owner=owner, git_ref=ref) |
797 | + job = self.makeJob( |
798 | + registrant=owner, owner=owner, git_ref=ref, with_builder=True) |
799 | repository.removeRefs([ref.path]) |
800 | self.assertIsNone(job.build.snap.git_ref) |
801 | expected_exception_msg = ("Source branch/repository for " |
802 | |
803 | === modified file 'lib/lp/soyuz/model/binarypackagebuildbehaviour.py' |
804 | --- lib/lp/soyuz/model/binarypackagebuildbehaviour.py 2018-03-01 17:36:31 +0000 |
805 | +++ lib/lp/soyuz/model/binarypackagebuildbehaviour.py 2018-05-09 17:22:34 +0000 |
806 | @@ -1,4 +1,4 @@ |
807 | -# Copyright 2009-2017 Canonical Ltd. This software is licensed under the |
808 | +# Copyright 2009-2018 Canonical Ltd. This software is licensed under the |
809 | # GNU Affero General Public License version 3 (see the file LICENSE). |
810 | |
811 | """Builder behaviour for binary package builds.""" |
812 | @@ -20,10 +20,7 @@ |
813 | BuildFarmJobBehaviourBase, |
814 | ) |
815 | from lp.registry.interfaces.pocket import PackagePublishingPocket |
816 | -from lp.services.webapp import ( |
817 | - canonical_url, |
818 | - urlappend, |
819 | - ) |
820 | +from lp.services.webapp import urlappend |
821 | from lp.soyuz.adapters.archivedependencies import ( |
822 | get_primary_current_component, |
823 | get_sources_list_for_building, |
824 | @@ -36,6 +33,8 @@ |
825 | class BinaryPackageBuildBehaviour(BuildFarmJobBehaviourBase): |
826 | """Define the behaviour of binary package builds.""" |
827 | |
828 | + builder_type = "binarypackage" |
829 | + |
830 | def getLogFileName(self): |
831 | """See `IBuildPackageJob`.""" |
832 | sourcename = self.build.source_package_release.name |
833 | @@ -60,6 +59,7 @@ |
834 | state)) |
835 | |
836 | def determineFilesToSend(self): |
837 | + """See `IBuildFarmJobBehaviour`.""" |
838 | # Build filemap structure with the files required in this build |
839 | # and send them to the slave. |
840 | if self.build.archive.private: |
841 | @@ -85,13 +85,6 @@ |
842 | 'password': self.build.archive.buildd_secret} |
843 | return filemap |
844 | |
845 | - @defer.inlineCallbacks |
846 | - def composeBuildRequest(self, logger): |
847 | - args = yield self._extraBuildArgs(self.build, logger=logger) |
848 | - defer.returnValue( |
849 | - ("binarypackage", self.build.distro_arch_series, |
850 | - self.determineFilesToSend(), args)) |
851 | - |
852 | def verifyBuildRequest(self, logger): |
853 | """Assert some pre-build checks. |
854 | |
855 | @@ -137,19 +130,19 @@ |
856 | build.distro_series.name)) |
857 | |
858 | @defer.inlineCallbacks |
859 | - def _extraBuildArgs(self, build, logger=None): |
860 | + def extraBuildArgs(self, logger=None): |
861 | """ |
862 | Return the extra arguments required by the slave for the given build. |
863 | """ |
864 | + build = self.build |
865 | das = build.distro_arch_series |
866 | |
867 | # Build extra arguments. |
868 | - args = {} |
869 | + args = yield super(BinaryPackageBuildBehaviour, self).extraBuildArgs( |
870 | + logger=logger) |
871 | args['arch_indep'] = build.arch_indep |
872 | args['distribution'] = das.distroseries.distribution.name |
873 | - args['series'] = das.distroseries.name |
874 | args['suite'] = das.distroseries.getSuite(build.pocket) |
875 | - args['arch_tag'] = das.architecturetag |
876 | |
877 | archive_purpose = build.archive.purpose |
878 | if (archive_purpose == ArchivePurpose.PPA and |
879 | @@ -171,8 +164,6 @@ |
880 | args['archives'], args['trusted_keys'] = ( |
881 | yield get_sources_list_for_building( |
882 | build, das, build.source_package_release.name, logger=logger)) |
883 | - args['archive_private'] = build.archive.private |
884 | - args['build_url'] = canonical_url(build) |
885 | args['build_debug_symbols'] = build.archive.build_debug_symbols |
886 | |
887 | defer.returnValue(args) |
888 | |
889 | === modified file 'lib/lp/soyuz/model/livefsbuildbehaviour.py' |
890 | --- lib/lp/soyuz/model/livefsbuildbehaviour.py 2018-03-01 17:36:31 +0000 |
891 | +++ lib/lp/soyuz/model/livefsbuildbehaviour.py 2018-05-09 17:22:34 +0000 |
892 | @@ -1,4 +1,4 @@ |
893 | -# Copyright 2014-2017 Canonical Ltd. This software is licensed under the |
894 | +# Copyright 2014-2018 Canonical Ltd. This software is licensed under the |
895 | # GNU Affero General Public License version 3 (see the file LICENSE). |
896 | |
897 | """An `IBuildFarmJobBehaviour` for `LiveFSBuild`. |
898 | @@ -24,7 +24,6 @@ |
899 | BuildFarmJobBehaviourBase, |
900 | ) |
901 | from lp.registry.interfaces.series import SeriesStatus |
902 | -from lp.services.webapp import canonical_url |
903 | from lp.soyuz.adapters.archivedependencies import ( |
904 | get_sources_list_for_building, |
905 | ) |
906 | @@ -38,6 +37,8 @@ |
907 | class LiveFSBuildBehaviour(BuildFarmJobBehaviourBase): |
908 | """Dispatches `LiveFSBuild` jobs to slaves.""" |
909 | |
910 | + builder_type = "livefs" |
911 | + |
912 | def getLogFileName(self): |
913 | das = self.build.distro_arch_series |
914 | archname = das.architecturetag |
915 | @@ -76,33 +77,28 @@ |
916 | "Missing chroot for %s" % build.distro_arch_series.displayname) |
917 | |
918 | @defer.inlineCallbacks |
919 | - def _extraBuildArgs(self, logger=None): |
920 | + def extraBuildArgs(self, logger=None): |
921 | """ |
922 | Return the extra arguments required by the slave for the given build. |
923 | """ |
924 | build = self.build |
925 | + base_args = yield super(LiveFSBuildBehaviour, self).extraBuildArgs( |
926 | + logger=logger) |
927 | # Non-trivial metadata values may have been security-wrapped, which |
928 | # is pointless here and just gets in the way of xmlrpclib |
929 | # serialisation. |
930 | args = dict(removeSecurityProxy(build.livefs.metadata)) |
931 | if build.metadata_override is not None: |
932 | args.update(removeSecurityProxy(build.metadata_override)) |
933 | - args["series"] = build.distro_series.name |
934 | + # Everything else overrides anything in the metadata. |
935 | + args.update(base_args) |
936 | args["pocket"] = build.pocket.name.lower() |
937 | - args["arch_tag"] = build.distro_arch_series.architecturetag |
938 | args["datestamp"] = build.version |
939 | args["archives"], args["trusted_keys"] = ( |
940 | yield get_sources_list_for_building( |
941 | build, build.distro_arch_series, None, logger=logger)) |
942 | - args["archive_private"] = build.archive.private |
943 | - args["build_url"] = canonical_url(build) |
944 | defer.returnValue(args) |
945 | |
946 | - @defer.inlineCallbacks |
947 | - def composeBuildRequest(self, logger): |
948 | - args = yield self._extraBuildArgs(logger=logger) |
949 | - defer.returnValue(("livefs", self.build.distro_arch_series, {}, args)) |
950 | - |
951 | def verifySuccessfulBuild(self): |
952 | """See `IBuildFarmJobBehaviour`.""" |
953 | # The implementation in BuildFarmJobBehaviourBase checks whether the |
954 | |
955 | === modified file 'lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py' |
956 | --- lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py 2018-03-01 17:36:31 +0000 |
957 | +++ lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py 2018-05-09 17:22:34 +0000 |
958 | @@ -143,6 +143,7 @@ |
959 | 'build_debug_symbols': archive.build_debug_symbols, |
960 | 'build_url': canonical_url(build), |
961 | 'distribution': das.distroseries.distribution.name, |
962 | + 'fast_cleanup': builder.virtualized, |
963 | 'ogrecomponent': component, |
964 | 'series': ds_name, |
965 | 'suite': suite, |
966 | @@ -333,18 +334,24 @@ |
967 | @defer.inlineCallbacks |
968 | def test_arch_indep(self): |
969 | # BinaryPackageBuild.arch_indep is passed through to the slave. |
970 | + builder = self.factory.makeBuilder() |
971 | build = self.factory.makeBinaryPackageBuild(arch_indep=False) |
972 | - extra_args = yield IBuildFarmJobBehaviour(build)._extraBuildArgs(build) |
973 | + behaviour = IBuildFarmJobBehaviour(build) |
974 | + behaviour.setBuilder(builder, None) |
975 | + extra_args = yield behaviour.extraBuildArgs() |
976 | self.assertFalse(extra_args['arch_indep']) |
977 | build = self.factory.makeBinaryPackageBuild(arch_indep=True) |
978 | - extra_args = yield IBuildFarmJobBehaviour(build)._extraBuildArgs(build) |
979 | + behaviour = IBuildFarmJobBehaviour(build) |
980 | + behaviour.setBuilder(builder, None) |
981 | + extra_args = yield behaviour.extraBuildArgs() |
982 | self.assertTrue(extra_args['arch_indep']) |
983 | |
984 | @defer.inlineCallbacks |
985 | def test_extraBuildArgs_archive_trusted_keys(self): |
986 | - # If the archive has a signing key, _extraBuildArgs sends it. |
987 | + # If the archive has a signing key, extraBuildArgs sends it. |
988 | yield self.useFixture(InProcessKeyServerFixture()).start() |
989 | archive = self.factory.makeArchive() |
990 | + builder = self.factory.makeBuilder() |
991 | key_path = os.path.join(gpgkeysdir, "ppa-sample@canonical.com.sec") |
992 | yield IArchiveSigningKey(archive).setSigningKey( |
993 | key_path, async_keyserver=True) |
994 | @@ -352,7 +359,9 @@ |
995 | self.factory.makeBinaryPackagePublishingHistory( |
996 | distroarchseries=build.distro_arch_series, pocket=build.pocket, |
997 | archive=archive, status=PackagePublishingStatus.PUBLISHED) |
998 | - args = yield IBuildFarmJobBehaviour(build)._extraBuildArgs(build) |
999 | + behaviour = IBuildFarmJobBehaviour(build) |
1000 | + behaviour.setBuilder(builder, None) |
1001 | + args = yield behaviour.extraBuildArgs() |
1002 | self.assertThat(args["trusted_keys"], MatchesListwise([ |
1003 | Base64KeyMatches("0D57E99656BEFB0897606EE9A022DD1F5001B46D"), |
1004 | ])) |
1005 | |
1006 | === modified file 'lib/lp/soyuz/tests/test_livefsbuildbehaviour.py' |
1007 | --- lib/lp/soyuz/tests/test_livefsbuildbehaviour.py 2018-03-01 17:36:31 +0000 |
1008 | +++ lib/lp/soyuz/tests/test_livefsbuildbehaviour.py 2018-05-09 17:22:34 +0000 |
1009 | @@ -69,7 +69,7 @@ |
1010 | self.useFixture(FeatureFixture({LIVEFS_FEATURE_FLAG: "on"})) |
1011 | |
1012 | def makeJob(self, archive=None, pocket=PackagePublishingPocket.RELEASE, |
1013 | - **kwargs): |
1014 | + with_builder=False, **kwargs): |
1015 | """Create a sample `ILiveFSBuildBehaviour`.""" |
1016 | if archive is None: |
1017 | distribution = self.factory.makeDistribution(name="distro") |
1018 | @@ -84,7 +84,12 @@ |
1019 | build = self.factory.makeLiveFSBuild( |
1020 | archive=archive, distroarchseries=distroarchseries, pocket=pocket, |
1021 | name="test-livefs", **kwargs) |
1022 | - return IBuildFarmJobBehaviour(build) |
1023 | + job = IBuildFarmJobBehaviour(build) |
1024 | + if with_builder: |
1025 | + builder = MockBuilder() |
1026 | + builder.processor = processor |
1027 | + job.setBuilder(builder, None) |
1028 | + return job |
1029 | |
1030 | |
1031 | class TestLiveFSBuildBehaviour(TestLiveFSBuildBehaviourBase): |
1032 | @@ -185,20 +190,22 @@ |
1033 | |
1034 | @defer.inlineCallbacks |
1035 | def test_extraBuildArgs(self): |
1036 | - # _extraBuildArgs returns a reasonable set of additional arguments. |
1037 | + # extraBuildArgs returns a reasonable set of additional arguments. |
1038 | job = self.makeJob( |
1039 | date_created=datetime(2014, 4, 25, 10, 38, 0, tzinfo=pytz.UTC), |
1040 | - metadata={"project": "distro", "subproject": "special"}) |
1041 | + metadata={"project": "distro", "subproject": "special"}, |
1042 | + with_builder=True) |
1043 | expected_archives, expected_trusted_keys = ( |
1044 | yield get_sources_list_for_building( |
1045 | job.build, job.build.distro_arch_series, None)) |
1046 | - extra_args = yield job._extraBuildArgs() |
1047 | + extra_args = yield job.extraBuildArgs() |
1048 | self.assertEqual({ |
1049 | "archive_private": False, |
1050 | "archives": expected_archives, |
1051 | "arch_tag": "i386", |
1052 | "build_url": canonical_url(job.build), |
1053 | "datestamp": "20140425-103800", |
1054 | + "fast_cleanup": True, |
1055 | "pocket": "release", |
1056 | "project": "distro", |
1057 | "subproject": "special", |
1058 | @@ -208,50 +215,62 @@ |
1059 | |
1060 | @defer.inlineCallbacks |
1061 | def test_extraBuildArgs_proposed(self): |
1062 | - # _extraBuildArgs returns appropriate arguments if asked to build a |
1063 | + # extraBuildArgs returns appropriate arguments if asked to build a |
1064 | # job for -proposed. |
1065 | job = self.makeJob( |
1066 | pocket=PackagePublishingPocket.PROPOSED, |
1067 | - metadata={"project": "distro"}) |
1068 | - args = yield job._extraBuildArgs() |
1069 | + metadata={"project": "distro"}, with_builder=True) |
1070 | + args = yield job.extraBuildArgs() |
1071 | self.assertEqual("unstable", args["series"]) |
1072 | self.assertEqual("proposed", args["pocket"]) |
1073 | |
1074 | @defer.inlineCallbacks |
1075 | def test_extraBuildArgs_no_security_proxy(self): |
1076 | - # _extraBuildArgs returns an object without security wrapping, even |
1077 | + # extraBuildArgs returns an object without security wrapping, even |
1078 | # if values in the metadata are (say) lists and hence get proxied by |
1079 | # Zope. |
1080 | - job = self.makeJob(metadata={"lb_args": ["--option=value"]}) |
1081 | - args = yield job._extraBuildArgs() |
1082 | + job = self.makeJob( |
1083 | + metadata={"lb_args": ["--option=value"]}, with_builder=True) |
1084 | + args = yield job.extraBuildArgs() |
1085 | self.assertEqual(["--option=value"], args["lb_args"]) |
1086 | self.assertIsNot(Proxy, type(args["lb_args"])) |
1087 | |
1088 | @defer.inlineCallbacks |
1089 | def test_extraBuildArgs_archive_trusted_keys(self): |
1090 | - # If the archive has a signing key, _extraBuildArgs sends it. |
1091 | + # If the archive has a signing key, extraBuildArgs sends it. |
1092 | yield self.useFixture(InProcessKeyServerFixture()).start() |
1093 | archive = self.factory.makeArchive() |
1094 | key_path = os.path.join(gpgkeysdir, "ppa-sample@canonical.com.sec") |
1095 | yield IArchiveSigningKey(archive).setSigningKey( |
1096 | key_path, async_keyserver=True) |
1097 | - job = self.makeJob(archive=archive) |
1098 | + job = self.makeJob(archive=archive, with_builder=True) |
1099 | self.factory.makeBinaryPackagePublishingHistory( |
1100 | distroarchseries=job.build.distro_arch_series, |
1101 | pocket=job.build.pocket, archive=archive, |
1102 | status=PackagePublishingStatus.PUBLISHED) |
1103 | - args = yield job._extraBuildArgs() |
1104 | + args = yield job.extraBuildArgs() |
1105 | self.assertThat(args["trusted_keys"], MatchesListwise([ |
1106 | Base64KeyMatches("0D57E99656BEFB0897606EE9A022DD1F5001B46D"), |
1107 | ])) |
1108 | |
1109 | @defer.inlineCallbacks |
1110 | + def test_extraBuildArgs_metadata_cannot_override_base(self): |
1111 | + # Items in the user-provided metadata cannot override the base |
1112 | + # arguments. |
1113 | + job = self.makeJob( |
1114 | + metadata={"project": "distro", "arch_tag": "nonsense"}, |
1115 | + with_builder=True) |
1116 | + args = yield job.extraBuildArgs() |
1117 | + self.assertEqual("distro", args["project"]) |
1118 | + self.assertEqual("i386", args["arch_tag"]) |
1119 | + |
1120 | + @defer.inlineCallbacks |
1121 | def test_composeBuildRequest(self): |
1122 | - job = self.makeJob() |
1123 | + job = self.makeJob(with_builder=True) |
1124 | lfa = self.factory.makeLibraryFileAlias(db_only=True) |
1125 | job.build.distro_arch_series.addOrUpdateChroot(lfa) |
1126 | build_request = yield job.composeBuildRequest(None) |
1127 | - args = yield job._extraBuildArgs() |
1128 | + args = yield job.extraBuildArgs() |
1129 | self.assertEqual( |
1130 | ('livefs', job.build.distro_arch_series, {}, args), build_request) |
1131 |