Merge ~twom/launchpad:oci-buildbehaviour into launchpad:master
- Git
- lp:~twom/launchpad
- oci-buildbehaviour
- Merge into master
Proposed by
Tom Wardill
Status: | Merged |
---|---|
Approved by: | Tom Wardill |
Approved revision: | 47145664aa7899c4393aff34ccbba9536ac7775b |
Merge reported by: | Otto Co-Pilot |
Merged at revision: | not available |
Proposed branch: | ~twom/launchpad:oci-buildbehaviour |
Merge into: | launchpad:master |
Diff against target: |
1557 lines (+1067/-166) 15 files modified
database/schema/security.cfg (+1/-0) lib/lp/buildmaster/model/buildfarmjobbehaviour.py (+17/-13) lib/lp/buildmaster/tests/mock_slaves.py (+9/-3) lib/lp/buildmaster/tests/snapbuildproxy.py (+123/-0) lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py (+1/-1) lib/lp/oci/configure.zcml (+6/-0) lib/lp/oci/interfaces/ocirecipebuild.py (+6/-0) lib/lp/oci/model/ocirecipebuild.py (+25/-4) lib/lp/oci/model/ocirecipebuildbehaviour.py (+182/-0) lib/lp/oci/tests/test_ocirecipebuild.py (+11/-1) lib/lp/oci/tests/test_ocirecipebuildbehaviour.py (+604/-0) lib/lp/snappy/model/snapbuildbehaviour.py (+52/-43) lib/lp/snappy/tests/test_snapbuildbehaviour.py (+18/-96) lib/lp/testing/factory.py (+11/-4) lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py (+1/-1) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Colin Watson (community) | Approve | ||
Review via email: mp+379201@code.launchpad.net |
This proposal supersedes a proposal from 2019-12-12.
Commit message
Add OCIRecipeBuildB
Description of the change
Add a buildbehaviour to drive OCI image builds.
Use mixin tests where sensible, override for places where the OCI methods differ.
To post a comment you must log in.
Revision history for this message
Colin Watson (cjwatson) : Posted in a previous version of this proposal | # |
review:
Needs Fixing
Revision history for this message
Tom Wardill (twom) : Posted in a previous version of this proposal | # |
Revision history for this message
Colin Watson (cjwatson) : Posted in a previous version of this proposal | # |
Revision history for this message
Colin Watson (cjwatson) : | # |
review:
Needs Fixing
Revision history for this message
Colin Watson (cjwatson) : | # |
Revision history for this message
Colin Watson (cjwatson) : | # |
review:
Needs Fixing
Revision history for this message
Colin Watson (cjwatson) : | # |
review:
Approve
Revision history for this message
Colin Watson (cjwatson) : | # |
review:
Approve
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/database/schema/security.cfg b/database/schema/security.cfg |
2 | index 306d206..d0832c1 100644 |
3 | --- a/database/schema/security.cfg |
4 | +++ b/database/schema/security.cfg |
5 | @@ -995,6 +995,7 @@ public.livefsbuild = SELECT, UPDATE |
6 | public.livefsfile = SELECT |
7 | public.ocifile = SELECT |
8 | public.ociproject = SELECT |
9 | +public.ociprojectname = SELECT |
10 | public.ocirecipe = SELECT |
11 | public.ocirecipebuild = SELECT, UPDATE |
12 | public.openididentifier = SELECT |
13 | diff --git a/lib/lp/buildmaster/model/buildfarmjobbehaviour.py b/lib/lp/buildmaster/model/buildfarmjobbehaviour.py |
14 | index 2f3515a..4b28693 100644 |
15 | --- a/lib/lp/buildmaster/model/buildfarmjobbehaviour.py |
16 | +++ b/lib/lp/buildmaster/model/buildfarmjobbehaviour.py |
17 | @@ -301,6 +301,22 @@ class BuildFarmJobBehaviourBase: |
18 | transaction.commit() |
19 | |
20 | @defer.inlineCallbacks |
21 | + def _downloadFiles(self, filemap, upload_path, logger): |
22 | + filenames_to_download = [] |
23 | + for filename, sha1 in filemap.items(): |
24 | + logger.info("Grabbing file: %s (%s)" % ( |
25 | + filename, self._slave.getURL(sha1))) |
26 | + out_file_name = os.path.join(upload_path, filename) |
27 | + # If the evaluated output file name is not within our |
28 | + # upload path, then we don't try to copy this or any |
29 | + # subsequent files. |
30 | + if not os.path.realpath(out_file_name).startswith(upload_path): |
31 | + raise BuildDaemonError( |
32 | + "Build returned a file named %r." % filename) |
33 | + filenames_to_download.append((sha1, out_file_name)) |
34 | + yield self._slave.getFiles(filenames_to_download, logger=logger) |
35 | + |
36 | + @defer.inlineCallbacks |
37 | def handleSuccess(self, slave_status, logger): |
38 | """Handle a package that built successfully. |
39 | |
40 | @@ -337,19 +353,7 @@ class BuildFarmJobBehaviourBase: |
41 | grab_dir, str(build.archive.id), build.distribution.name) |
42 | os.makedirs(upload_path) |
43 | |
44 | - filenames_to_download = [] |
45 | - for filename, sha1 in filemap.items(): |
46 | - logger.info("Grabbing file: %s (%s)" % ( |
47 | - filename, self._slave.getURL(sha1))) |
48 | - out_file_name = os.path.join(upload_path, filename) |
49 | - # If the evaluated output file name is not within our |
50 | - # upload path, then we don't try to copy this or any |
51 | - # subsequent files. |
52 | - if not os.path.realpath(out_file_name).startswith(upload_path): |
53 | - raise BuildDaemonError( |
54 | - "Build returned a file named %r." % filename) |
55 | - filenames_to_download.append((sha1, out_file_name)) |
56 | - yield self._slave.getFiles(filenames_to_download, logger=logger) |
57 | + yield self._downloadFiles(filemap, upload_path, logger) |
58 | |
59 | transaction.commit() |
60 | |
61 | diff --git a/lib/lp/buildmaster/tests/mock_slaves.py b/lib/lp/buildmaster/tests/mock_slaves.py |
62 | index a19a85d..32633ea 100644 |
63 | --- a/lib/lp/buildmaster/tests/mock_slaves.py |
64 | +++ b/lib/lp/buildmaster/tests/mock_slaves.py |
65 | @@ -194,7 +194,8 @@ class WaitingSlave(OkSlave): |
66 | |
67 | # By default, the slave only has a buildlog, but callsites |
68 | # can update this list as needed. |
69 | - self.valid_file_hashes = ['buildlog'] |
70 | + self.valid_files = {'buildlog': ''} |
71 | + self._got_file_record = [] |
72 | |
73 | def status(self): |
74 | self.call_log.append('status') |
75 | @@ -208,12 +209,17 @@ class WaitingSlave(OkSlave): |
76 | |
77 | def getFile(self, hash, file_to_write): |
78 | self.call_log.append('getFile') |
79 | - if hash in self.valid_file_hashes: |
80 | - content = "This is a %s" % hash |
81 | + if hash in self.valid_files: |
82 | if isinstance(file_to_write, types.StringTypes): |
83 | file_to_write = open(file_to_write, 'wb') |
84 | + if not self.valid_files[hash]: |
85 | + content = b"This is a %s" % hash |
86 | + else: |
87 | + with open(self.valid_files[hash], 'rb') as source: |
88 | + content = source.read() |
89 | file_to_write.write(content) |
90 | file_to_write.close() |
91 | + self._got_file_record.append(hash) |
92 | return defer.succeed(None) |
93 | |
94 | |
95 | diff --git a/lib/lp/buildmaster/tests/snapbuildproxy.py b/lib/lp/buildmaster/tests/snapbuildproxy.py |
96 | new file mode 100644 |
97 | index 0000000..21d973e |
98 | --- /dev/null |
99 | +++ b/lib/lp/buildmaster/tests/snapbuildproxy.py |
100 | @@ -0,0 +1,123 @@ |
101 | +# Copyright 2015-2019 Canonical Ltd. This software is licensed under the |
102 | +# GNU Affero General Public License version 3 (see the file LICENSE). |
103 | + |
104 | +"""Fixtures for dealing with the build time 'snap' HTTP proxy.""" |
105 | + |
106 | +from __future__ import absolute_import, print_function, unicode_literals |
107 | + |
108 | +__metaclass__ = type |
109 | + |
110 | +from datetime import datetime |
111 | +import json |
112 | +from textwrap import dedent |
113 | +import uuid |
114 | + |
115 | +import fixtures |
116 | +from six.moves.urllib_parse import urlsplit |
117 | +from testtools.matchers import ( |
118 | + Equals, |
119 | + HasLength, |
120 | + MatchesStructure, |
121 | + ) |
122 | +from twisted.internet import ( |
123 | + defer, |
124 | + endpoints, |
125 | + reactor, |
126 | + ) |
127 | +from twisted.python.compat import nativeString |
128 | +from twisted.web import ( |
129 | + resource, |
130 | + server, |
131 | + ) |
132 | + |
133 | +from lp.services.config import config |
134 | + |
135 | + |
136 | +class ProxyAuthAPITokensResource(resource.Resource): |
137 | + """A test tokens resource for the proxy authentication API.""" |
138 | + |
139 | + isLeaf = True |
140 | + |
141 | + def __init__(self): |
142 | + resource.Resource.__init__(self) |
143 | + self.requests = [] |
144 | + |
145 | + def render_POST(self, request): |
146 | + content = request.content.read() |
147 | + self.requests.append({ |
148 | + "method": request.method, |
149 | + "uri": request.uri, |
150 | + "headers": dict(request.requestHeaders.getAllRawHeaders()), |
151 | + "content": content, |
152 | + }) |
153 | + username = json.loads(content)["username"] |
154 | + return json.dumps({ |
155 | + "username": username, |
156 | + "secret": uuid.uuid4().hex, |
157 | + "timestamp": datetime.utcnow().isoformat(), |
158 | + }) |
159 | + |
160 | + |
161 | +class InProcessProxyAuthAPIFixture(fixtures.Fixture): |
162 | + """A fixture that pretends to be the proxy authentication API. |
163 | + |
164 | + Users of this fixture must call the `start` method, which returns a |
165 | + `Deferred`, and arrange for that to get back to the reactor. This is |
166 | + necessary because the basic fixture API does not allow `setUp` to return |
167 | + anything. For example: |
168 | + |
169 | + class TestSomething(TestCase): |
170 | + |
171 | + run_tests_with = AsynchronousDeferredRunTest.make_factory( |
172 | + timeout=10) |
173 | + |
174 | + @defer.inlineCallbacks |
175 | + def setUp(self): |
176 | + super(TestSomething, self).setUp() |
177 | + yield self.useFixture(InProcessProxyAuthAPIFixture()).start() |
178 | + """ |
179 | + |
180 | + @defer.inlineCallbacks |
181 | + def start(self): |
182 | + root = resource.Resource() |
183 | + self.tokens = ProxyAuthAPITokensResource() |
184 | + root.putChild("tokens", self.tokens) |
185 | + endpoint = endpoints.serverFromString(reactor, nativeString("tcp:0")) |
186 | + site = server.Site(self.tokens) |
187 | + self.addCleanup(site.stopFactory) |
188 | + port = yield endpoint.listen(site) |
189 | + self.addCleanup(port.stopListening) |
190 | + config.push("in-process-proxy-auth-api-fixture", dedent(""" |
191 | + [snappy] |
192 | + builder_proxy_auth_api_admin_secret: admin-secret |
193 | + builder_proxy_auth_api_endpoint: http://%s:%s/tokens |
194 | + """) % |
195 | + (port.getHost().host, port.getHost().port)) |
196 | + self.addCleanup(config.pop, "in-process-proxy-auth-api-fixture") |
197 | + |
198 | + |
199 | +class ProxyURLMatcher(MatchesStructure): |
200 | + """Check that a string is a valid url for a snap build proxy.""" |
201 | + |
202 | + def __init__(self, job, now): |
203 | + super(ProxyURLMatcher, self).__init__( |
204 | + scheme=Equals("http"), |
205 | + username=Equals("{}-{}".format( |
206 | + job.build.build_cookie, int(now))), |
207 | + password=HasLength(32), |
208 | + hostname=Equals(config.snappy.builder_proxy_host), |
209 | + port=Equals(config.snappy.builder_proxy_port), |
210 | + path=Equals("")) |
211 | + |
212 | + def match(self, matchee): |
213 | + super(ProxyURLMatcher, self).match(urlsplit(matchee)) |
214 | + |
215 | + |
216 | +class RevocationEndpointMatcher(Equals): |
217 | + """Check that a string is a valid endpoint for proxy token revocation.""" |
218 | + |
219 | + def __init__(self, job, now): |
220 | + super(RevocationEndpointMatcher, self).__init__( |
221 | + "{}/{}-{}".format( |
222 | + config.snappy.builder_proxy_auth_api_endpoint, |
223 | + job.build.build_cookie, int(now))) |
224 | diff --git a/lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py b/lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py |
225 | index 697ff52..5046d8c 100644 |
226 | --- a/lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py |
227 | +++ b/lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py |
228 | @@ -337,7 +337,7 @@ class TestHandleStatusMixin: |
229 | self.builder = self.factory.makeBuilder() |
230 | self.build.buildqueue_record.markAsBuilding(self.builder) |
231 | self.slave = WaitingSlave('BuildStatus.OK') |
232 | - self.slave.valid_file_hashes.append('test_file_hash') |
233 | + self.slave.valid_files['test_file_hash'] = '' |
234 | self.interactor = BuilderInteractor() |
235 | self.behaviour = self.interactor.getBuildBehaviour( |
236 | self.build.buildqueue_record, self.builder, self.slave) |
237 | diff --git a/lib/lp/oci/configure.zcml b/lib/lp/oci/configure.zcml |
238 | index b86afcb..a01178a 100644 |
239 | --- a/lib/lp/oci/configure.zcml |
240 | +++ b/lib/lp/oci/configure.zcml |
241 | @@ -63,4 +63,10 @@ |
242 | <allow interface="lp.buildmaster.interfaces.buildfarmjob.ISpecificBuildFarmJobSource" /> |
243 | </securedutility> |
244 | |
245 | + <adapter |
246 | + for="lp.oci.interfaces.ocirecipebuild.IOCIRecipeBuild" |
247 | + provides="lp.buildmaster.interfaces.buildfarmjobbehaviour.IBuildFarmJobBehaviour" |
248 | + factory="lp.oci.model.ocirecipebuildbehaviour.OCIRecipeBuildBehaviour" |
249 | + permission="zope.Public" /> |
250 | + |
251 | </configure> |
252 | diff --git a/lib/lp/oci/interfaces/ocirecipebuild.py b/lib/lp/oci/interfaces/ocirecipebuild.py |
253 | index 30c3835..a41e2c8 100644 |
254 | --- a/lib/lp/oci/interfaces/ocirecipebuild.py |
255 | +++ b/lib/lp/oci/interfaces/ocirecipebuild.py |
256 | @@ -27,6 +27,7 @@ from lp.oci.interfaces.ocirecipe import IOCIRecipe |
257 | from lp.services.database.constants import DEFAULT |
258 | from lp.services.fields import PublicPersonChoice |
259 | from lp.services.librarian.interfaces import ILibraryFileAlias |
260 | +from lp.soyuz.interfaces.distroarchseries import IDistroArchSeries |
261 | |
262 | |
263 | class IOCIRecipeBuildEdit(Interface): |
264 | @@ -82,6 +83,11 @@ class IOCIRecipeBuildView(IPackageBuild): |
265 | :return: The corresponding `ILibraryFileAlias`. |
266 | """ |
267 | |
268 | + distro_arch_series = Reference( |
269 | + IDistroArchSeries, |
270 | + title=_("The series and architecture for which to build."), |
271 | + required=True, readonly=True) |
272 | + |
273 | |
274 | class IOCIRecipeBuildAdmin(Interface): |
275 | # XXX twom 2020-02-10 This will probably need rescore() implementing |
276 | diff --git a/lib/lp/oci/model/ocirecipebuild.py b/lib/lp/oci/model/ocirecipebuild.py |
277 | index c010817..12688cd 100644 |
278 | --- a/lib/lp/oci/model/ocirecipebuild.py |
279 | +++ b/lib/lp/oci/model/ocirecipebuild.py |
280 | @@ -44,7 +44,9 @@ from lp.oci.interfaces.ocirecipebuild import ( |
281 | IOCIRecipeBuild, |
282 | IOCIRecipeBuildSet, |
283 | ) |
284 | +from lp.registry.interfaces.pocket import PackagePublishingPocket |
285 | from lp.registry.model.person import Person |
286 | +from lp.services.config import config |
287 | from lp.services.database.bulk import load_related |
288 | from lp.services.database.constants import DEFAULT |
289 | from lp.services.database.decoratedresultset import DecoratedResultSet |
290 | @@ -128,10 +130,16 @@ class OCIRecipeBuild(PackageBuildMixin, Storm): |
291 | build_farm_job_id = Int(name='build_farm_job', allow_none=False) |
292 | build_farm_job = Reference(build_farm_job_id, 'BuildFarmJob.id') |
293 | |
294 | - # Stub attributes to match the IPackageBuild interface that we |
295 | - # are not using in this implementation at this time. |
296 | - pocket = None |
297 | - distro_series = None |
298 | + # We only care about the pocket from a building environment POV, |
299 | + # it is not a target, nor referenced in the final build. |
300 | + pocket = PackagePublishingPocket.UPDATES |
301 | + |
302 | + @property |
303 | + def distro_series(self): |
304 | + # XXX twom 2020-02-14 - This really needs to be set elsewhere, |
305 | + # as this may not be an LTS release and ties the OCI target to |
306 | + # a completely unrelated process. |
307 | + return self.distribution.currentseries |
308 | |
309 | def __init__(self, build_farm_job, requester, recipe, |
310 | processor, virtualized, date_created): |
311 | @@ -257,6 +265,19 @@ class OCIRecipeBuild(PackageBuildMixin, Storm): |
312 | # pillar isn't just a distribution |
313 | return self.recipe.oci_project.distribution |
314 | |
315 | + @property |
316 | + def distro_arch_series(self): |
317 | + return self.distribution.currentseries.getDistroArchSeriesByProcessor( |
318 | + self.processor) |
319 | + |
320 | + def notify(self, extra_info=None): |
321 | + """See `IPackageBuild`.""" |
322 | + if not config.builddmaster.send_build_notification: |
323 | + return |
324 | + if self.status == BuildStatus.FULLYBUILT: |
325 | + return |
326 | + # XXX twom 2019-12-11 This should send mail |
327 | + |
328 | |
329 | @implementer(IOCIRecipeBuildSet) |
330 | class OCIRecipeBuildSet(SpecificBuildFarmJobSourceMixin): |
331 | diff --git a/lib/lp/oci/model/ocirecipebuildbehaviour.py b/lib/lp/oci/model/ocirecipebuildbehaviour.py |
332 | new file mode 100644 |
333 | index 0000000..f0debfa |
334 | --- /dev/null |
335 | +++ b/lib/lp/oci/model/ocirecipebuildbehaviour.py |
336 | @@ -0,0 +1,182 @@ |
337 | +# Copyright 2019 Canonical Ltd. This software is licensed under the |
338 | +# GNU Affero General Public License version 3 (see the file LICENSE). |
339 | + |
340 | +"""An `IBuildFarmJobBehaviour` for `OCIRecipeBuild`. |
341 | + |
342 | +Dispatches OCI image build jobs to build-farm slaves. |
343 | +""" |
344 | + |
345 | +from __future__ import absolute_import, print_function, unicode_literals |
346 | + |
347 | +__metaclass__ = type |
348 | +__all__ = [ |
349 | + 'OCIRecipeBuildBehaviour', |
350 | + ] |
351 | + |
352 | + |
353 | +import json |
354 | +import os |
355 | + |
356 | +from twisted.internet import defer |
357 | +from zope.interface import implementer |
358 | + |
359 | +from lp.app.errors import NotFoundError |
360 | +from lp.buildmaster.enums import BuildBaseImageType |
361 | +from lp.buildmaster.interfaces.builder import ( |
362 | + BuildDaemonError, |
363 | + CannotBuild, |
364 | + ) |
365 | +from lp.buildmaster.interfaces.buildfarmjobbehaviour import ( |
366 | + IBuildFarmJobBehaviour, |
367 | + ) |
368 | +from lp.buildmaster.model.buildfarmjobbehaviour import ( |
369 | + BuildFarmJobBehaviourBase, |
370 | + ) |
371 | +from lp.registry.interfaces.series import SeriesStatus |
372 | +from lp.services.librarian.utils import copy_and_close |
373 | +from lp.snappy.model.snapbuildbehaviour import SnapProxyMixin |
374 | +from lp.soyuz.adapters.archivedependencies import ( |
375 | + get_sources_list_for_building, |
376 | + ) |
377 | + |
378 | + |
379 | +@implementer(IBuildFarmJobBehaviour) |
380 | +class OCIRecipeBuildBehaviour(SnapProxyMixin, BuildFarmJobBehaviourBase): |
381 | + |
382 | + builder_type = "oci" |
383 | + image_types = [BuildBaseImageType.LXD, BuildBaseImageType.CHROOT] |
384 | + |
385 | + def getLogFileName(self): |
386 | + series = self.build.distro_series |
387 | + |
388 | + # Examples: |
389 | + # buildlog_oci_ubuntu_wily_amd64_name_FULLYBUILT.txt |
390 | + return 'buildlog_oci_%s_%s_%s_%s_%s' % ( |
391 | + series.distribution.name, series.name, |
392 | + self.build.processor.name, self.build.recipe.name, |
393 | + self.build.status.name) |
394 | + |
395 | + def verifyBuildRequest(self, logger): |
396 | + """Assert some pre-build checks. |
397 | + |
398 | + The build request is checked: |
399 | + * Virtualized builds can't build on a non-virtual builder |
400 | + * Ensure that we have a chroot |
401 | + """ |
402 | + build = self.build |
403 | + if build.virtualized and not self._builder.virtualized: |
404 | + raise AssertionError( |
405 | + "Attempt to build virtual item on a non-virtual builder.") |
406 | + |
407 | + chroot = build.distro_arch_series.getChroot(pocket=build.pocket) |
408 | + if chroot is None: |
409 | + raise CannotBuild( |
410 | + "Missing chroot for %s" % build.distro_arch_series.displayname) |
411 | + |
412 | + @defer.inlineCallbacks |
413 | + def extraBuildArgs(self, logger=None): |
414 | + """ |
415 | + Return the extra arguments required by the slave for the given build. |
416 | + """ |
417 | + build = self.build |
418 | + args = yield super(OCIRecipeBuildBehaviour, self).extraBuildArgs( |
419 | + logger=logger) |
420 | + yield self.addProxyArgs(args) |
421 | + # XXX twom 2020-02-17 This may need to be more complex, and involve |
422 | + # distribution name. |
423 | + args["name"] = build.recipe.name |
424 | + args["archives"], args["trusted_keys"] = ( |
425 | + yield get_sources_list_for_building( |
426 | + build, build.distro_arch_series, None, |
427 | + tools_source=None, tools_fingerprint=None, |
428 | + logger=logger)) |
429 | + |
430 | + args['build_file'] = build.recipe.build_file |
431 | + |
432 | + if build.recipe.git_ref is not None: |
433 | + args["git_repository"] = ( |
434 | + build.recipe.git_repository.git_https_url) |
435 | + else: |
436 | + raise CannotBuild( |
437 | + "Source repository for ~%s/%s has been deleted." % |
438 | + (build.recipe.owner.name, build.recipe.name)) |
439 | + |
440 | + if build.recipe.git_path != "HEAD": |
441 | + args["git_path"] = build.recipe.git_ref.name |
442 | + |
443 | + defer.returnValue(args) |
444 | + |
445 | + def _ensureFilePath(self, file_name, file_path, upload_path): |
446 | + # If the evaluated output file name is not within our |
447 | + # upload path, then we don't try to copy this or any |
448 | + # subsequent files. |
449 | + if not os.path.normpath(file_path).startswith(upload_path + '/'): |
450 | + raise BuildDaemonError( |
451 | + "Build returned a file named '%s'." % file_name) |
452 | + |
453 | + @defer.inlineCallbacks |
454 | + def _fetchIntermediaryFile(self, name, filemap, upload_path): |
455 | + file_hash = filemap[name] |
456 | + file_path = os.path.join(upload_path, name) |
457 | + self._ensureFilePath(name, file_path, upload_path) |
458 | + yield self._slave.getFile(file_hash, file_path) |
459 | + |
460 | + with open(file_path, 'r') as file_fp: |
461 | + contents = json.load(file_fp) |
462 | + defer.returnValue(contents) |
463 | + |
464 | + def _extractLayerFiles(self, upload_path, section, config, digests, files): |
465 | + # These are different sets of ids, in the same order |
466 | + # layer_id is the filename, diff_id is the internal (docker) id |
467 | + for diff_id in config['rootfs']['diff_ids']: |
468 | + for digests_section in digests: |
469 | + layer_id = digests_section[diff_id]['layer_id'] |
470 | + # This is in the form '<id>/layer.tar', we only need the first |
471 | + layer_filename = "{}.tar.gz".format(layer_id.split('/')[0]) |
472 | + digest = digests_section[diff_id]['digest'] |
473 | + try: |
474 | + _, librarian_file, _ = self.build.getLayerFileByDigest( |
475 | + digest) |
476 | + except NotFoundError: |
477 | + files.add(layer_filename) |
478 | + continue |
479 | + layer_path = os.path.join(upload_path, layer_filename) |
480 | + librarian_file.open() |
481 | + copy_and_close(librarian_file, open(layer_path, 'wb')) |
482 | + |
483 | + def _convertToRetrievableFile(self, upload_path, file_name, filemap): |
484 | + file_path = os.path.join(upload_path, file_name) |
485 | + self._ensureFilePath(file_name, file_path, upload_path) |
486 | + return (filemap[file_name], file_path) |
487 | + |
488 | + @defer.inlineCallbacks |
489 | + def _downloadFiles(self, filemap, upload_path, logger): |
490 | + """Download required artifact files.""" |
491 | + # We don't want to download all of the files that have been created, |
492 | + # just the ones that are mentioned in the manifest and config. |
493 | + |
494 | + manifest = yield self._fetchIntermediaryFile( |
495 | + 'manifest.json', filemap, upload_path) |
496 | + digests = yield self._fetchIntermediaryFile( |
497 | + 'digests.json', filemap, upload_path) |
498 | + |
499 | + files = set() |
500 | + for section in manifest: |
501 | + config = yield self._fetchIntermediaryFile( |
502 | + section['Config'], filemap, upload_path) |
503 | + self._extractLayerFiles( |
504 | + upload_path, section, config, digests, files) |
505 | + |
506 | + files_to_download = [ |
507 | + self._convertToRetrievableFile(upload_path, filename, filemap) |
508 | + for filename in files] |
509 | + yield self._slave.getFiles(files_to_download, logger=logger) |
510 | + |
511 | + def verifySuccessfulBuild(self): |
512 | + """See `IBuildFarmJobBehaviour`.""" |
513 | + # The implementation in BuildFarmJobBehaviourBase checks whether the |
514 | + # target suite is modifiable in the target archive. However, an |
515 | + # `OCIRecipeBuild` does not use an archive in this manner. |
516 | + # We do, however, refuse to build for |
517 | + # obsolete series. |
518 | + assert self.build.distro_series.status != SeriesStatus.OBSOLETE |
519 | diff --git a/lib/lp/oci/tests/test_ocirecipebuild.py b/lib/lp/oci/tests/test_ocirecipebuild.py |
520 | index b768e1d..60aaf93 100644 |
521 | --- a/lib/lp/oci/tests/test_ocirecipebuild.py |
522 | +++ b/lib/lp/oci/tests/test_ocirecipebuild.py |
523 | @@ -16,11 +16,13 @@ from lp.app.errors import NotFoundError |
524 | from lp.buildmaster.enums import BuildStatus |
525 | from lp.buildmaster.interfaces.buildqueue import IBuildQueue |
526 | from lp.buildmaster.interfaces.packagebuild import IPackageBuild |
527 | +from lp.buildmaster.interfaces.processor import IProcessorSet |
528 | from lp.oci.interfaces.ocirecipebuild import ( |
529 | IOCIRecipeBuild, |
530 | IOCIRecipeBuildSet, |
531 | ) |
532 | from lp.oci.model.ocirecipebuild import OCIRecipeBuildSet |
533 | +from lp.registry.interfaces.series import SeriesStatus |
534 | from lp.services.propertycache import clear_property_cache |
535 | from lp.testing import ( |
536 | admin_logged_in, |
537 | @@ -148,8 +150,16 @@ class TestOCIRecipeBuildSet(TestCaseWithFactory): |
538 | |
539 | def test_new(self): |
540 | requester = self.factory.makePerson() |
541 | - recipe = self.factory.makeOCIRecipe() |
542 | + distribution = self.factory.makeDistribution() |
543 | + distroseries = self.factory.makeDistroSeries( |
544 | + distribution=distribution, status=SeriesStatus.CURRENT) |
545 | + processor = getUtility(IProcessorSet).getByName("386") |
546 | + distro_arch_series = self.factory.makeDistroArchSeries( |
547 | + distroseries=distroseries, architecturetag="i386", |
548 | + processor=processor) |
549 | distro_arch_series = self.factory.makeDistroArchSeries() |
550 | + oci_project = self.factory.makeOCIProject(pillar=distribution) |
551 | + recipe = self.factory.makeOCIRecipe(oci_project=oci_project) |
552 | target = getUtility(IOCIRecipeBuildSet).new( |
553 | requester, recipe, distro_arch_series) |
554 | with admin_logged_in(): |
555 | diff --git a/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py b/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py |
556 | new file mode 100644 |
557 | index 0000000..e9ac224 |
558 | --- /dev/null |
559 | +++ b/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py |
560 | @@ -0,0 +1,604 @@ |
561 | +# Copyright 2015-2019 Canonical Ltd. This software is licensed under the |
562 | +# GNU Affero General Public License version 3 (see the file LICENSE). |
563 | + |
564 | +"""Tests for `OCIRecipeBuildBehaviour`.""" |
565 | + |
566 | +from __future__ import absolute_import, print_function, unicode_literals |
567 | + |
568 | +__metaclass__ = type |
569 | + |
570 | +import base64 |
571 | +from datetime import datetime |
572 | +import json |
573 | +import os |
574 | +import shutil |
575 | +import tempfile |
576 | +import time |
577 | +import uuid |
578 | + |
579 | +import fixtures |
580 | +from six.moves.urllib_parse import urlsplit |
581 | +from testtools import ExpectedException |
582 | +from testtools.matchers import ( |
583 | + AfterPreprocessing, |
584 | + ContainsDict, |
585 | + Equals, |
586 | + Is, |
587 | + IsInstance, |
588 | + MatchesDict, |
589 | + MatchesListwise, |
590 | + StartsWith, |
591 | + ) |
592 | +from testtools.twistedsupport import ( |
593 | + AsynchronousDeferredRunTestForBrokenTwisted, |
594 | + ) |
595 | +from twisted.internet import defer |
596 | +from twisted.trial.unittest import TestCase as TrialTestCase |
597 | +from zope.component import getUtility |
598 | +from zope.security.proxy import removeSecurityProxy |
599 | + |
600 | +from lp.buildmaster.enums import ( |
601 | + BuildBaseImageType, |
602 | + BuildStatus, |
603 | + ) |
604 | +from lp.buildmaster.interactor import BuilderInteractor |
605 | +from lp.buildmaster.interfaces.builder import ( |
606 | + BuildDaemonError, |
607 | + CannotBuild, |
608 | + ) |
609 | +from lp.buildmaster.interfaces.buildfarmjobbehaviour import ( |
610 | + IBuildFarmJobBehaviour, |
611 | + ) |
612 | +from lp.buildmaster.interfaces.processor import IProcessorSet |
613 | +from lp.buildmaster.tests.mock_slaves import ( |
614 | + MockBuilder, |
615 | + OkSlave, |
616 | + SlaveTestHelpers, |
617 | + WaitingSlave, |
618 | + ) |
619 | +from lp.buildmaster.tests.snapbuildproxy import ( |
620 | + InProcessProxyAuthAPIFixture, |
621 | + ProxyURLMatcher, |
622 | + RevocationEndpointMatcher, |
623 | + ) |
624 | +from lp.buildmaster.tests.test_buildfarmjobbehaviour import ( |
625 | + TestGetUploadMethodsMixin, |
626 | + ) |
627 | +from lp.oci.model.ocirecipebuildbehaviour import OCIRecipeBuildBehaviour |
628 | +from lp.registry.interfaces.series import SeriesStatus |
629 | +from lp.services.config import config |
630 | +from lp.services.log.logger import DevNullLogger |
631 | +from lp.services.webapp import canonical_url |
632 | +from lp.soyuz.adapters.archivedependencies import ( |
633 | + get_sources_list_for_building, |
634 | + ) |
635 | +from lp.testing import TestCaseWithFactory |
636 | +from lp.testing.dbuser import dbuser |
637 | +from lp.testing.factory import LaunchpadObjectFactory |
638 | +from lp.testing.fakemethod import FakeMethod |
639 | +from lp.testing.layers import LaunchpadZopelessLayer |
640 | +from lp.testing.mail_helpers import pop_notifications |
641 | + |
642 | + |
643 | +class MakeOCIBuildMixin: |
644 | + |
645 | + def makeBuild(self): |
646 | + build = self.factory.makeOCIRecipeBuild() |
647 | + self.factory.makeDistroSeries( |
648 | + distribution=build.recipe.oci_project.distribution, |
649 | + status=SeriesStatus.CURRENT) |
650 | + build.queueBuild() |
651 | + return build |
652 | + |
653 | + def makeUnmodifiableBuild(self): |
654 | + build = self.factory.makeOCIRecipeBuild() |
655 | + build.distro_arch_series = 'failed' |
656 | + build.queueBuild() |
657 | + return build |
658 | + |
659 | + def makeJob(self, git_ref, recipe=None): |
660 | + """Create a sample `IOCIRecipeBuildBehaviour`.""" |
661 | + if recipe is None: |
662 | + build = self.factory.makeOCIRecipeBuild() |
663 | + else: |
664 | + build = self.factory.makeOCIRecipeBuild(recipe=recipe) |
665 | + build.recipe.git_ref = git_ref |
666 | + |
667 | + job = IBuildFarmJobBehaviour(build) |
668 | + builder = MockBuilder() |
669 | + builder.processor = job.build.processor |
670 | + slave = self.useFixture(SlaveTestHelpers()).getClientSlave() |
671 | + job.setBuilder(builder, slave) |
672 | + self.addCleanup(slave.pool.closeCachedConnections) |
673 | + |
674 | + # Taken from test_archivedependencies.py |
675 | + for component_name in ("main", "universe"): |
676 | + self.factory.makeComponentSelection( |
677 | + build.distro_arch_series.distroseries, component_name) |
678 | + |
679 | + return job |
680 | + |
681 | + |
682 | +class TestOCIBuildBehaviour(TestCaseWithFactory): |
683 | + |
684 | + layer = LaunchpadZopelessLayer |
685 | + |
686 | + def test_provides_interface(self): |
687 | + # OCIRecipeBuildBehaviour provides IBuildFarmJobBehaviour. |
688 | + job = OCIRecipeBuildBehaviour(self.factory.makeOCIRecipeBuild()) |
689 | + self.assertProvides(job, IBuildFarmJobBehaviour) |
690 | + |
691 | + def test_adapts_IOCIRecipeBuild(self): |
692 | + # IBuildFarmJobBehaviour adapts an IOCIRecipeBuild. |
693 | + build = self.factory.makeOCIRecipeBuild() |
694 | + job = IBuildFarmJobBehaviour(build) |
695 | + self.assertProvides(job, IBuildFarmJobBehaviour) |
696 | + |
697 | + |
698 | +class TestAsyncOCIRecipeBuildBehaviour(MakeOCIBuildMixin, TestCaseWithFactory): |
699 | + |
700 | + run_tests_with = AsynchronousDeferredRunTestForBrokenTwisted.make_factory( |
701 | + timeout=10) |
702 | + layer = LaunchpadZopelessLayer |
703 | + |
704 | + @defer.inlineCallbacks |
705 | + def setUp(self): |
706 | + super(TestAsyncOCIRecipeBuildBehaviour, self).setUp() |
707 | + build_username = 'OCIBUILD-1' |
708 | + self.token = {'secret': uuid.uuid4().get_hex(), |
709 | + 'username': build_username, |
710 | + 'timestamp': datetime.utcnow().isoformat()} |
711 | + self.proxy_url = ("http://{username}:{password}" |
712 | + "@{host}:{port}".format( |
713 | + username=self.token['username'], |
714 | + password=self.token['secret'], |
715 | + host=config.snappy.builder_proxy_host, |
716 | + port=config.snappy.builder_proxy_port)) |
717 | + self.proxy_api = self.useFixture(InProcessProxyAuthAPIFixture()) |
718 | + yield self.proxy_api.start() |
719 | + self.now = time.time() |
720 | + self.useFixture(fixtures.MockPatch( |
721 | + "time.time", return_value=self.now)) |
722 | + |
723 | + @defer.inlineCallbacks |
724 | + def test_composeBuildRequest(self): |
725 | + [ref] = self.factory.makeGitRefs() |
726 | + job = self.makeJob(git_ref=ref) |
727 | + lfa = self.factory.makeLibraryFileAlias(db_only=True) |
728 | + job.build.distro_arch_series.addOrUpdateChroot(lfa) |
729 | + build_request = yield job.composeBuildRequest(None) |
730 | + self.assertThat(build_request, MatchesListwise([ |
731 | + Equals('oci'), |
732 | + Equals(job.build.distro_arch_series), |
733 | + Equals(job.build.pocket), |
734 | + Equals({}), |
735 | + IsInstance(dict), |
736 | + ])) |
737 | + |
738 | + @defer.inlineCallbacks |
739 | + def test_requestProxyToken_unconfigured(self): |
740 | + self.pushConfig("snappy", builder_proxy_auth_api_admin_secret=None) |
741 | + [ref] = self.factory.makeGitRefs() |
742 | + job = self.makeJob(git_ref=ref) |
743 | + expected_exception_msg = ( |
744 | + "builder_proxy_auth_api_admin_secret is not configured.") |
745 | + with ExpectedException(CannotBuild, expected_exception_msg): |
746 | + yield job.extraBuildArgs() |
747 | + |
748 | + @defer.inlineCallbacks |
749 | + def test_requestProxyToken(self): |
750 | + [ref] = self.factory.makeGitRefs() |
751 | + job = self.makeJob(git_ref=ref) |
752 | + yield job.extraBuildArgs() |
753 | + self.assertThat(self.proxy_api.tokens.requests, MatchesListwise([ |
754 | + MatchesDict({ |
755 | + "method": Equals("POST"), |
756 | + "uri": Equals(urlsplit( |
757 | + config.snappy.builder_proxy_auth_api_endpoint).path), |
758 | + "headers": ContainsDict({ |
759 | + b"Authorization": MatchesListwise([ |
760 | + Equals(b"Basic " + base64.b64encode( |
761 | + b"admin-launchpad.test:admin-secret"))]), |
762 | + b"Content-Type": MatchesListwise([ |
763 | + Equals(b"application/json; charset=UTF-8"), |
764 | + ]), |
765 | + }), |
766 | + "content": AfterPreprocessing(json.loads, MatchesDict({ |
767 | + "username": StartsWith(job.build.build_cookie + "-"), |
768 | + })), |
769 | + }), |
770 | + ])) |
771 | + |
772 | + @defer.inlineCallbacks |
773 | + def test_extraBuildArgs_git(self): |
774 | + # extraBuildArgs returns appropriate arguments if asked to build a |
775 | + # job for a Git branch. |
776 | + [ref] = self.factory.makeGitRefs() |
777 | + job = self.makeJob(git_ref=ref) |
778 | + expected_archives, expected_trusted_keys = ( |
779 | + yield get_sources_list_for_building( |
780 | + job.build, job.build.distro_arch_series, None)) |
781 | + for archive_line in expected_archives: |
782 | + self.assertIn('universe', archive_line) |
783 | + with dbuser(config.builddmaster.dbuser): |
784 | + args = yield job.extraBuildArgs() |
785 | + self.assertThat(args, MatchesDict({ |
786 | + "archive_private": Is(False), |
787 | + "archives": Equals(expected_archives), |
788 | + "arch_tag": Equals("i386"), |
789 | + "build_file": Equals(job.build.recipe.build_file), |
790 | + "build_url": Equals(canonical_url(job.build)), |
791 | + "fast_cleanup": Is(True), |
792 | + "git_repository": Equals(ref.repository.git_https_url), |
793 | + "git_path": Equals(ref.name), |
794 | + "name": Equals(job.build.recipe.name), |
795 | + "proxy_url": ProxyURLMatcher(job, self.now), |
796 | + "revocation_endpoint": RevocationEndpointMatcher(job, self.now), |
797 | + "series": Equals(job.build.distro_arch_series.distroseries.name), |
798 | + "trusted_keys": Equals(expected_trusted_keys), |
799 | + })) |
800 | + |
801 | + @defer.inlineCallbacks |
802 | + def test_extraBuildArgs_git_HEAD(self): |
803 | + # extraBuildArgs returns appropriate arguments if asked to build a |
804 | + # job for the default branch in a Launchpad-hosted Git repository. |
805 | + [ref] = self.factory.makeGitRefs() |
806 | + removeSecurityProxy(ref.repository)._default_branch = ref.path |
807 | + job = self.makeJob(git_ref=ref.repository.getRefByPath("HEAD")) |
808 | + expected_archives, expected_trusted_keys = ( |
809 | + yield get_sources_list_for_building( |
810 | + job.build, job.build.distro_arch_series, None)) |
811 | + for archive_line in expected_archives: |
812 | + self.assertIn('universe', archive_line) |
813 | + with dbuser(config.builddmaster.dbuser): |
814 | + args = yield job.extraBuildArgs() |
815 | + self.assertThat(args, MatchesDict({ |
816 | + "archive_private": Is(False), |
817 | + "archives": Equals(expected_archives), |
818 | + "arch_tag": Equals("i386"), |
819 | + "build_file": Equals(job.build.recipe.build_file), |
820 | + "build_url": Equals(canonical_url(job.build)), |
821 | + "fast_cleanup": Is(True), |
822 | + "git_repository": Equals(ref.repository.git_https_url), |
823 | + "name": Equals(job.build.recipe.name), |
824 | + "proxy_url": ProxyURLMatcher(job, self.now), |
825 | + "revocation_endpoint": RevocationEndpointMatcher(job, self.now), |
826 | + "series": Equals(job.build.distro_arch_series.distroseries.name), |
827 | + "trusted_keys": Equals(expected_trusted_keys), |
828 | + })) |
829 | + |
830 | + @defer.inlineCallbacks |
831 | + def test_composeBuildRequest_proxy_url_set(self): |
832 | + [ref] = self.factory.makeGitRefs() |
833 | + job = self.makeJob(git_ref=ref) |
834 | + build_request = yield job.composeBuildRequest(None) |
835 | + self.assertThat( |
836 | + build_request[4]["proxy_url"], ProxyURLMatcher(job, self.now)) |
837 | + |
838 | + @defer.inlineCallbacks |
839 | + def test_composeBuildRequest_git_ref_deleted(self): |
840 | + # If the source Git reference has been deleted, composeBuildRequest |
841 | + # raises CannotBuild. |
842 | + repository = self.factory.makeGitRepository() |
843 | + [ref] = self.factory.makeGitRefs(repository=repository) |
844 | + owner = self.factory.makePerson(name="oci-owner") |
845 | + |
846 | + distribution = self.factory.makeDistribution() |
847 | + distroseries = self.factory.makeDistroSeries( |
848 | + distribution=distribution, status=SeriesStatus.CURRENT) |
849 | + processor = getUtility(IProcessorSet).getByName("386") |
850 | + self.factory.makeDistroArchSeries( |
851 | + distroseries=distroseries, architecturetag="i386", |
852 | + processor=processor) |
853 | + |
854 | + oci_project = self.factory.makeOCIProject( |
855 | + pillar=distribution, registrant=owner) |
856 | + recipe = self.factory.makeOCIRecipe( |
857 | + oci_project=oci_project, registrant=owner, owner=owner, |
858 | + git_ref=ref) |
859 | + job = self.makeJob(ref, recipe=recipe) |
860 | + repository.removeRefs([ref.path]) |
861 | + self.assertIsNone(job.build.recipe.git_ref) |
862 | + expected_exception_msg = ("Source repository for " |
863 | + "~oci-owner/{} has been deleted.".format( |
864 | + recipe.name)) |
865 | + with ExpectedException(CannotBuild, expected_exception_msg): |
866 | + yield job.composeBuildRequest(None) |
867 | + |
868 | + @defer.inlineCallbacks |
869 | + def test_dispatchBuildToSlave_prefers_lxd(self): |
870 | + self.pushConfig("snappy", builder_proxy_host=None) |
871 | + [ref] = self.factory.makeGitRefs() |
872 | + job = self.makeJob(git_ref=ref) |
873 | + builder = MockBuilder() |
874 | + builder.processor = job.build.processor |
875 | + slave = OkSlave() |
876 | + job.setBuilder(builder, slave) |
877 | + chroot_lfa = self.factory.makeLibraryFileAlias(db_only=True) |
878 | + job.build.distro_arch_series.addOrUpdateChroot( |
879 | + chroot_lfa, image_type=BuildBaseImageType.CHROOT) |
880 | + lxd_lfa = self.factory.makeLibraryFileAlias(db_only=True) |
881 | + job.build.distro_arch_series.addOrUpdateChroot( |
882 | + lxd_lfa, image_type=BuildBaseImageType.LXD) |
883 | + yield job.dispatchBuildToSlave(DevNullLogger()) |
884 | + self.assertEqual( |
885 | + ('ensurepresent', lxd_lfa.http_url, '', ''), slave.call_log[0]) |
886 | + |
887 | + @defer.inlineCallbacks |
888 | + def test_dispatchBuildToSlave_falls_back_to_chroot(self): |
889 | + self.pushConfig("snappy", builder_proxy_host=None) |
890 | + [ref] = self.factory.makeGitRefs() |
891 | + job = self.makeJob(git_ref=ref) |
892 | + builder = MockBuilder() |
893 | + builder.processor = job.build.processor |
894 | + slave = OkSlave() |
895 | + job.setBuilder(builder, slave) |
896 | + chroot_lfa = self.factory.makeLibraryFileAlias(db_only=True) |
897 | + job.build.distro_arch_series.addOrUpdateChroot( |
898 | + chroot_lfa, image_type=BuildBaseImageType.CHROOT) |
899 | + yield job.dispatchBuildToSlave(DevNullLogger()) |
900 | + self.assertEqual( |
901 | + ('ensurepresent', chroot_lfa.http_url, '', ''), slave.call_log[0]) |
902 | + |
903 | + |
904 | +class TestHandleStatusForOCIRecipeBuild(MakeOCIBuildMixin, TrialTestCase, |
905 | + fixtures.TestWithFixtures): |
906 | + # This is mostly copied from TestHandleStatusMixin, however |
907 | + # we can't use all of those tests, due to the way OCIRecipeBuildBehaviour |
908 | + # parses the file contents, rather than just retrieving all that are |
909 | + # available. There's also some differences in the filemap handling, as |
910 | + # we need a much more complex filemap here. |
911 | + |
912 | + layer = LaunchpadZopelessLayer |
913 | + |
914 | + def pushConfig(self, section, **kwargs): |
915 | + """Push some key-value pairs into a section of the config. |
916 | + |
917 | + The config values will be restored during test tearDown. |
918 | + """ |
919 | + # Taken from lp/testing.py as we're using TrialTestCase, |
920 | + # not lp.testing.TestCase, as we need to handle the deferred |
921 | + # correctly. |
922 | + name = self.factory.getUniqueString() |
923 | + body = '\n'.join("%s: %s" % (k, v) for k, v in kwargs.iteritems()) |
924 | + config.push(name, "\n[%s]\n%s\n" % (section, body)) |
925 | + self.addCleanup(config.pop, name) |
926 | + |
927 | + def _createTestFile(self, name, content, hash): |
928 | + path = os.path.join(self.test_files_dir, name) |
929 | + with open(path, 'wb') as fp: |
930 | + fp.write(content) |
931 | + self.slave.valid_files[hash] = path |
932 | + |
933 | + def setUp(self): |
934 | + super(TestHandleStatusForOCIRecipeBuild, self).setUp() |
935 | + self.factory = LaunchpadObjectFactory() |
936 | + self.build = self.makeBuild() |
937 | + # For the moment, we require a builder for the build so that |
938 | + # handleStatus_OK can get a reference to the slave. |
939 | + self.builder = self.factory.makeBuilder() |
940 | + self.build.buildqueue_record.markAsBuilding(self.builder) |
941 | + self.slave = WaitingSlave('BuildStatus.OK') |
942 | + self.slave.valid_files['test_file_hash'] = '' |
943 | + self.interactor = BuilderInteractor() |
944 | + self.behaviour = self.interactor.getBuildBehaviour( |
945 | + self.build.buildqueue_record, self.builder, self.slave) |
946 | + |
947 | + # We overwrite the buildmaster root to use a temp directory. |
948 | + tempdir = tempfile.mkdtemp() |
949 | + self.addCleanup(shutil.rmtree, tempdir) |
950 | + self.upload_root = tempdir |
951 | + self.pushConfig('builddmaster', root=self.upload_root) |
952 | + |
953 | + # We stub out our build's getUploaderCommand() method so |
954 | + # we can check whether it was called as well as |
955 | + # verifySuccessfulUpload(). |
956 | + removeSecurityProxy(self.build).verifySuccessfulUpload = FakeMethod( |
957 | + result=True) |
958 | + |
959 | + digests = [{ |
960 | + "diff_id_1": { |
961 | + "digest": "digest_1", |
962 | + "source": "test/base_1", |
963 | + "layer_id": "layer_1" |
964 | + }, |
965 | + "diff_id_2": { |
966 | + "digest": "digest_2", |
967 | + "source": "", |
968 | + "layer_id": "layer_2" |
969 | + } |
970 | + }] |
971 | + |
972 | + self.test_files_dir = tempfile.mkdtemp() |
973 | + self._createTestFile('buildlog', '', 'buildlog') |
974 | + self._createTestFile( |
975 | + 'manifest.json', |
976 | + '[{"Config": "config_file_1.json", ' |
977 | + '"Layers": ["layer_1/layer.tar", "layer_2/layer.tar"]}]', |
978 | + 'manifest_hash') |
979 | + self._createTestFile( |
980 | + 'digests.json', |
981 | + json.dumps(digests), |
982 | + 'digests_hash') |
983 | + self._createTestFile( |
984 | + 'config_file_1.json', |
985 | + '{"rootfs": {"diff_ids": ["diff_id_1", "diff_id_2"]}}', |
986 | + 'config_1_hash') |
987 | + self._createTestFile( |
988 | + 'layer_2.tar.gz', |
989 | + '', |
990 | + 'layer_2_hash' |
991 | + ) |
992 | + |
993 | + self.filemap = { |
994 | + 'manifest.json': 'manifest_hash', |
995 | + 'digests.json': 'digests_hash', |
996 | + 'config_file_1.json': 'config_1_hash', |
997 | + 'layer_1.tar.gz': 'layer_1_hash', |
998 | + 'layer_2.tar.gz': 'layer_2_hash' |
999 | + } |
1000 | + self.factory.makeOCIFile( |
1001 | + build=self.build, layer_file_digest=u'digest_1', |
1002 | + content="retrieved from librarian") |
1003 | + |
1004 | + def assertResultCount(self, count, result): |
1005 | + self.assertEqual( |
1006 | + 1, len(os.listdir(os.path.join(self.upload_root, result)))) |
1007 | + |
1008 | + @defer.inlineCallbacks |
1009 | + def test_handleStatus_OK_normal_image(self): |
1010 | + with dbuser(config.builddmaster.dbuser): |
1011 | + yield self.behaviour.handleStatus( |
1012 | + self.build.buildqueue_record, 'OK', |
1013 | + {'filemap': self.filemap}) |
1014 | + self.assertEqual( |
1015 | + ['buildlog', 'manifest_hash', 'digests_hash', 'config_1_hash', |
1016 | + 'layer_2_hash'], |
1017 | + self.slave._got_file_record) |
1018 | + # This hash should not appear as it is already in the librarian |
1019 | + self.assertNotIn('layer_1_hash', self.slave._got_file_record) |
1020 | + self.assertEqual(BuildStatus.UPLOADING, self.build.status) |
1021 | + self.assertResultCount(1, "incoming") |
1022 | + |
1023 | + # layer_1 should have been retrieved from the librarian |
1024 | + layer_1_path = os.path.join( |
1025 | + self.upload_root, |
1026 | + "incoming", |
1027 | + self.behaviour.getUploadDirLeaf(self.build.build_cookie), |
1028 | + str(self.build.archive.id), |
1029 | + self.build.distribution.name, |
1030 | + "layer_1.tar.gz" |
1031 | + ) |
1032 | + with open(layer_1_path, 'rb') as layer_1_fp: |
1033 | + contents = layer_1_fp.read() |
1034 | + self.assertEqual(contents, b'retrieved from librarian') |
1035 | + |
1036 | + @defer.inlineCallbacks |
1037 | + def test_handleStatus_OK_absolute_filepath(self): |
1038 | + |
1039 | + self._createTestFile( |
1040 | + 'manifest.json', |
1041 | + '[{"Config": "/notvalid/config_file_1.json", ' |
1042 | + '"Layers": ["layer_1/layer.tar", "layer_2/layer.tar"]}]', |
1043 | + 'manifest_hash') |
1044 | + |
1045 | + self.filemap['/notvalid/config_file_1.json'] = 'config_1_hash' |
1046 | + |
1047 | + # A filemap that tries to write to files outside of the upload |
1048 | + # directory will not be collected. |
1049 | + with ExpectedException( |
1050 | + BuildDaemonError, |
1051 | + "Build returned a file named " |
1052 | + "'/notvalid/config_file_1.json'."): |
1053 | + with dbuser(config.builddmaster.dbuser): |
1054 | + yield self.behaviour.handleStatus( |
1055 | + self.build.buildqueue_record, 'OK', |
1056 | + {'filemap': self.filemap}) |
1057 | + |
1058 | + @defer.inlineCallbacks |
1059 | + def test_handleStatus_OK_relative_filepath(self): |
1060 | + |
1061 | + self._createTestFile( |
1062 | + 'manifest.json', |
1063 | + '[{"Config": "../config_file_1.json", ' |
1064 | + '"Layers": ["layer_1/layer.tar", "layer_2/layer.tar"]}]', |
1065 | + 'manifest_hash') |
1066 | + |
1067 | + self.filemap['../config_file_1.json'] = 'config_1_hash' |
1068 | + # A filemap that tries to write to files outside of |
1069 | + # the upload directory will not be collected. |
1070 | + with ExpectedException( |
1071 | + BuildDaemonError, |
1072 | + "Build returned a file named '../config_file_1.json'."): |
1073 | + with dbuser(config.builddmaster.dbuser): |
1074 | + yield self.behaviour.handleStatus( |
1075 | + self.build.buildqueue_record, 'OK', |
1076 | + {'filemap': self.filemap}) |
1077 | + |
1078 | + @defer.inlineCallbacks |
1079 | + def test_handleStatus_OK_sets_build_log(self): |
1080 | + # The build log is set during handleStatus. |
1081 | + self.assertEqual(None, self.build.log) |
1082 | + with dbuser(config.builddmaster.dbuser): |
1083 | + yield self.behaviour.handleStatus( |
1084 | + self.build.buildqueue_record, 'OK', |
1085 | + {'filemap': self.filemap}) |
1086 | + self.assertNotEqual(None, self.build.log) |
1087 | + |
1088 | + @defer.inlineCallbacks |
1089 | + def test_handleStatus_ABORTED_cancels_cancelling(self): |
1090 | + with dbuser(config.builddmaster.dbuser): |
1091 | + self.build.updateStatus(BuildStatus.CANCELLING) |
1092 | + yield self.behaviour.handleStatus( |
1093 | + self.build.buildqueue_record, "ABORTED", {}) |
1094 | + self.assertEqual(0, len(pop_notifications()), "Notifications received") |
1095 | + self.assertEqual(BuildStatus.CANCELLED, self.build.status) |
1096 | + |
1097 | + @defer.inlineCallbacks |
1098 | + def test_handleStatus_ABORTED_illegal_when_building(self): |
1099 | + self.builder.vm_host = "fake_vm_host" |
1100 | + self.behaviour = self.interactor.getBuildBehaviour( |
1101 | + self.build.buildqueue_record, self.builder, self.slave) |
1102 | + with dbuser(config.builddmaster.dbuser): |
1103 | + self.build.updateStatus(BuildStatus.BUILDING) |
1104 | + with ExpectedException( |
1105 | + BuildDaemonError, |
1106 | + "Build returned unexpected status: u'ABORTED'"): |
1107 | + yield self.behaviour.handleStatus( |
1108 | + self.build.buildqueue_record, "ABORTED", {}) |
1109 | + |
1110 | + @defer.inlineCallbacks |
1111 | + def test_handleStatus_ABORTED_cancelling_sets_build_log(self): |
1112 | + # If a build is intentionally cancelled, the build log is set. |
1113 | + self.assertEqual(None, self.build.log) |
1114 | + with dbuser(config.builddmaster.dbuser): |
1115 | + self.build.updateStatus(BuildStatus.CANCELLING) |
1116 | + yield self.behaviour.handleStatus( |
1117 | + self.build.buildqueue_record, "ABORTED", {}) |
1118 | + self.assertNotEqual(None, self.build.log) |
1119 | + |
1120 | + @defer.inlineCallbacks |
1121 | + def test_date_finished_set(self): |
1122 | + # The date finished is updated during handleStatus_OK. |
1123 | + self.assertEqual(None, self.build.date_finished) |
1124 | + with dbuser(config.builddmaster.dbuser): |
1125 | + yield self.behaviour.handleStatus( |
1126 | + self.build.buildqueue_record, 'OK', |
1127 | + {'filemap': self.filemap}) |
1128 | + self.assertNotEqual(None, self.build.date_finished) |
1129 | + |
1130 | + @defer.inlineCallbacks |
1131 | + def test_givenback_collection(self): |
1132 | + with ExpectedException( |
1133 | + BuildDaemonError, |
1134 | + "Build returned unexpected status: u'GIVENBACK'"): |
1135 | + with dbuser(config.builddmaster.dbuser): |
1136 | + yield self.behaviour.handleStatus( |
1137 | + self.build.buildqueue_record, "GIVENBACK", {}) |
1138 | + |
1139 | + @defer.inlineCallbacks |
1140 | + def test_builderfail_collection(self): |
1141 | + with ExpectedException( |
1142 | + BuildDaemonError, |
1143 | + "Build returned unexpected status: u'BUILDERFAIL'"): |
1144 | + with dbuser(config.builddmaster.dbuser): |
1145 | + yield self.behaviour.handleStatus( |
1146 | + self.build.buildqueue_record, "BUILDERFAIL", {}) |
1147 | + |
1148 | + @defer.inlineCallbacks |
1149 | + def test_invalid_status_collection(self): |
1150 | + with ExpectedException( |
1151 | + BuildDaemonError, |
1152 | + "Build returned unexpected status: u'BORKED'"): |
1153 | + with dbuser(config.builddmaster.dbuser): |
1154 | + yield self.behaviour.handleStatus( |
1155 | + self.build.buildqueue_record, "BORKED", {}) |
1156 | + |
1157 | + |
1158 | +class TestGetUploadMethodsForOCIRecipeBuild( |
1159 | + MakeOCIBuildMixin, TestGetUploadMethodsMixin, TrialTestCase): |
1160 | + """IPackageBuild.getUpload-related methods work with OCI recipe builds.""" |
1161 | + |
1162 | + def setUp(self): |
1163 | + super(TestGetUploadMethodsForOCIRecipeBuild, self).__init__(self) |
1164 | + self.factory = LaunchpadObjectFactory() |
1165 | diff --git a/lib/lp/snappy/model/snapbuildbehaviour.py b/lib/lp/snappy/model/snapbuildbehaviour.py |
1166 | index cda3cff..cbcba1d 100644 |
1167 | --- a/lib/lp/snappy/model/snapbuildbehaviour.py |
1168 | +++ b/lib/lp/snappy/model/snapbuildbehaviour.py |
1169 | @@ -11,6 +11,7 @@ from __future__ import absolute_import, print_function, unicode_literals |
1170 | __metaclass__ = type |
1171 | __all__ = [ |
1172 | 'SnapBuildBehaviour', |
1173 | + 'SnapProxyMixin', |
1174 | ] |
1175 | |
1176 | import base64 |
1177 | @@ -61,9 +62,58 @@ def format_as_rfc3339(timestamp): |
1178 | return timestamp.replace(microsecond=0, tzinfo=None).isoformat() + 'Z' |
1179 | |
1180 | |
1181 | +class SnapProxyMixin: |
1182 | + """Methods for handling builds with the Snap Build Proxy enabled.""" |
1183 | + |
1184 | + @defer.inlineCallbacks |
1185 | + def addProxyArgs(self, args, allow_internet=True): |
1186 | + if config.snappy.builder_proxy_host and allow_internet: |
1187 | + token = yield self._requestProxyToken() |
1188 | + args["proxy_url"] = ( |
1189 | + "http://{username}:{password}@{host}:{port}".format( |
1190 | + username=token['username'], |
1191 | + password=token['secret'], |
1192 | + host=config.snappy.builder_proxy_host, |
1193 | + port=config.snappy.builder_proxy_port)) |
1194 | + args["revocation_endpoint"] = ( |
1195 | + "{endpoint}/{token}".format( |
1196 | + endpoint=config.snappy.builder_proxy_auth_api_endpoint, |
1197 | + token=token['username'])) |
1198 | + |
1199 | + @defer.inlineCallbacks |
1200 | + def _requestProxyToken(self): |
1201 | + admin_username = config.snappy.builder_proxy_auth_api_admin_username |
1202 | + if not admin_username: |
1203 | + raise CannotBuild( |
1204 | + "builder_proxy_auth_api_admin_username is not configured.") |
1205 | + secret = config.snappy.builder_proxy_auth_api_admin_secret |
1206 | + if not secret: |
1207 | + raise CannotBuild( |
1208 | + "builder_proxy_auth_api_admin_secret is not configured.") |
1209 | + url = config.snappy.builder_proxy_auth_api_endpoint |
1210 | + if not secret: |
1211 | + raise CannotBuild( |
1212 | + "builder_proxy_auth_api_endpoint is not configured.") |
1213 | + timestamp = int(time.time()) |
1214 | + proxy_username = '{build_id}-{timestamp}'.format( |
1215 | + build_id=self.build.build_cookie, |
1216 | + timestamp=timestamp) |
1217 | + auth_string = '{}:{}'.format(admin_username, secret).strip() |
1218 | + auth_header = b'Basic ' + base64.b64encode(auth_string) |
1219 | + |
1220 | + response = yield treq.post( |
1221 | + url, headers={'Authorization': auth_header}, |
1222 | + json={'username': proxy_username}, |
1223 | + reactor=self._slave.reactor, |
1224 | + pool=self._slave.pool) |
1225 | + response = yield check_status(response) |
1226 | + token = yield treq.json_content(response) |
1227 | + defer.returnValue(token) |
1228 | + |
1229 | + |
1230 | @adapter(ISnapBuild) |
1231 | @implementer(IBuildFarmJobBehaviour) |
1232 | -class SnapBuildBehaviour(BuildFarmJobBehaviourBase): |
1233 | +class SnapBuildBehaviour(SnapProxyMixin, BuildFarmJobBehaviourBase): |
1234 | """Dispatches `SnapBuild` jobs to slaves.""" |
1235 | |
1236 | builder_type = "snap" |
1237 | @@ -111,18 +161,7 @@ class SnapBuildBehaviour(BuildFarmJobBehaviourBase): |
1238 | build = self.build |
1239 | args = yield super(SnapBuildBehaviour, self).extraBuildArgs( |
1240 | logger=logger) |
1241 | - if config.snappy.builder_proxy_host and build.snap.allow_internet: |
1242 | - token = yield self._requestProxyToken() |
1243 | - args["proxy_url"] = ( |
1244 | - "http://{username}:{password}@{host}:{port}".format( |
1245 | - username=token['username'], |
1246 | - password=token['secret'], |
1247 | - host=config.snappy.builder_proxy_host, |
1248 | - port=config.snappy.builder_proxy_port)) |
1249 | - args["revocation_endpoint"] = ( |
1250 | - "{endpoint}/{token}".format( |
1251 | - endpoint=config.snappy.builder_proxy_auth_api_endpoint, |
1252 | - token=token['username'])) |
1253 | + yield self.addProxyArgs(args, build.snap.allow_internet) |
1254 | args["name"] = build.snap.store_name or build.snap.name |
1255 | channels = build.channels or {} |
1256 | if "snapcraft" not in channels: |
1257 | @@ -188,36 +227,6 @@ class SnapBuildBehaviour(BuildFarmJobBehaviourBase): |
1258 | args["build_request_timestamp"] = timestamp |
1259 | defer.returnValue(args) |
1260 | |
1261 | - @defer.inlineCallbacks |
1262 | - def _requestProxyToken(self): |
1263 | - admin_username = config.snappy.builder_proxy_auth_api_admin_username |
1264 | - if not admin_username: |
1265 | - raise CannotBuild( |
1266 | - "builder_proxy_auth_api_admin_username is not configured.") |
1267 | - secret = config.snappy.builder_proxy_auth_api_admin_secret |
1268 | - if not secret: |
1269 | - raise CannotBuild( |
1270 | - "builder_proxy_auth_api_admin_secret is not configured.") |
1271 | - url = config.snappy.builder_proxy_auth_api_endpoint |
1272 | - if not secret: |
1273 | - raise CannotBuild( |
1274 | - "builder_proxy_auth_api_endpoint is not configured.") |
1275 | - timestamp = int(time.time()) |
1276 | - proxy_username = '{build_id}-{timestamp}'.format( |
1277 | - build_id=self.build.build_cookie, |
1278 | - timestamp=timestamp) |
1279 | - auth_string = '{}:{}'.format(admin_username, secret).strip() |
1280 | - auth_header = b'Basic ' + base64.b64encode(auth_string) |
1281 | - |
1282 | - response = yield treq.post( |
1283 | - url, headers={'Authorization': auth_header}, |
1284 | - json={'username': proxy_username}, |
1285 | - reactor=self._slave.reactor, |
1286 | - pool=self._slave.pool) |
1287 | - response = yield check_status(response) |
1288 | - token = yield treq.json_content(response) |
1289 | - defer.returnValue(token) |
1290 | - |
1291 | def verifySuccessfulBuild(self): |
1292 | """See `IBuildFarmJobBehaviour`.""" |
1293 | # The implementation in BuildFarmJobBehaviourBase checks whether the |
1294 | diff --git a/lib/lp/snappy/tests/test_snapbuildbehaviour.py b/lib/lp/snappy/tests/test_snapbuildbehaviour.py |
1295 | index 2755523..f851008 100644 |
1296 | --- a/lib/lp/snappy/tests/test_snapbuildbehaviour.py |
1297 | +++ b/lib/lp/snappy/tests/test_snapbuildbehaviour.py |
1298 | @@ -24,7 +24,6 @@ from testtools.matchers import ( |
1299 | AfterPreprocessing, |
1300 | ContainsDict, |
1301 | Equals, |
1302 | - HasLength, |
1303 | Is, |
1304 | IsInstance, |
1305 | MatchesDict, |
1306 | @@ -38,13 +37,10 @@ from testtools.twistedsupport import ( |
1307 | import transaction |
1308 | from twisted.internet import ( |
1309 | defer, |
1310 | - endpoints, |
1311 | reactor, |
1312 | ) |
1313 | -from twisted.python.compat import nativeString |
1314 | from twisted.trial.unittest import TestCase as TrialTestCase |
1315 | from twisted.web import ( |
1316 | - resource, |
1317 | server, |
1318 | xmlrpc, |
1319 | ) |
1320 | @@ -71,6 +67,11 @@ from lp.buildmaster.tests.mock_slaves import ( |
1321 | OkSlave, |
1322 | SlaveTestHelpers, |
1323 | ) |
1324 | +from lp.buildmaster.tests.snapbuildproxy import ( |
1325 | + InProcessProxyAuthAPIFixture, |
1326 | + ProxyURLMatcher, |
1327 | + RevocationEndpointMatcher, |
1328 | + ) |
1329 | from lp.buildmaster.tests.test_buildfarmjobbehaviour import ( |
1330 | TestGetUploadMethodsMixin, |
1331 | TestHandleStatusMixin, |
1332 | @@ -100,7 +101,6 @@ from lp.soyuz.adapters.archivedependencies import ( |
1333 | ) |
1334 | from lp.soyuz.enums import PackagePublishingStatus |
1335 | from lp.soyuz.interfaces.archive import ArchiveDisabled |
1336 | -from lp.soyuz.interfaces.component import IComponentSet |
1337 | from lp.soyuz.tests.soyuz import Base64KeyMatches |
1338 | from lp.testing import ( |
1339 | TestCase, |
1340 | @@ -116,69 +116,6 @@ from lp.testing.layers import LaunchpadZopelessLayer |
1341 | from lp.xmlrpc.interfaces import IPrivateApplication |
1342 | |
1343 | |
1344 | -class ProxyAuthAPITokensResource(resource.Resource): |
1345 | - """A test tokens resource for the proxy authentication API.""" |
1346 | - |
1347 | - isLeaf = True |
1348 | - |
1349 | - def __init__(self): |
1350 | - resource.Resource.__init__(self) |
1351 | - self.requests = [] |
1352 | - |
1353 | - def render_POST(self, request): |
1354 | - content = request.content.read() |
1355 | - self.requests.append({ |
1356 | - "method": request.method, |
1357 | - "uri": request.uri, |
1358 | - "headers": dict(request.requestHeaders.getAllRawHeaders()), |
1359 | - "content": content, |
1360 | - }) |
1361 | - username = json.loads(content)["username"] |
1362 | - return json.dumps({ |
1363 | - "username": username, |
1364 | - "secret": uuid.uuid4().hex, |
1365 | - "timestamp": datetime.utcnow().isoformat(), |
1366 | - }) |
1367 | - |
1368 | - |
1369 | -class InProcessProxyAuthAPIFixture(fixtures.Fixture): |
1370 | - """A fixture that pretends to be the proxy authentication API. |
1371 | - |
1372 | - Users of this fixture must call the `start` method, which returns a |
1373 | - `Deferred`, and arrange for that to get back to the reactor. This is |
1374 | - necessary because the basic fixture API does not allow `setUp` to return |
1375 | - anything. For example: |
1376 | - |
1377 | - class TestSomething(TestCase): |
1378 | - |
1379 | - run_tests_with = AsynchronousDeferredRunTest.make_factory( |
1380 | - timeout=10) |
1381 | - |
1382 | - @defer.inlineCallbacks |
1383 | - def setUp(self): |
1384 | - super(TestSomething, self).setUp() |
1385 | - yield self.useFixture(InProcessProxyAuthAPIFixture()).start() |
1386 | - """ |
1387 | - |
1388 | - @defer.inlineCallbacks |
1389 | - def start(self): |
1390 | - root = resource.Resource() |
1391 | - self.tokens = ProxyAuthAPITokensResource() |
1392 | - root.putChild("tokens", self.tokens) |
1393 | - endpoint = endpoints.serverFromString(reactor, nativeString("tcp:0")) |
1394 | - site = server.Site(self.tokens) |
1395 | - self.addCleanup(site.stopFactory) |
1396 | - port = yield endpoint.listen(site) |
1397 | - self.addCleanup(port.stopListening) |
1398 | - config.push("in-process-proxy-auth-api-fixture", dedent(""" |
1399 | - [snappy] |
1400 | - builder_proxy_auth_api_admin_secret: admin-secret |
1401 | - builder_proxy_auth_api_endpoint: http://%s:%s/tokens |
1402 | - """) % |
1403 | - (port.getHost().host, port.getHost().port)) |
1404 | - self.addCleanup(config.pop, "in-process-proxy-auth-api-fixture") |
1405 | - |
1406 | - |
1407 | class InProcessAuthServer(xmlrpc.XMLRPC): |
1408 | |
1409 | def __init__(self, *args, **kwargs): |
1410 | @@ -380,21 +317,6 @@ class TestAsyncSnapBuildBehaviour(TestSnapBuildBehaviourBase): |
1411 | self.addCleanup(slave.pool.closeCachedConnections) |
1412 | return job |
1413 | |
1414 | - def getProxyURLMatcher(self, job): |
1415 | - return AfterPreprocessing(urlsplit, MatchesStructure( |
1416 | - scheme=Equals("http"), |
1417 | - username=Equals("{}-{}".format( |
1418 | - job.build.build_cookie, int(self.now))), |
1419 | - password=HasLength(32), |
1420 | - hostname=Equals(config.snappy.builder_proxy_host), |
1421 | - port=Equals(config.snappy.builder_proxy_port), |
1422 | - path=Equals(""))) |
1423 | - |
1424 | - def getRevocationEndpointMatcher(self, job): |
1425 | - return Equals("{}/{}-{}".format( |
1426 | - config.snappy.builder_proxy_auth_api_endpoint, |
1427 | - job.build.build_cookie, int(self.now))) |
1428 | - |
1429 | @defer.inlineCallbacks |
1430 | def test_composeBuildRequest(self): |
1431 | job = self.makeJob() |
1432 | @@ -466,8 +388,8 @@ class TestAsyncSnapBuildBehaviour(TestSnapBuildBehaviourBase): |
1433 | "fast_cleanup": Is(True), |
1434 | "name": Equals("test-snap"), |
1435 | "private": Is(False), |
1436 | - "proxy_url": self.getProxyURLMatcher(job), |
1437 | - "revocation_endpoint": self.getRevocationEndpointMatcher(job), |
1438 | + "proxy_url": ProxyURLMatcher(job, self.now), |
1439 | + "revocation_endpoint": RevocationEndpointMatcher(job, self.now), |
1440 | "series": Equals("unstable"), |
1441 | "trusted_keys": Equals(expected_trusted_keys), |
1442 | })) |
1443 | @@ -507,8 +429,8 @@ class TestAsyncSnapBuildBehaviour(TestSnapBuildBehaviourBase): |
1444 | "git_path": Equals(ref.name), |
1445 | "name": Equals("test-snap"), |
1446 | "private": Is(False), |
1447 | - "proxy_url": self.getProxyURLMatcher(job), |
1448 | - "revocation_endpoint": self.getRevocationEndpointMatcher(job), |
1449 | + "proxy_url": ProxyURLMatcher(job, self.now), |
1450 | + "revocation_endpoint": RevocationEndpointMatcher(job, self.now), |
1451 | "series": Equals("unstable"), |
1452 | "trusted_keys": Equals(expected_trusted_keys), |
1453 | })) |
1454 | @@ -537,8 +459,8 @@ class TestAsyncSnapBuildBehaviour(TestSnapBuildBehaviourBase): |
1455 | "git_repository": Equals(ref.repository.git_https_url), |
1456 | "name": Equals("test-snap"), |
1457 | "private": Is(False), |
1458 | - "proxy_url": self.getProxyURLMatcher(job), |
1459 | - "revocation_endpoint": self.getRevocationEndpointMatcher(job), |
1460 | + "proxy_url": ProxyURLMatcher(job, self.now), |
1461 | + "revocation_endpoint": RevocationEndpointMatcher(job, self.now), |
1462 | "series": Equals("unstable"), |
1463 | "trusted_keys": Equals(expected_trusted_keys), |
1464 | })) |
1465 | @@ -584,8 +506,8 @@ class TestAsyncSnapBuildBehaviour(TestSnapBuildBehaviourBase): |
1466 | "git_path": Equals(ref.name), |
1467 | "name": Equals("test-snap"), |
1468 | "private": Is(True), |
1469 | - "proxy_url": self.getProxyURLMatcher(job), |
1470 | - "revocation_endpoint": self.getRevocationEndpointMatcher(job), |
1471 | + "proxy_url": ProxyURLMatcher(job, self.now), |
1472 | + "revocation_endpoint": RevocationEndpointMatcher(job, self.now), |
1473 | "series": Equals("unstable"), |
1474 | "trusted_keys": Equals(expected_trusted_keys), |
1475 | })) |
1476 | @@ -616,8 +538,8 @@ class TestAsyncSnapBuildBehaviour(TestSnapBuildBehaviourBase): |
1477 | "git_path": Equals("master"), |
1478 | "name": Equals("test-snap"), |
1479 | "private": Is(False), |
1480 | - "proxy_url": self.getProxyURLMatcher(job), |
1481 | - "revocation_endpoint": self.getRevocationEndpointMatcher(job), |
1482 | + "proxy_url": ProxyURLMatcher(job, self.now), |
1483 | + "revocation_endpoint": RevocationEndpointMatcher(job, self.now), |
1484 | "series": Equals("unstable"), |
1485 | "trusted_keys": Equals(expected_trusted_keys), |
1486 | })) |
1487 | @@ -646,8 +568,8 @@ class TestAsyncSnapBuildBehaviour(TestSnapBuildBehaviourBase): |
1488 | "git_repository": Equals(url), |
1489 | "name": Equals("test-snap"), |
1490 | "private": Is(False), |
1491 | - "proxy_url": self.getProxyURLMatcher(job), |
1492 | - "revocation_endpoint": self.getRevocationEndpointMatcher(job), |
1493 | + "proxy_url": ProxyURLMatcher(job, self.now), |
1494 | + "revocation_endpoint": RevocationEndpointMatcher(job, self.now), |
1495 | "series": Equals("unstable"), |
1496 | "trusted_keys": Equals(expected_trusted_keys), |
1497 | })) |
1498 | @@ -808,7 +730,7 @@ class TestAsyncSnapBuildBehaviour(TestSnapBuildBehaviourBase): |
1499 | job = self.makeJob() |
1500 | build_request = yield job.composeBuildRequest(None) |
1501 | self.assertThat( |
1502 | - build_request[4]["proxy_url"], self.getProxyURLMatcher(job)) |
1503 | + build_request[4]["proxy_url"], ProxyURLMatcher(job, self.now)) |
1504 | |
1505 | @defer.inlineCallbacks |
1506 | def test_composeBuildRequest_deleted(self): |
1507 | diff --git a/lib/lp/testing/factory.py b/lib/lp/testing/factory.py |
1508 | index 85a05b6..c402c4c 100644 |
1509 | --- a/lib/lp/testing/factory.py |
1510 | +++ b/lib/lp/testing/factory.py |
1511 | @@ -4991,9 +4991,15 @@ class BareLaunchpadObjectFactory(ObjectFactory): |
1512 | if requester is None: |
1513 | requester = self.makePerson() |
1514 | if distro_arch_series is None: |
1515 | - distro_arch_series = self.makeDistroArchSeries() |
1516 | + distroseries = self.makeDistroSeries(status=SeriesStatus.CURRENT) |
1517 | + processor = getUtility(IProcessorSet).getByName("386") |
1518 | + distro_arch_series = self.makeDistroArchSeries( |
1519 | + distroseries=distroseries, architecturetag="i386", |
1520 | + processor=processor) |
1521 | if recipe is None: |
1522 | - recipe = self.makeOCIRecipe() |
1523 | + oci_project = self.makeOCIProject( |
1524 | + pillar=distro_arch_series.distroseries.distribution) |
1525 | + recipe = self.makeOCIRecipe(oci_project=oci_project) |
1526 | oci_build = getUtility(IOCIRecipeBuildSet).new( |
1527 | requester, recipe, distro_arch_series, date_created) |
1528 | if duration is not None: |
1529 | @@ -5010,12 +5016,13 @@ class BareLaunchpadObjectFactory(ObjectFactory): |
1530 | return oci_build |
1531 | |
1532 | def makeOCIFile(self, build=None, library_file=None, |
1533 | - layer_file_digest=None): |
1534 | + layer_file_digest=None, content=None, filename=None): |
1535 | """Make a new OCIFile.""" |
1536 | if build is None: |
1537 | build = self.makeOCIRecipeBuild() |
1538 | if library_file is None: |
1539 | - library_file = self.makeLibraryFileAlias() |
1540 | + library_file = self.makeLibraryFileAlias( |
1541 | + content=content, filename=filename) |
1542 | return OCIFile(build=build, library_file=library_file, |
1543 | layer_file_digest=layer_file_digest) |
1544 | |
1545 | diff --git a/lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py b/lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py |
1546 | index b387251..32dd553 100644 |
1547 | --- a/lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py |
1548 | +++ b/lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py |
1549 | @@ -138,7 +138,7 @@ class TestTranslationTemplatesBuildBehaviour( |
1550 | buildqueue = FakeBuildQueue(behaviour) |
1551 | path = behaviour.templates_tarball_path |
1552 | # Poke the file we're expecting into the mock slave. |
1553 | - behaviour._slave.valid_file_hashes.append(path) |
1554 | + behaviour._slave.valid_files[path] = '' |
1555 | |
1556 | def got_tarball(filename): |
1557 | tarball = open(filename, 'r') |