Merge ~twom/launchpad:oci-buildbehaviour into launchpad:master

Proposed by Tom Wardill
Status: Merged
Approved by: Tom Wardill
Approved revision: 47145664aa7899c4393aff34ccbba9536ac7775b
Merge reported by: Otto Co-Pilot
Merged at revision: not available
Proposed branch: ~twom/launchpad:oci-buildbehaviour
Merge into: launchpad:master
Diff against target: 1557 lines (+1067/-166)
15 files modified
database/schema/security.cfg (+1/-0)
lib/lp/buildmaster/model/buildfarmjobbehaviour.py (+17/-13)
lib/lp/buildmaster/tests/mock_slaves.py (+9/-3)
lib/lp/buildmaster/tests/snapbuildproxy.py (+123/-0)
lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py (+1/-1)
lib/lp/oci/configure.zcml (+6/-0)
lib/lp/oci/interfaces/ocirecipebuild.py (+6/-0)
lib/lp/oci/model/ocirecipebuild.py (+25/-4)
lib/lp/oci/model/ocirecipebuildbehaviour.py (+182/-0)
lib/lp/oci/tests/test_ocirecipebuild.py (+11/-1)
lib/lp/oci/tests/test_ocirecipebuildbehaviour.py (+604/-0)
lib/lp/snappy/model/snapbuildbehaviour.py (+52/-43)
lib/lp/snappy/tests/test_snapbuildbehaviour.py (+18/-96)
lib/lp/testing/factory.py (+11/-4)
lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py (+1/-1)
Reviewer Review Type Date Requested Status
Colin Watson (community) Approve
Review via email: mp+379201@code.launchpad.net

This proposal supersedes a proposal from 2019-12-12.

Commit message

Add OCIRecipeBuildBehaviour

Description of the change

Add a buildbehaviour to drive OCI image builds.
Use mixin tests where sensible, override for places where the OCI methods differ.

To post a comment you must log in.
Revision history for this message
Colin Watson (cjwatson) : Posted in a previous version of this proposal
review: Needs Fixing
Revision history for this message
Tom Wardill (twom) : Posted in a previous version of this proposal
Revision history for this message
Colin Watson (cjwatson) : Posted in a previous version of this proposal
Revision history for this message
Colin Watson (cjwatson) :
review: Needs Fixing
Revision history for this message
Colin Watson (cjwatson) :
Revision history for this message
Colin Watson (cjwatson) :
review: Needs Fixing
Revision history for this message
Colin Watson (cjwatson) :
review: Approve
Revision history for this message
Colin Watson (cjwatson) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/database/schema/security.cfg b/database/schema/security.cfg
2index 306d206..d0832c1 100644
3--- a/database/schema/security.cfg
4+++ b/database/schema/security.cfg
5@@ -995,6 +995,7 @@ public.livefsbuild = SELECT, UPDATE
6 public.livefsfile = SELECT
7 public.ocifile = SELECT
8 public.ociproject = SELECT
9+public.ociprojectname = SELECT
10 public.ocirecipe = SELECT
11 public.ocirecipebuild = SELECT, UPDATE
12 public.openididentifier = SELECT
13diff --git a/lib/lp/buildmaster/model/buildfarmjobbehaviour.py b/lib/lp/buildmaster/model/buildfarmjobbehaviour.py
14index 2f3515a..4b28693 100644
15--- a/lib/lp/buildmaster/model/buildfarmjobbehaviour.py
16+++ b/lib/lp/buildmaster/model/buildfarmjobbehaviour.py
17@@ -301,6 +301,22 @@ class BuildFarmJobBehaviourBase:
18 transaction.commit()
19
20 @defer.inlineCallbacks
21+ def _downloadFiles(self, filemap, upload_path, logger):
22+ filenames_to_download = []
23+ for filename, sha1 in filemap.items():
24+ logger.info("Grabbing file: %s (%s)" % (
25+ filename, self._slave.getURL(sha1)))
26+ out_file_name = os.path.join(upload_path, filename)
27+ # If the evaluated output file name is not within our
28+ # upload path, then we don't try to copy this or any
29+ # subsequent files.
30+ if not os.path.realpath(out_file_name).startswith(upload_path):
31+ raise BuildDaemonError(
32+ "Build returned a file named %r." % filename)
33+ filenames_to_download.append((sha1, out_file_name))
34+ yield self._slave.getFiles(filenames_to_download, logger=logger)
35+
36+ @defer.inlineCallbacks
37 def handleSuccess(self, slave_status, logger):
38 """Handle a package that built successfully.
39
40@@ -337,19 +353,7 @@ class BuildFarmJobBehaviourBase:
41 grab_dir, str(build.archive.id), build.distribution.name)
42 os.makedirs(upload_path)
43
44- filenames_to_download = []
45- for filename, sha1 in filemap.items():
46- logger.info("Grabbing file: %s (%s)" % (
47- filename, self._slave.getURL(sha1)))
48- out_file_name = os.path.join(upload_path, filename)
49- # If the evaluated output file name is not within our
50- # upload path, then we don't try to copy this or any
51- # subsequent files.
52- if not os.path.realpath(out_file_name).startswith(upload_path):
53- raise BuildDaemonError(
54- "Build returned a file named %r." % filename)
55- filenames_to_download.append((sha1, out_file_name))
56- yield self._slave.getFiles(filenames_to_download, logger=logger)
57+ yield self._downloadFiles(filemap, upload_path, logger)
58
59 transaction.commit()
60
61diff --git a/lib/lp/buildmaster/tests/mock_slaves.py b/lib/lp/buildmaster/tests/mock_slaves.py
62index a19a85d..32633ea 100644
63--- a/lib/lp/buildmaster/tests/mock_slaves.py
64+++ b/lib/lp/buildmaster/tests/mock_slaves.py
65@@ -194,7 +194,8 @@ class WaitingSlave(OkSlave):
66
67 # By default, the slave only has a buildlog, but callsites
68 # can update this list as needed.
69- self.valid_file_hashes = ['buildlog']
70+ self.valid_files = {'buildlog': ''}
71+ self._got_file_record = []
72
73 def status(self):
74 self.call_log.append('status')
75@@ -208,12 +209,17 @@ class WaitingSlave(OkSlave):
76
77 def getFile(self, hash, file_to_write):
78 self.call_log.append('getFile')
79- if hash in self.valid_file_hashes:
80- content = "This is a %s" % hash
81+ if hash in self.valid_files:
82 if isinstance(file_to_write, types.StringTypes):
83 file_to_write = open(file_to_write, 'wb')
84+ if not self.valid_files[hash]:
85+ content = b"This is a %s" % hash
86+ else:
87+ with open(self.valid_files[hash], 'rb') as source:
88+ content = source.read()
89 file_to_write.write(content)
90 file_to_write.close()
91+ self._got_file_record.append(hash)
92 return defer.succeed(None)
93
94
95diff --git a/lib/lp/buildmaster/tests/snapbuildproxy.py b/lib/lp/buildmaster/tests/snapbuildproxy.py
96new file mode 100644
97index 0000000..21d973e
98--- /dev/null
99+++ b/lib/lp/buildmaster/tests/snapbuildproxy.py
100@@ -0,0 +1,123 @@
101+# Copyright 2015-2019 Canonical Ltd. This software is licensed under the
102+# GNU Affero General Public License version 3 (see the file LICENSE).
103+
104+"""Fixtures for dealing with the build time 'snap' HTTP proxy."""
105+
106+from __future__ import absolute_import, print_function, unicode_literals
107+
108+__metaclass__ = type
109+
110+from datetime import datetime
111+import json
112+from textwrap import dedent
113+import uuid
114+
115+import fixtures
116+from six.moves.urllib_parse import urlsplit
117+from testtools.matchers import (
118+ Equals,
119+ HasLength,
120+ MatchesStructure,
121+ )
122+from twisted.internet import (
123+ defer,
124+ endpoints,
125+ reactor,
126+ )
127+from twisted.python.compat import nativeString
128+from twisted.web import (
129+ resource,
130+ server,
131+ )
132+
133+from lp.services.config import config
134+
135+
136+class ProxyAuthAPITokensResource(resource.Resource):
137+ """A test tokens resource for the proxy authentication API."""
138+
139+ isLeaf = True
140+
141+ def __init__(self):
142+ resource.Resource.__init__(self)
143+ self.requests = []
144+
145+ def render_POST(self, request):
146+ content = request.content.read()
147+ self.requests.append({
148+ "method": request.method,
149+ "uri": request.uri,
150+ "headers": dict(request.requestHeaders.getAllRawHeaders()),
151+ "content": content,
152+ })
153+ username = json.loads(content)["username"]
154+ return json.dumps({
155+ "username": username,
156+ "secret": uuid.uuid4().hex,
157+ "timestamp": datetime.utcnow().isoformat(),
158+ })
159+
160+
161+class InProcessProxyAuthAPIFixture(fixtures.Fixture):
162+ """A fixture that pretends to be the proxy authentication API.
163+
164+ Users of this fixture must call the `start` method, which returns a
165+ `Deferred`, and arrange for that to get back to the reactor. This is
166+ necessary because the basic fixture API does not allow `setUp` to return
167+ anything. For example:
168+
169+ class TestSomething(TestCase):
170+
171+ run_tests_with = AsynchronousDeferredRunTest.make_factory(
172+ timeout=10)
173+
174+ @defer.inlineCallbacks
175+ def setUp(self):
176+ super(TestSomething, self).setUp()
177+ yield self.useFixture(InProcessProxyAuthAPIFixture()).start()
178+ """
179+
180+ @defer.inlineCallbacks
181+ def start(self):
182+ root = resource.Resource()
183+ self.tokens = ProxyAuthAPITokensResource()
184+ root.putChild("tokens", self.tokens)
185+ endpoint = endpoints.serverFromString(reactor, nativeString("tcp:0"))
186+ site = server.Site(self.tokens)
187+ self.addCleanup(site.stopFactory)
188+ port = yield endpoint.listen(site)
189+ self.addCleanup(port.stopListening)
190+ config.push("in-process-proxy-auth-api-fixture", dedent("""
191+ [snappy]
192+ builder_proxy_auth_api_admin_secret: admin-secret
193+ builder_proxy_auth_api_endpoint: http://%s:%s/tokens
194+ """) %
195+ (port.getHost().host, port.getHost().port))
196+ self.addCleanup(config.pop, "in-process-proxy-auth-api-fixture")
197+
198+
199+class ProxyURLMatcher(MatchesStructure):
200+ """Check that a string is a valid url for a snap build proxy."""
201+
202+ def __init__(self, job, now):
203+ super(ProxyURLMatcher, self).__init__(
204+ scheme=Equals("http"),
205+ username=Equals("{}-{}".format(
206+ job.build.build_cookie, int(now))),
207+ password=HasLength(32),
208+ hostname=Equals(config.snappy.builder_proxy_host),
209+ port=Equals(config.snappy.builder_proxy_port),
210+ path=Equals(""))
211+
212+ def match(self, matchee):
213+ super(ProxyURLMatcher, self).match(urlsplit(matchee))
214+
215+
216+class RevocationEndpointMatcher(Equals):
217+ """Check that a string is a valid endpoint for proxy token revocation."""
218+
219+ def __init__(self, job, now):
220+ super(RevocationEndpointMatcher, self).__init__(
221+ "{}/{}-{}".format(
222+ config.snappy.builder_proxy_auth_api_endpoint,
223+ job.build.build_cookie, int(now)))
224diff --git a/lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py b/lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py
225index 697ff52..5046d8c 100644
226--- a/lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py
227+++ b/lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py
228@@ -337,7 +337,7 @@ class TestHandleStatusMixin:
229 self.builder = self.factory.makeBuilder()
230 self.build.buildqueue_record.markAsBuilding(self.builder)
231 self.slave = WaitingSlave('BuildStatus.OK')
232- self.slave.valid_file_hashes.append('test_file_hash')
233+ self.slave.valid_files['test_file_hash'] = ''
234 self.interactor = BuilderInteractor()
235 self.behaviour = self.interactor.getBuildBehaviour(
236 self.build.buildqueue_record, self.builder, self.slave)
237diff --git a/lib/lp/oci/configure.zcml b/lib/lp/oci/configure.zcml
238index b86afcb..a01178a 100644
239--- a/lib/lp/oci/configure.zcml
240+++ b/lib/lp/oci/configure.zcml
241@@ -63,4 +63,10 @@
242 <allow interface="lp.buildmaster.interfaces.buildfarmjob.ISpecificBuildFarmJobSource" />
243 </securedutility>
244
245+ <adapter
246+ for="lp.oci.interfaces.ocirecipebuild.IOCIRecipeBuild"
247+ provides="lp.buildmaster.interfaces.buildfarmjobbehaviour.IBuildFarmJobBehaviour"
248+ factory="lp.oci.model.ocirecipebuildbehaviour.OCIRecipeBuildBehaviour"
249+ permission="zope.Public" />
250+
251 </configure>
252diff --git a/lib/lp/oci/interfaces/ocirecipebuild.py b/lib/lp/oci/interfaces/ocirecipebuild.py
253index 30c3835..a41e2c8 100644
254--- a/lib/lp/oci/interfaces/ocirecipebuild.py
255+++ b/lib/lp/oci/interfaces/ocirecipebuild.py
256@@ -27,6 +27,7 @@ from lp.oci.interfaces.ocirecipe import IOCIRecipe
257 from lp.services.database.constants import DEFAULT
258 from lp.services.fields import PublicPersonChoice
259 from lp.services.librarian.interfaces import ILibraryFileAlias
260+from lp.soyuz.interfaces.distroarchseries import IDistroArchSeries
261
262
263 class IOCIRecipeBuildEdit(Interface):
264@@ -82,6 +83,11 @@ class IOCIRecipeBuildView(IPackageBuild):
265 :return: The corresponding `ILibraryFileAlias`.
266 """
267
268+ distro_arch_series = Reference(
269+ IDistroArchSeries,
270+ title=_("The series and architecture for which to build."),
271+ required=True, readonly=True)
272+
273
274 class IOCIRecipeBuildAdmin(Interface):
275 # XXX twom 2020-02-10 This will probably need rescore() implementing
276diff --git a/lib/lp/oci/model/ocirecipebuild.py b/lib/lp/oci/model/ocirecipebuild.py
277index c010817..12688cd 100644
278--- a/lib/lp/oci/model/ocirecipebuild.py
279+++ b/lib/lp/oci/model/ocirecipebuild.py
280@@ -44,7 +44,9 @@ from lp.oci.interfaces.ocirecipebuild import (
281 IOCIRecipeBuild,
282 IOCIRecipeBuildSet,
283 )
284+from lp.registry.interfaces.pocket import PackagePublishingPocket
285 from lp.registry.model.person import Person
286+from lp.services.config import config
287 from lp.services.database.bulk import load_related
288 from lp.services.database.constants import DEFAULT
289 from lp.services.database.decoratedresultset import DecoratedResultSet
290@@ -128,10 +130,16 @@ class OCIRecipeBuild(PackageBuildMixin, Storm):
291 build_farm_job_id = Int(name='build_farm_job', allow_none=False)
292 build_farm_job = Reference(build_farm_job_id, 'BuildFarmJob.id')
293
294- # Stub attributes to match the IPackageBuild interface that we
295- # are not using in this implementation at this time.
296- pocket = None
297- distro_series = None
298+ # We only care about the pocket from a building environment POV,
299+ # it is not a target, nor referenced in the final build.
300+ pocket = PackagePublishingPocket.UPDATES
301+
302+ @property
303+ def distro_series(self):
304+ # XXX twom 2020-02-14 - This really needs to be set elsewhere,
305+ # as this may not be an LTS release and ties the OCI target to
306+ # a completely unrelated process.
307+ return self.distribution.currentseries
308
309 def __init__(self, build_farm_job, requester, recipe,
310 processor, virtualized, date_created):
311@@ -257,6 +265,19 @@ class OCIRecipeBuild(PackageBuildMixin, Storm):
312 # pillar isn't just a distribution
313 return self.recipe.oci_project.distribution
314
315+ @property
316+ def distro_arch_series(self):
317+ return self.distribution.currentseries.getDistroArchSeriesByProcessor(
318+ self.processor)
319+
320+ def notify(self, extra_info=None):
321+ """See `IPackageBuild`."""
322+ if not config.builddmaster.send_build_notification:
323+ return
324+ if self.status == BuildStatus.FULLYBUILT:
325+ return
326+ # XXX twom 2019-12-11 This should send mail
327+
328
329 @implementer(IOCIRecipeBuildSet)
330 class OCIRecipeBuildSet(SpecificBuildFarmJobSourceMixin):
331diff --git a/lib/lp/oci/model/ocirecipebuildbehaviour.py b/lib/lp/oci/model/ocirecipebuildbehaviour.py
332new file mode 100644
333index 0000000..f0debfa
334--- /dev/null
335+++ b/lib/lp/oci/model/ocirecipebuildbehaviour.py
336@@ -0,0 +1,182 @@
337+# Copyright 2019 Canonical Ltd. This software is licensed under the
338+# GNU Affero General Public License version 3 (see the file LICENSE).
339+
340+"""An `IBuildFarmJobBehaviour` for `OCIRecipeBuild`.
341+
342+Dispatches OCI image build jobs to build-farm slaves.
343+"""
344+
345+from __future__ import absolute_import, print_function, unicode_literals
346+
347+__metaclass__ = type
348+__all__ = [
349+ 'OCIRecipeBuildBehaviour',
350+ ]
351+
352+
353+import json
354+import os
355+
356+from twisted.internet import defer
357+from zope.interface import implementer
358+
359+from lp.app.errors import NotFoundError
360+from lp.buildmaster.enums import BuildBaseImageType
361+from lp.buildmaster.interfaces.builder import (
362+ BuildDaemonError,
363+ CannotBuild,
364+ )
365+from lp.buildmaster.interfaces.buildfarmjobbehaviour import (
366+ IBuildFarmJobBehaviour,
367+ )
368+from lp.buildmaster.model.buildfarmjobbehaviour import (
369+ BuildFarmJobBehaviourBase,
370+ )
371+from lp.registry.interfaces.series import SeriesStatus
372+from lp.services.librarian.utils import copy_and_close
373+from lp.snappy.model.snapbuildbehaviour import SnapProxyMixin
374+from lp.soyuz.adapters.archivedependencies import (
375+ get_sources_list_for_building,
376+ )
377+
378+
379+@implementer(IBuildFarmJobBehaviour)
380+class OCIRecipeBuildBehaviour(SnapProxyMixin, BuildFarmJobBehaviourBase):
381+
382+ builder_type = "oci"
383+ image_types = [BuildBaseImageType.LXD, BuildBaseImageType.CHROOT]
384+
385+ def getLogFileName(self):
386+ series = self.build.distro_series
387+
388+ # Examples:
389+ # buildlog_oci_ubuntu_wily_amd64_name_FULLYBUILT.txt
390+ return 'buildlog_oci_%s_%s_%s_%s_%s' % (
391+ series.distribution.name, series.name,
392+ self.build.processor.name, self.build.recipe.name,
393+ self.build.status.name)
394+
395+ def verifyBuildRequest(self, logger):
396+ """Assert some pre-build checks.
397+
398+ The build request is checked:
399+ * Virtualized builds can't build on a non-virtual builder
400+ * Ensure that we have a chroot
401+ """
402+ build = self.build
403+ if build.virtualized and not self._builder.virtualized:
404+ raise AssertionError(
405+ "Attempt to build virtual item on a non-virtual builder.")
406+
407+ chroot = build.distro_arch_series.getChroot(pocket=build.pocket)
408+ if chroot is None:
409+ raise CannotBuild(
410+ "Missing chroot for %s" % build.distro_arch_series.displayname)
411+
412+ @defer.inlineCallbacks
413+ def extraBuildArgs(self, logger=None):
414+ """
415+ Return the extra arguments required by the slave for the given build.
416+ """
417+ build = self.build
418+ args = yield super(OCIRecipeBuildBehaviour, self).extraBuildArgs(
419+ logger=logger)
420+ yield self.addProxyArgs(args)
421+ # XXX twom 2020-02-17 This may need to be more complex, and involve
422+ # distribution name.
423+ args["name"] = build.recipe.name
424+ args["archives"], args["trusted_keys"] = (
425+ yield get_sources_list_for_building(
426+ build, build.distro_arch_series, None,
427+ tools_source=None, tools_fingerprint=None,
428+ logger=logger))
429+
430+ args['build_file'] = build.recipe.build_file
431+
432+ if build.recipe.git_ref is not None:
433+ args["git_repository"] = (
434+ build.recipe.git_repository.git_https_url)
435+ else:
436+ raise CannotBuild(
437+ "Source repository for ~%s/%s has been deleted." %
438+ (build.recipe.owner.name, build.recipe.name))
439+
440+ if build.recipe.git_path != "HEAD":
441+ args["git_path"] = build.recipe.git_ref.name
442+
443+ defer.returnValue(args)
444+
445+ def _ensureFilePath(self, file_name, file_path, upload_path):
446+ # If the evaluated output file name is not within our
447+ # upload path, then we don't try to copy this or any
448+ # subsequent files.
449+ if not os.path.normpath(file_path).startswith(upload_path + '/'):
450+ raise BuildDaemonError(
451+ "Build returned a file named '%s'." % file_name)
452+
453+ @defer.inlineCallbacks
454+ def _fetchIntermediaryFile(self, name, filemap, upload_path):
455+ file_hash = filemap[name]
456+ file_path = os.path.join(upload_path, name)
457+ self._ensureFilePath(name, file_path, upload_path)
458+ yield self._slave.getFile(file_hash, file_path)
459+
460+ with open(file_path, 'r') as file_fp:
461+ contents = json.load(file_fp)
462+ defer.returnValue(contents)
463+
464+ def _extractLayerFiles(self, upload_path, section, config, digests, files):
465+ # These are different sets of ids, in the same order
466+ # layer_id is the filename, diff_id is the internal (docker) id
467+ for diff_id in config['rootfs']['diff_ids']:
468+ for digests_section in digests:
469+ layer_id = digests_section[diff_id]['layer_id']
470+ # This is in the form '<id>/layer.tar', we only need the first
471+ layer_filename = "{}.tar.gz".format(layer_id.split('/')[0])
472+ digest = digests_section[diff_id]['digest']
473+ try:
474+ _, librarian_file, _ = self.build.getLayerFileByDigest(
475+ digest)
476+ except NotFoundError:
477+ files.add(layer_filename)
478+ continue
479+ layer_path = os.path.join(upload_path, layer_filename)
480+ librarian_file.open()
481+ copy_and_close(librarian_file, open(layer_path, 'wb'))
482+
483+ def _convertToRetrievableFile(self, upload_path, file_name, filemap):
484+ file_path = os.path.join(upload_path, file_name)
485+ self._ensureFilePath(file_name, file_path, upload_path)
486+ return (filemap[file_name], file_path)
487+
488+ @defer.inlineCallbacks
489+ def _downloadFiles(self, filemap, upload_path, logger):
490+ """Download required artifact files."""
491+ # We don't want to download all of the files that have been created,
492+ # just the ones that are mentioned in the manifest and config.
493+
494+ manifest = yield self._fetchIntermediaryFile(
495+ 'manifest.json', filemap, upload_path)
496+ digests = yield self._fetchIntermediaryFile(
497+ 'digests.json', filemap, upload_path)
498+
499+ files = set()
500+ for section in manifest:
501+ config = yield self._fetchIntermediaryFile(
502+ section['Config'], filemap, upload_path)
503+ self._extractLayerFiles(
504+ upload_path, section, config, digests, files)
505+
506+ files_to_download = [
507+ self._convertToRetrievableFile(upload_path, filename, filemap)
508+ for filename in files]
509+ yield self._slave.getFiles(files_to_download, logger=logger)
510+
511+ def verifySuccessfulBuild(self):
512+ """See `IBuildFarmJobBehaviour`."""
513+ # The implementation in BuildFarmJobBehaviourBase checks whether the
514+ # target suite is modifiable in the target archive. However, an
515+ # `OCIRecipeBuild` does not use an archive in this manner.
516+ # We do, however, refuse to build for
517+ # obsolete series.
518+ assert self.build.distro_series.status != SeriesStatus.OBSOLETE
519diff --git a/lib/lp/oci/tests/test_ocirecipebuild.py b/lib/lp/oci/tests/test_ocirecipebuild.py
520index b768e1d..60aaf93 100644
521--- a/lib/lp/oci/tests/test_ocirecipebuild.py
522+++ b/lib/lp/oci/tests/test_ocirecipebuild.py
523@@ -16,11 +16,13 @@ from lp.app.errors import NotFoundError
524 from lp.buildmaster.enums import BuildStatus
525 from lp.buildmaster.interfaces.buildqueue import IBuildQueue
526 from lp.buildmaster.interfaces.packagebuild import IPackageBuild
527+from lp.buildmaster.interfaces.processor import IProcessorSet
528 from lp.oci.interfaces.ocirecipebuild import (
529 IOCIRecipeBuild,
530 IOCIRecipeBuildSet,
531 )
532 from lp.oci.model.ocirecipebuild import OCIRecipeBuildSet
533+from lp.registry.interfaces.series import SeriesStatus
534 from lp.services.propertycache import clear_property_cache
535 from lp.testing import (
536 admin_logged_in,
537@@ -148,8 +150,16 @@ class TestOCIRecipeBuildSet(TestCaseWithFactory):
538
539 def test_new(self):
540 requester = self.factory.makePerson()
541- recipe = self.factory.makeOCIRecipe()
542+ distribution = self.factory.makeDistribution()
543+ distroseries = self.factory.makeDistroSeries(
544+ distribution=distribution, status=SeriesStatus.CURRENT)
545+ processor = getUtility(IProcessorSet).getByName("386")
546+ distro_arch_series = self.factory.makeDistroArchSeries(
547+ distroseries=distroseries, architecturetag="i386",
548+ processor=processor)
549 distro_arch_series = self.factory.makeDistroArchSeries()
550+ oci_project = self.factory.makeOCIProject(pillar=distribution)
551+ recipe = self.factory.makeOCIRecipe(oci_project=oci_project)
552 target = getUtility(IOCIRecipeBuildSet).new(
553 requester, recipe, distro_arch_series)
554 with admin_logged_in():
555diff --git a/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py b/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py
556new file mode 100644
557index 0000000..e9ac224
558--- /dev/null
559+++ b/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py
560@@ -0,0 +1,604 @@
561+# Copyright 2015-2019 Canonical Ltd. This software is licensed under the
562+# GNU Affero General Public License version 3 (see the file LICENSE).
563+
564+"""Tests for `OCIRecipeBuildBehaviour`."""
565+
566+from __future__ import absolute_import, print_function, unicode_literals
567+
568+__metaclass__ = type
569+
570+import base64
571+from datetime import datetime
572+import json
573+import os
574+import shutil
575+import tempfile
576+import time
577+import uuid
578+
579+import fixtures
580+from six.moves.urllib_parse import urlsplit
581+from testtools import ExpectedException
582+from testtools.matchers import (
583+ AfterPreprocessing,
584+ ContainsDict,
585+ Equals,
586+ Is,
587+ IsInstance,
588+ MatchesDict,
589+ MatchesListwise,
590+ StartsWith,
591+ )
592+from testtools.twistedsupport import (
593+ AsynchronousDeferredRunTestForBrokenTwisted,
594+ )
595+from twisted.internet import defer
596+from twisted.trial.unittest import TestCase as TrialTestCase
597+from zope.component import getUtility
598+from zope.security.proxy import removeSecurityProxy
599+
600+from lp.buildmaster.enums import (
601+ BuildBaseImageType,
602+ BuildStatus,
603+ )
604+from lp.buildmaster.interactor import BuilderInteractor
605+from lp.buildmaster.interfaces.builder import (
606+ BuildDaemonError,
607+ CannotBuild,
608+ )
609+from lp.buildmaster.interfaces.buildfarmjobbehaviour import (
610+ IBuildFarmJobBehaviour,
611+ )
612+from lp.buildmaster.interfaces.processor import IProcessorSet
613+from lp.buildmaster.tests.mock_slaves import (
614+ MockBuilder,
615+ OkSlave,
616+ SlaveTestHelpers,
617+ WaitingSlave,
618+ )
619+from lp.buildmaster.tests.snapbuildproxy import (
620+ InProcessProxyAuthAPIFixture,
621+ ProxyURLMatcher,
622+ RevocationEndpointMatcher,
623+ )
624+from lp.buildmaster.tests.test_buildfarmjobbehaviour import (
625+ TestGetUploadMethodsMixin,
626+ )
627+from lp.oci.model.ocirecipebuildbehaviour import OCIRecipeBuildBehaviour
628+from lp.registry.interfaces.series import SeriesStatus
629+from lp.services.config import config
630+from lp.services.log.logger import DevNullLogger
631+from lp.services.webapp import canonical_url
632+from lp.soyuz.adapters.archivedependencies import (
633+ get_sources_list_for_building,
634+ )
635+from lp.testing import TestCaseWithFactory
636+from lp.testing.dbuser import dbuser
637+from lp.testing.factory import LaunchpadObjectFactory
638+from lp.testing.fakemethod import FakeMethod
639+from lp.testing.layers import LaunchpadZopelessLayer
640+from lp.testing.mail_helpers import pop_notifications
641+
642+
643+class MakeOCIBuildMixin:
644+
645+ def makeBuild(self):
646+ build = self.factory.makeOCIRecipeBuild()
647+ self.factory.makeDistroSeries(
648+ distribution=build.recipe.oci_project.distribution,
649+ status=SeriesStatus.CURRENT)
650+ build.queueBuild()
651+ return build
652+
653+ def makeUnmodifiableBuild(self):
654+ build = self.factory.makeOCIRecipeBuild()
655+ build.distro_arch_series = 'failed'
656+ build.queueBuild()
657+ return build
658+
659+ def makeJob(self, git_ref, recipe=None):
660+ """Create a sample `IOCIRecipeBuildBehaviour`."""
661+ if recipe is None:
662+ build = self.factory.makeOCIRecipeBuild()
663+ else:
664+ build = self.factory.makeOCIRecipeBuild(recipe=recipe)
665+ build.recipe.git_ref = git_ref
666+
667+ job = IBuildFarmJobBehaviour(build)
668+ builder = MockBuilder()
669+ builder.processor = job.build.processor
670+ slave = self.useFixture(SlaveTestHelpers()).getClientSlave()
671+ job.setBuilder(builder, slave)
672+ self.addCleanup(slave.pool.closeCachedConnections)
673+
674+ # Taken from test_archivedependencies.py
675+ for component_name in ("main", "universe"):
676+ self.factory.makeComponentSelection(
677+ build.distro_arch_series.distroseries, component_name)
678+
679+ return job
680+
681+
682+class TestOCIBuildBehaviour(TestCaseWithFactory):
683+
684+ layer = LaunchpadZopelessLayer
685+
686+ def test_provides_interface(self):
687+ # OCIRecipeBuildBehaviour provides IBuildFarmJobBehaviour.
688+ job = OCIRecipeBuildBehaviour(self.factory.makeOCIRecipeBuild())
689+ self.assertProvides(job, IBuildFarmJobBehaviour)
690+
691+ def test_adapts_IOCIRecipeBuild(self):
692+ # IBuildFarmJobBehaviour adapts an IOCIRecipeBuild.
693+ build = self.factory.makeOCIRecipeBuild()
694+ job = IBuildFarmJobBehaviour(build)
695+ self.assertProvides(job, IBuildFarmJobBehaviour)
696+
697+
698+class TestAsyncOCIRecipeBuildBehaviour(MakeOCIBuildMixin, TestCaseWithFactory):
699+
700+ run_tests_with = AsynchronousDeferredRunTestForBrokenTwisted.make_factory(
701+ timeout=10)
702+ layer = LaunchpadZopelessLayer
703+
704+ @defer.inlineCallbacks
705+ def setUp(self):
706+ super(TestAsyncOCIRecipeBuildBehaviour, self).setUp()
707+ build_username = 'OCIBUILD-1'
708+ self.token = {'secret': uuid.uuid4().get_hex(),
709+ 'username': build_username,
710+ 'timestamp': datetime.utcnow().isoformat()}
711+ self.proxy_url = ("http://{username}:{password}"
712+ "@{host}:{port}".format(
713+ username=self.token['username'],
714+ password=self.token['secret'],
715+ host=config.snappy.builder_proxy_host,
716+ port=config.snappy.builder_proxy_port))
717+ self.proxy_api = self.useFixture(InProcessProxyAuthAPIFixture())
718+ yield self.proxy_api.start()
719+ self.now = time.time()
720+ self.useFixture(fixtures.MockPatch(
721+ "time.time", return_value=self.now))
722+
723+ @defer.inlineCallbacks
724+ def test_composeBuildRequest(self):
725+ [ref] = self.factory.makeGitRefs()
726+ job = self.makeJob(git_ref=ref)
727+ lfa = self.factory.makeLibraryFileAlias(db_only=True)
728+ job.build.distro_arch_series.addOrUpdateChroot(lfa)
729+ build_request = yield job.composeBuildRequest(None)
730+ self.assertThat(build_request, MatchesListwise([
731+ Equals('oci'),
732+ Equals(job.build.distro_arch_series),
733+ Equals(job.build.pocket),
734+ Equals({}),
735+ IsInstance(dict),
736+ ]))
737+
738+ @defer.inlineCallbacks
739+ def test_requestProxyToken_unconfigured(self):
740+ self.pushConfig("snappy", builder_proxy_auth_api_admin_secret=None)
741+ [ref] = self.factory.makeGitRefs()
742+ job = self.makeJob(git_ref=ref)
743+ expected_exception_msg = (
744+ "builder_proxy_auth_api_admin_secret is not configured.")
745+ with ExpectedException(CannotBuild, expected_exception_msg):
746+ yield job.extraBuildArgs()
747+
748+ @defer.inlineCallbacks
749+ def test_requestProxyToken(self):
750+ [ref] = self.factory.makeGitRefs()
751+ job = self.makeJob(git_ref=ref)
752+ yield job.extraBuildArgs()
753+ self.assertThat(self.proxy_api.tokens.requests, MatchesListwise([
754+ MatchesDict({
755+ "method": Equals("POST"),
756+ "uri": Equals(urlsplit(
757+ config.snappy.builder_proxy_auth_api_endpoint).path),
758+ "headers": ContainsDict({
759+ b"Authorization": MatchesListwise([
760+ Equals(b"Basic " + base64.b64encode(
761+ b"admin-launchpad.test:admin-secret"))]),
762+ b"Content-Type": MatchesListwise([
763+ Equals(b"application/json; charset=UTF-8"),
764+ ]),
765+ }),
766+ "content": AfterPreprocessing(json.loads, MatchesDict({
767+ "username": StartsWith(job.build.build_cookie + "-"),
768+ })),
769+ }),
770+ ]))
771+
772+ @defer.inlineCallbacks
773+ def test_extraBuildArgs_git(self):
774+ # extraBuildArgs returns appropriate arguments if asked to build a
775+ # job for a Git branch.
776+ [ref] = self.factory.makeGitRefs()
777+ job = self.makeJob(git_ref=ref)
778+ expected_archives, expected_trusted_keys = (
779+ yield get_sources_list_for_building(
780+ job.build, job.build.distro_arch_series, None))
781+ for archive_line in expected_archives:
782+ self.assertIn('universe', archive_line)
783+ with dbuser(config.builddmaster.dbuser):
784+ args = yield job.extraBuildArgs()
785+ self.assertThat(args, MatchesDict({
786+ "archive_private": Is(False),
787+ "archives": Equals(expected_archives),
788+ "arch_tag": Equals("i386"),
789+ "build_file": Equals(job.build.recipe.build_file),
790+ "build_url": Equals(canonical_url(job.build)),
791+ "fast_cleanup": Is(True),
792+ "git_repository": Equals(ref.repository.git_https_url),
793+ "git_path": Equals(ref.name),
794+ "name": Equals(job.build.recipe.name),
795+ "proxy_url": ProxyURLMatcher(job, self.now),
796+ "revocation_endpoint": RevocationEndpointMatcher(job, self.now),
797+ "series": Equals(job.build.distro_arch_series.distroseries.name),
798+ "trusted_keys": Equals(expected_trusted_keys),
799+ }))
800+
801+ @defer.inlineCallbacks
802+ def test_extraBuildArgs_git_HEAD(self):
803+ # extraBuildArgs returns appropriate arguments if asked to build a
804+ # job for the default branch in a Launchpad-hosted Git repository.
805+ [ref] = self.factory.makeGitRefs()
806+ removeSecurityProxy(ref.repository)._default_branch = ref.path
807+ job = self.makeJob(git_ref=ref.repository.getRefByPath("HEAD"))
808+ expected_archives, expected_trusted_keys = (
809+ yield get_sources_list_for_building(
810+ job.build, job.build.distro_arch_series, None))
811+ for archive_line in expected_archives:
812+ self.assertIn('universe', archive_line)
813+ with dbuser(config.builddmaster.dbuser):
814+ args = yield job.extraBuildArgs()
815+ self.assertThat(args, MatchesDict({
816+ "archive_private": Is(False),
817+ "archives": Equals(expected_archives),
818+ "arch_tag": Equals("i386"),
819+ "build_file": Equals(job.build.recipe.build_file),
820+ "build_url": Equals(canonical_url(job.build)),
821+ "fast_cleanup": Is(True),
822+ "git_repository": Equals(ref.repository.git_https_url),
823+ "name": Equals(job.build.recipe.name),
824+ "proxy_url": ProxyURLMatcher(job, self.now),
825+ "revocation_endpoint": RevocationEndpointMatcher(job, self.now),
826+ "series": Equals(job.build.distro_arch_series.distroseries.name),
827+ "trusted_keys": Equals(expected_trusted_keys),
828+ }))
829+
830+ @defer.inlineCallbacks
831+ def test_composeBuildRequest_proxy_url_set(self):
832+ [ref] = self.factory.makeGitRefs()
833+ job = self.makeJob(git_ref=ref)
834+ build_request = yield job.composeBuildRequest(None)
835+ self.assertThat(
836+ build_request[4]["proxy_url"], ProxyURLMatcher(job, self.now))
837+
838+ @defer.inlineCallbacks
839+ def test_composeBuildRequest_git_ref_deleted(self):
840+ # If the source Git reference has been deleted, composeBuildRequest
841+ # raises CannotBuild.
842+ repository = self.factory.makeGitRepository()
843+ [ref] = self.factory.makeGitRefs(repository=repository)
844+ owner = self.factory.makePerson(name="oci-owner")
845+
846+ distribution = self.factory.makeDistribution()
847+ distroseries = self.factory.makeDistroSeries(
848+ distribution=distribution, status=SeriesStatus.CURRENT)
849+ processor = getUtility(IProcessorSet).getByName("386")
850+ self.factory.makeDistroArchSeries(
851+ distroseries=distroseries, architecturetag="i386",
852+ processor=processor)
853+
854+ oci_project = self.factory.makeOCIProject(
855+ pillar=distribution, registrant=owner)
856+ recipe = self.factory.makeOCIRecipe(
857+ oci_project=oci_project, registrant=owner, owner=owner,
858+ git_ref=ref)
859+ job = self.makeJob(ref, recipe=recipe)
860+ repository.removeRefs([ref.path])
861+ self.assertIsNone(job.build.recipe.git_ref)
862+ expected_exception_msg = ("Source repository for "
863+ "~oci-owner/{} has been deleted.".format(
864+ recipe.name))
865+ with ExpectedException(CannotBuild, expected_exception_msg):
866+ yield job.composeBuildRequest(None)
867+
868+ @defer.inlineCallbacks
869+ def test_dispatchBuildToSlave_prefers_lxd(self):
870+ self.pushConfig("snappy", builder_proxy_host=None)
871+ [ref] = self.factory.makeGitRefs()
872+ job = self.makeJob(git_ref=ref)
873+ builder = MockBuilder()
874+ builder.processor = job.build.processor
875+ slave = OkSlave()
876+ job.setBuilder(builder, slave)
877+ chroot_lfa = self.factory.makeLibraryFileAlias(db_only=True)
878+ job.build.distro_arch_series.addOrUpdateChroot(
879+ chroot_lfa, image_type=BuildBaseImageType.CHROOT)
880+ lxd_lfa = self.factory.makeLibraryFileAlias(db_only=True)
881+ job.build.distro_arch_series.addOrUpdateChroot(
882+ lxd_lfa, image_type=BuildBaseImageType.LXD)
883+ yield job.dispatchBuildToSlave(DevNullLogger())
884+ self.assertEqual(
885+ ('ensurepresent', lxd_lfa.http_url, '', ''), slave.call_log[0])
886+
887+ @defer.inlineCallbacks
888+ def test_dispatchBuildToSlave_falls_back_to_chroot(self):
889+ self.pushConfig("snappy", builder_proxy_host=None)
890+ [ref] = self.factory.makeGitRefs()
891+ job = self.makeJob(git_ref=ref)
892+ builder = MockBuilder()
893+ builder.processor = job.build.processor
894+ slave = OkSlave()
895+ job.setBuilder(builder, slave)
896+ chroot_lfa = self.factory.makeLibraryFileAlias(db_only=True)
897+ job.build.distro_arch_series.addOrUpdateChroot(
898+ chroot_lfa, image_type=BuildBaseImageType.CHROOT)
899+ yield job.dispatchBuildToSlave(DevNullLogger())
900+ self.assertEqual(
901+ ('ensurepresent', chroot_lfa.http_url, '', ''), slave.call_log[0])
902+
903+
904+class TestHandleStatusForOCIRecipeBuild(MakeOCIBuildMixin, TrialTestCase,
905+ fixtures.TestWithFixtures):
906+ # This is mostly copied from TestHandleStatusMixin, however
907+ # we can't use all of those tests, due to the way OCIRecipeBuildBehaviour
908+ # parses the file contents, rather than just retrieving all that are
909+ # available. There's also some differences in the filemap handling, as
910+ # we need a much more complex filemap here.
911+
912+ layer = LaunchpadZopelessLayer
913+
914+ def pushConfig(self, section, **kwargs):
915+ """Push some key-value pairs into a section of the config.
916+
917+ The config values will be restored during test tearDown.
918+ """
919+ # Taken from lp/testing.py as we're using TrialTestCase,
920+ # not lp.testing.TestCase, as we need to handle the deferred
921+ # correctly.
922+ name = self.factory.getUniqueString()
923+ body = '\n'.join("%s: %s" % (k, v) for k, v in kwargs.iteritems())
924+ config.push(name, "\n[%s]\n%s\n" % (section, body))
925+ self.addCleanup(config.pop, name)
926+
927+ def _createTestFile(self, name, content, hash):
928+ path = os.path.join(self.test_files_dir, name)
929+ with open(path, 'wb') as fp:
930+ fp.write(content)
931+ self.slave.valid_files[hash] = path
932+
933+ def setUp(self):
934+ super(TestHandleStatusForOCIRecipeBuild, self).setUp()
935+ self.factory = LaunchpadObjectFactory()
936+ self.build = self.makeBuild()
937+ # For the moment, we require a builder for the build so that
938+ # handleStatus_OK can get a reference to the slave.
939+ self.builder = self.factory.makeBuilder()
940+ self.build.buildqueue_record.markAsBuilding(self.builder)
941+ self.slave = WaitingSlave('BuildStatus.OK')
942+ self.slave.valid_files['test_file_hash'] = ''
943+ self.interactor = BuilderInteractor()
944+ self.behaviour = self.interactor.getBuildBehaviour(
945+ self.build.buildqueue_record, self.builder, self.slave)
946+
947+ # We overwrite the buildmaster root to use a temp directory.
948+ tempdir = tempfile.mkdtemp()
949+ self.addCleanup(shutil.rmtree, tempdir)
950+ self.upload_root = tempdir
951+ self.pushConfig('builddmaster', root=self.upload_root)
952+
953+ # We stub out our build's getUploaderCommand() method so
954+ # we can check whether it was called as well as
955+ # verifySuccessfulUpload().
956+ removeSecurityProxy(self.build).verifySuccessfulUpload = FakeMethod(
957+ result=True)
958+
959+ digests = [{
960+ "diff_id_1": {
961+ "digest": "digest_1",
962+ "source": "test/base_1",
963+ "layer_id": "layer_1"
964+ },
965+ "diff_id_2": {
966+ "digest": "digest_2",
967+ "source": "",
968+ "layer_id": "layer_2"
969+ }
970+ }]
971+
972+ self.test_files_dir = tempfile.mkdtemp()
973+ self._createTestFile('buildlog', '', 'buildlog')
974+ self._createTestFile(
975+ 'manifest.json',
976+ '[{"Config": "config_file_1.json", '
977+ '"Layers": ["layer_1/layer.tar", "layer_2/layer.tar"]}]',
978+ 'manifest_hash')
979+ self._createTestFile(
980+ 'digests.json',
981+ json.dumps(digests),
982+ 'digests_hash')
983+ self._createTestFile(
984+ 'config_file_1.json',
985+ '{"rootfs": {"diff_ids": ["diff_id_1", "diff_id_2"]}}',
986+ 'config_1_hash')
987+ self._createTestFile(
988+ 'layer_2.tar.gz',
989+ '',
990+ 'layer_2_hash'
991+ )
992+
993+ self.filemap = {
994+ 'manifest.json': 'manifest_hash',
995+ 'digests.json': 'digests_hash',
996+ 'config_file_1.json': 'config_1_hash',
997+ 'layer_1.tar.gz': 'layer_1_hash',
998+ 'layer_2.tar.gz': 'layer_2_hash'
999+ }
1000+ self.factory.makeOCIFile(
1001+ build=self.build, layer_file_digest=u'digest_1',
1002+ content="retrieved from librarian")
1003+
1004+ def assertResultCount(self, count, result):
1005+ self.assertEqual(
1006+ 1, len(os.listdir(os.path.join(self.upload_root, result))))
1007+
1008+ @defer.inlineCallbacks
1009+ def test_handleStatus_OK_normal_image(self):
1010+ with dbuser(config.builddmaster.dbuser):
1011+ yield self.behaviour.handleStatus(
1012+ self.build.buildqueue_record, 'OK',
1013+ {'filemap': self.filemap})
1014+ self.assertEqual(
1015+ ['buildlog', 'manifest_hash', 'digests_hash', 'config_1_hash',
1016+ 'layer_2_hash'],
1017+ self.slave._got_file_record)
1018+ # This hash should not appear as it is already in the librarian
1019+ self.assertNotIn('layer_1_hash', self.slave._got_file_record)
1020+ self.assertEqual(BuildStatus.UPLOADING, self.build.status)
1021+ self.assertResultCount(1, "incoming")
1022+
1023+ # layer_1 should have been retrieved from the librarian
1024+ layer_1_path = os.path.join(
1025+ self.upload_root,
1026+ "incoming",
1027+ self.behaviour.getUploadDirLeaf(self.build.build_cookie),
1028+ str(self.build.archive.id),
1029+ self.build.distribution.name,
1030+ "layer_1.tar.gz"
1031+ )
1032+ with open(layer_1_path, 'rb') as layer_1_fp:
1033+ contents = layer_1_fp.read()
1034+ self.assertEqual(contents, b'retrieved from librarian')
1035+
1036+ @defer.inlineCallbacks
1037+ def test_handleStatus_OK_absolute_filepath(self):
1038+
1039+ self._createTestFile(
1040+ 'manifest.json',
1041+ '[{"Config": "/notvalid/config_file_1.json", '
1042+ '"Layers": ["layer_1/layer.tar", "layer_2/layer.tar"]}]',
1043+ 'manifest_hash')
1044+
1045+ self.filemap['/notvalid/config_file_1.json'] = 'config_1_hash'
1046+
1047+ # A filemap that tries to write to files outside of the upload
1048+ # directory will not be collected.
1049+ with ExpectedException(
1050+ BuildDaemonError,
1051+ "Build returned a file named "
1052+ "'/notvalid/config_file_1.json'."):
1053+ with dbuser(config.builddmaster.dbuser):
1054+ yield self.behaviour.handleStatus(
1055+ self.build.buildqueue_record, 'OK',
1056+ {'filemap': self.filemap})
1057+
1058+ @defer.inlineCallbacks
1059+ def test_handleStatus_OK_relative_filepath(self):
1060+
1061+ self._createTestFile(
1062+ 'manifest.json',
1063+ '[{"Config": "../config_file_1.json", '
1064+ '"Layers": ["layer_1/layer.tar", "layer_2/layer.tar"]}]',
1065+ 'manifest_hash')
1066+
1067+ self.filemap['../config_file_1.json'] = 'config_1_hash'
1068+ # A filemap that tries to write to files outside of
1069+ # the upload directory will not be collected.
1070+ with ExpectedException(
1071+ BuildDaemonError,
1072+ "Build returned a file named '../config_file_1.json'."):
1073+ with dbuser(config.builddmaster.dbuser):
1074+ yield self.behaviour.handleStatus(
1075+ self.build.buildqueue_record, 'OK',
1076+ {'filemap': self.filemap})
1077+
1078+ @defer.inlineCallbacks
1079+ def test_handleStatus_OK_sets_build_log(self):
1080+ # The build log is set during handleStatus.
1081+ self.assertEqual(None, self.build.log)
1082+ with dbuser(config.builddmaster.dbuser):
1083+ yield self.behaviour.handleStatus(
1084+ self.build.buildqueue_record, 'OK',
1085+ {'filemap': self.filemap})
1086+ self.assertNotEqual(None, self.build.log)
1087+
1088+ @defer.inlineCallbacks
1089+ def test_handleStatus_ABORTED_cancels_cancelling(self):
1090+ with dbuser(config.builddmaster.dbuser):
1091+ self.build.updateStatus(BuildStatus.CANCELLING)
1092+ yield self.behaviour.handleStatus(
1093+ self.build.buildqueue_record, "ABORTED", {})
1094+ self.assertEqual(0, len(pop_notifications()), "Notifications received")
1095+ self.assertEqual(BuildStatus.CANCELLED, self.build.status)
1096+
1097+ @defer.inlineCallbacks
1098+ def test_handleStatus_ABORTED_illegal_when_building(self):
1099+ self.builder.vm_host = "fake_vm_host"
1100+ self.behaviour = self.interactor.getBuildBehaviour(
1101+ self.build.buildqueue_record, self.builder, self.slave)
1102+ with dbuser(config.builddmaster.dbuser):
1103+ self.build.updateStatus(BuildStatus.BUILDING)
1104+ with ExpectedException(
1105+ BuildDaemonError,
1106+ "Build returned unexpected status: u'ABORTED'"):
1107+ yield self.behaviour.handleStatus(
1108+ self.build.buildqueue_record, "ABORTED", {})
1109+
1110+ @defer.inlineCallbacks
1111+ def test_handleStatus_ABORTED_cancelling_sets_build_log(self):
1112+ # If a build is intentionally cancelled, the build log is set.
1113+ self.assertEqual(None, self.build.log)
1114+ with dbuser(config.builddmaster.dbuser):
1115+ self.build.updateStatus(BuildStatus.CANCELLING)
1116+ yield self.behaviour.handleStatus(
1117+ self.build.buildqueue_record, "ABORTED", {})
1118+ self.assertNotEqual(None, self.build.log)
1119+
1120+ @defer.inlineCallbacks
1121+ def test_date_finished_set(self):
1122+ # The date finished is updated during handleStatus_OK.
1123+ self.assertEqual(None, self.build.date_finished)
1124+ with dbuser(config.builddmaster.dbuser):
1125+ yield self.behaviour.handleStatus(
1126+ self.build.buildqueue_record, 'OK',
1127+ {'filemap': self.filemap})
1128+ self.assertNotEqual(None, self.build.date_finished)
1129+
1130+ @defer.inlineCallbacks
1131+ def test_givenback_collection(self):
1132+ with ExpectedException(
1133+ BuildDaemonError,
1134+ "Build returned unexpected status: u'GIVENBACK'"):
1135+ with dbuser(config.builddmaster.dbuser):
1136+ yield self.behaviour.handleStatus(
1137+ self.build.buildqueue_record, "GIVENBACK", {})
1138+
1139+ @defer.inlineCallbacks
1140+ def test_builderfail_collection(self):
1141+ with ExpectedException(
1142+ BuildDaemonError,
1143+ "Build returned unexpected status: u'BUILDERFAIL'"):
1144+ with dbuser(config.builddmaster.dbuser):
1145+ yield self.behaviour.handleStatus(
1146+ self.build.buildqueue_record, "BUILDERFAIL", {})
1147+
1148+ @defer.inlineCallbacks
1149+ def test_invalid_status_collection(self):
1150+ with ExpectedException(
1151+ BuildDaemonError,
1152+ "Build returned unexpected status: u'BORKED'"):
1153+ with dbuser(config.builddmaster.dbuser):
1154+ yield self.behaviour.handleStatus(
1155+ self.build.buildqueue_record, "BORKED", {})
1156+
1157+
1158+class TestGetUploadMethodsForOCIRecipeBuild(
1159+ MakeOCIBuildMixin, TestGetUploadMethodsMixin, TrialTestCase):
1160+ """IPackageBuild.getUpload-related methods work with OCI recipe builds."""
1161+
1162+ def setUp(self):
1163+ super(TestGetUploadMethodsForOCIRecipeBuild, self).__init__(self)
1164+ self.factory = LaunchpadObjectFactory()
1165diff --git a/lib/lp/snappy/model/snapbuildbehaviour.py b/lib/lp/snappy/model/snapbuildbehaviour.py
1166index cda3cff..cbcba1d 100644
1167--- a/lib/lp/snappy/model/snapbuildbehaviour.py
1168+++ b/lib/lp/snappy/model/snapbuildbehaviour.py
1169@@ -11,6 +11,7 @@ from __future__ import absolute_import, print_function, unicode_literals
1170 __metaclass__ = type
1171 __all__ = [
1172 'SnapBuildBehaviour',
1173+ 'SnapProxyMixin',
1174 ]
1175
1176 import base64
1177@@ -61,9 +62,58 @@ def format_as_rfc3339(timestamp):
1178 return timestamp.replace(microsecond=0, tzinfo=None).isoformat() + 'Z'
1179
1180
1181+class SnapProxyMixin:
1182+ """Methods for handling builds with the Snap Build Proxy enabled."""
1183+
1184+ @defer.inlineCallbacks
1185+ def addProxyArgs(self, args, allow_internet=True):
1186+ if config.snappy.builder_proxy_host and allow_internet:
1187+ token = yield self._requestProxyToken()
1188+ args["proxy_url"] = (
1189+ "http://{username}:{password}@{host}:{port}".format(
1190+ username=token['username'],
1191+ password=token['secret'],
1192+ host=config.snappy.builder_proxy_host,
1193+ port=config.snappy.builder_proxy_port))
1194+ args["revocation_endpoint"] = (
1195+ "{endpoint}/{token}".format(
1196+ endpoint=config.snappy.builder_proxy_auth_api_endpoint,
1197+ token=token['username']))
1198+
1199+ @defer.inlineCallbacks
1200+ def _requestProxyToken(self):
1201+ admin_username = config.snappy.builder_proxy_auth_api_admin_username
1202+ if not admin_username:
1203+ raise CannotBuild(
1204+ "builder_proxy_auth_api_admin_username is not configured.")
1205+ secret = config.snappy.builder_proxy_auth_api_admin_secret
1206+ if not secret:
1207+ raise CannotBuild(
1208+ "builder_proxy_auth_api_admin_secret is not configured.")
1209+ url = config.snappy.builder_proxy_auth_api_endpoint
1210+ if not secret:
1211+ raise CannotBuild(
1212+ "builder_proxy_auth_api_endpoint is not configured.")
1213+ timestamp = int(time.time())
1214+ proxy_username = '{build_id}-{timestamp}'.format(
1215+ build_id=self.build.build_cookie,
1216+ timestamp=timestamp)
1217+ auth_string = '{}:{}'.format(admin_username, secret).strip()
1218+ auth_header = b'Basic ' + base64.b64encode(auth_string)
1219+
1220+ response = yield treq.post(
1221+ url, headers={'Authorization': auth_header},
1222+ json={'username': proxy_username},
1223+ reactor=self._slave.reactor,
1224+ pool=self._slave.pool)
1225+ response = yield check_status(response)
1226+ token = yield treq.json_content(response)
1227+ defer.returnValue(token)
1228+
1229+
1230 @adapter(ISnapBuild)
1231 @implementer(IBuildFarmJobBehaviour)
1232-class SnapBuildBehaviour(BuildFarmJobBehaviourBase):
1233+class SnapBuildBehaviour(SnapProxyMixin, BuildFarmJobBehaviourBase):
1234 """Dispatches `SnapBuild` jobs to slaves."""
1235
1236 builder_type = "snap"
1237@@ -111,18 +161,7 @@ class SnapBuildBehaviour(BuildFarmJobBehaviourBase):
1238 build = self.build
1239 args = yield super(SnapBuildBehaviour, self).extraBuildArgs(
1240 logger=logger)
1241- if config.snappy.builder_proxy_host and build.snap.allow_internet:
1242- token = yield self._requestProxyToken()
1243- args["proxy_url"] = (
1244- "http://{username}:{password}@{host}:{port}".format(
1245- username=token['username'],
1246- password=token['secret'],
1247- host=config.snappy.builder_proxy_host,
1248- port=config.snappy.builder_proxy_port))
1249- args["revocation_endpoint"] = (
1250- "{endpoint}/{token}".format(
1251- endpoint=config.snappy.builder_proxy_auth_api_endpoint,
1252- token=token['username']))
1253+ yield self.addProxyArgs(args, build.snap.allow_internet)
1254 args["name"] = build.snap.store_name or build.snap.name
1255 channels = build.channels or {}
1256 if "snapcraft" not in channels:
1257@@ -188,36 +227,6 @@ class SnapBuildBehaviour(BuildFarmJobBehaviourBase):
1258 args["build_request_timestamp"] = timestamp
1259 defer.returnValue(args)
1260
1261- @defer.inlineCallbacks
1262- def _requestProxyToken(self):
1263- admin_username = config.snappy.builder_proxy_auth_api_admin_username
1264- if not admin_username:
1265- raise CannotBuild(
1266- "builder_proxy_auth_api_admin_username is not configured.")
1267- secret = config.snappy.builder_proxy_auth_api_admin_secret
1268- if not secret:
1269- raise CannotBuild(
1270- "builder_proxy_auth_api_admin_secret is not configured.")
1271- url = config.snappy.builder_proxy_auth_api_endpoint
1272- if not secret:
1273- raise CannotBuild(
1274- "builder_proxy_auth_api_endpoint is not configured.")
1275- timestamp = int(time.time())
1276- proxy_username = '{build_id}-{timestamp}'.format(
1277- build_id=self.build.build_cookie,
1278- timestamp=timestamp)
1279- auth_string = '{}:{}'.format(admin_username, secret).strip()
1280- auth_header = b'Basic ' + base64.b64encode(auth_string)
1281-
1282- response = yield treq.post(
1283- url, headers={'Authorization': auth_header},
1284- json={'username': proxy_username},
1285- reactor=self._slave.reactor,
1286- pool=self._slave.pool)
1287- response = yield check_status(response)
1288- token = yield treq.json_content(response)
1289- defer.returnValue(token)
1290-
1291 def verifySuccessfulBuild(self):
1292 """See `IBuildFarmJobBehaviour`."""
1293 # The implementation in BuildFarmJobBehaviourBase checks whether the
1294diff --git a/lib/lp/snappy/tests/test_snapbuildbehaviour.py b/lib/lp/snappy/tests/test_snapbuildbehaviour.py
1295index 2755523..f851008 100644
1296--- a/lib/lp/snappy/tests/test_snapbuildbehaviour.py
1297+++ b/lib/lp/snappy/tests/test_snapbuildbehaviour.py
1298@@ -24,7 +24,6 @@ from testtools.matchers import (
1299 AfterPreprocessing,
1300 ContainsDict,
1301 Equals,
1302- HasLength,
1303 Is,
1304 IsInstance,
1305 MatchesDict,
1306@@ -38,13 +37,10 @@ from testtools.twistedsupport import (
1307 import transaction
1308 from twisted.internet import (
1309 defer,
1310- endpoints,
1311 reactor,
1312 )
1313-from twisted.python.compat import nativeString
1314 from twisted.trial.unittest import TestCase as TrialTestCase
1315 from twisted.web import (
1316- resource,
1317 server,
1318 xmlrpc,
1319 )
1320@@ -71,6 +67,11 @@ from lp.buildmaster.tests.mock_slaves import (
1321 OkSlave,
1322 SlaveTestHelpers,
1323 )
1324+from lp.buildmaster.tests.snapbuildproxy import (
1325+ InProcessProxyAuthAPIFixture,
1326+ ProxyURLMatcher,
1327+ RevocationEndpointMatcher,
1328+ )
1329 from lp.buildmaster.tests.test_buildfarmjobbehaviour import (
1330 TestGetUploadMethodsMixin,
1331 TestHandleStatusMixin,
1332@@ -100,7 +101,6 @@ from lp.soyuz.adapters.archivedependencies import (
1333 )
1334 from lp.soyuz.enums import PackagePublishingStatus
1335 from lp.soyuz.interfaces.archive import ArchiveDisabled
1336-from lp.soyuz.interfaces.component import IComponentSet
1337 from lp.soyuz.tests.soyuz import Base64KeyMatches
1338 from lp.testing import (
1339 TestCase,
1340@@ -116,69 +116,6 @@ from lp.testing.layers import LaunchpadZopelessLayer
1341 from lp.xmlrpc.interfaces import IPrivateApplication
1342
1343
1344-class ProxyAuthAPITokensResource(resource.Resource):
1345- """A test tokens resource for the proxy authentication API."""
1346-
1347- isLeaf = True
1348-
1349- def __init__(self):
1350- resource.Resource.__init__(self)
1351- self.requests = []
1352-
1353- def render_POST(self, request):
1354- content = request.content.read()
1355- self.requests.append({
1356- "method": request.method,
1357- "uri": request.uri,
1358- "headers": dict(request.requestHeaders.getAllRawHeaders()),
1359- "content": content,
1360- })
1361- username = json.loads(content)["username"]
1362- return json.dumps({
1363- "username": username,
1364- "secret": uuid.uuid4().hex,
1365- "timestamp": datetime.utcnow().isoformat(),
1366- })
1367-
1368-
1369-class InProcessProxyAuthAPIFixture(fixtures.Fixture):
1370- """A fixture that pretends to be the proxy authentication API.
1371-
1372- Users of this fixture must call the `start` method, which returns a
1373- `Deferred`, and arrange for that to get back to the reactor. This is
1374- necessary because the basic fixture API does not allow `setUp` to return
1375- anything. For example:
1376-
1377- class TestSomething(TestCase):
1378-
1379- run_tests_with = AsynchronousDeferredRunTest.make_factory(
1380- timeout=10)
1381-
1382- @defer.inlineCallbacks
1383- def setUp(self):
1384- super(TestSomething, self).setUp()
1385- yield self.useFixture(InProcessProxyAuthAPIFixture()).start()
1386- """
1387-
1388- @defer.inlineCallbacks
1389- def start(self):
1390- root = resource.Resource()
1391- self.tokens = ProxyAuthAPITokensResource()
1392- root.putChild("tokens", self.tokens)
1393- endpoint = endpoints.serverFromString(reactor, nativeString("tcp:0"))
1394- site = server.Site(self.tokens)
1395- self.addCleanup(site.stopFactory)
1396- port = yield endpoint.listen(site)
1397- self.addCleanup(port.stopListening)
1398- config.push("in-process-proxy-auth-api-fixture", dedent("""
1399- [snappy]
1400- builder_proxy_auth_api_admin_secret: admin-secret
1401- builder_proxy_auth_api_endpoint: http://%s:%s/tokens
1402- """) %
1403- (port.getHost().host, port.getHost().port))
1404- self.addCleanup(config.pop, "in-process-proxy-auth-api-fixture")
1405-
1406-
1407 class InProcessAuthServer(xmlrpc.XMLRPC):
1408
1409 def __init__(self, *args, **kwargs):
1410@@ -380,21 +317,6 @@ class TestAsyncSnapBuildBehaviour(TestSnapBuildBehaviourBase):
1411 self.addCleanup(slave.pool.closeCachedConnections)
1412 return job
1413
1414- def getProxyURLMatcher(self, job):
1415- return AfterPreprocessing(urlsplit, MatchesStructure(
1416- scheme=Equals("http"),
1417- username=Equals("{}-{}".format(
1418- job.build.build_cookie, int(self.now))),
1419- password=HasLength(32),
1420- hostname=Equals(config.snappy.builder_proxy_host),
1421- port=Equals(config.snappy.builder_proxy_port),
1422- path=Equals("")))
1423-
1424- def getRevocationEndpointMatcher(self, job):
1425- return Equals("{}/{}-{}".format(
1426- config.snappy.builder_proxy_auth_api_endpoint,
1427- job.build.build_cookie, int(self.now)))
1428-
1429 @defer.inlineCallbacks
1430 def test_composeBuildRequest(self):
1431 job = self.makeJob()
1432@@ -466,8 +388,8 @@ class TestAsyncSnapBuildBehaviour(TestSnapBuildBehaviourBase):
1433 "fast_cleanup": Is(True),
1434 "name": Equals("test-snap"),
1435 "private": Is(False),
1436- "proxy_url": self.getProxyURLMatcher(job),
1437- "revocation_endpoint": self.getRevocationEndpointMatcher(job),
1438+ "proxy_url": ProxyURLMatcher(job, self.now),
1439+ "revocation_endpoint": RevocationEndpointMatcher(job, self.now),
1440 "series": Equals("unstable"),
1441 "trusted_keys": Equals(expected_trusted_keys),
1442 }))
1443@@ -507,8 +429,8 @@ class TestAsyncSnapBuildBehaviour(TestSnapBuildBehaviourBase):
1444 "git_path": Equals(ref.name),
1445 "name": Equals("test-snap"),
1446 "private": Is(False),
1447- "proxy_url": self.getProxyURLMatcher(job),
1448- "revocation_endpoint": self.getRevocationEndpointMatcher(job),
1449+ "proxy_url": ProxyURLMatcher(job, self.now),
1450+ "revocation_endpoint": RevocationEndpointMatcher(job, self.now),
1451 "series": Equals("unstable"),
1452 "trusted_keys": Equals(expected_trusted_keys),
1453 }))
1454@@ -537,8 +459,8 @@ class TestAsyncSnapBuildBehaviour(TestSnapBuildBehaviourBase):
1455 "git_repository": Equals(ref.repository.git_https_url),
1456 "name": Equals("test-snap"),
1457 "private": Is(False),
1458- "proxy_url": self.getProxyURLMatcher(job),
1459- "revocation_endpoint": self.getRevocationEndpointMatcher(job),
1460+ "proxy_url": ProxyURLMatcher(job, self.now),
1461+ "revocation_endpoint": RevocationEndpointMatcher(job, self.now),
1462 "series": Equals("unstable"),
1463 "trusted_keys": Equals(expected_trusted_keys),
1464 }))
1465@@ -584,8 +506,8 @@ class TestAsyncSnapBuildBehaviour(TestSnapBuildBehaviourBase):
1466 "git_path": Equals(ref.name),
1467 "name": Equals("test-snap"),
1468 "private": Is(True),
1469- "proxy_url": self.getProxyURLMatcher(job),
1470- "revocation_endpoint": self.getRevocationEndpointMatcher(job),
1471+ "proxy_url": ProxyURLMatcher(job, self.now),
1472+ "revocation_endpoint": RevocationEndpointMatcher(job, self.now),
1473 "series": Equals("unstable"),
1474 "trusted_keys": Equals(expected_trusted_keys),
1475 }))
1476@@ -616,8 +538,8 @@ class TestAsyncSnapBuildBehaviour(TestSnapBuildBehaviourBase):
1477 "git_path": Equals("master"),
1478 "name": Equals("test-snap"),
1479 "private": Is(False),
1480- "proxy_url": self.getProxyURLMatcher(job),
1481- "revocation_endpoint": self.getRevocationEndpointMatcher(job),
1482+ "proxy_url": ProxyURLMatcher(job, self.now),
1483+ "revocation_endpoint": RevocationEndpointMatcher(job, self.now),
1484 "series": Equals("unstable"),
1485 "trusted_keys": Equals(expected_trusted_keys),
1486 }))
1487@@ -646,8 +568,8 @@ class TestAsyncSnapBuildBehaviour(TestSnapBuildBehaviourBase):
1488 "git_repository": Equals(url),
1489 "name": Equals("test-snap"),
1490 "private": Is(False),
1491- "proxy_url": self.getProxyURLMatcher(job),
1492- "revocation_endpoint": self.getRevocationEndpointMatcher(job),
1493+ "proxy_url": ProxyURLMatcher(job, self.now),
1494+ "revocation_endpoint": RevocationEndpointMatcher(job, self.now),
1495 "series": Equals("unstable"),
1496 "trusted_keys": Equals(expected_trusted_keys),
1497 }))
1498@@ -808,7 +730,7 @@ class TestAsyncSnapBuildBehaviour(TestSnapBuildBehaviourBase):
1499 job = self.makeJob()
1500 build_request = yield job.composeBuildRequest(None)
1501 self.assertThat(
1502- build_request[4]["proxy_url"], self.getProxyURLMatcher(job))
1503+ build_request[4]["proxy_url"], ProxyURLMatcher(job, self.now))
1504
1505 @defer.inlineCallbacks
1506 def test_composeBuildRequest_deleted(self):
1507diff --git a/lib/lp/testing/factory.py b/lib/lp/testing/factory.py
1508index 85a05b6..c402c4c 100644
1509--- a/lib/lp/testing/factory.py
1510+++ b/lib/lp/testing/factory.py
1511@@ -4991,9 +4991,15 @@ class BareLaunchpadObjectFactory(ObjectFactory):
1512 if requester is None:
1513 requester = self.makePerson()
1514 if distro_arch_series is None:
1515- distro_arch_series = self.makeDistroArchSeries()
1516+ distroseries = self.makeDistroSeries(status=SeriesStatus.CURRENT)
1517+ processor = getUtility(IProcessorSet).getByName("386")
1518+ distro_arch_series = self.makeDistroArchSeries(
1519+ distroseries=distroseries, architecturetag="i386",
1520+ processor=processor)
1521 if recipe is None:
1522- recipe = self.makeOCIRecipe()
1523+ oci_project = self.makeOCIProject(
1524+ pillar=distro_arch_series.distroseries.distribution)
1525+ recipe = self.makeOCIRecipe(oci_project=oci_project)
1526 oci_build = getUtility(IOCIRecipeBuildSet).new(
1527 requester, recipe, distro_arch_series, date_created)
1528 if duration is not None:
1529@@ -5010,12 +5016,13 @@ class BareLaunchpadObjectFactory(ObjectFactory):
1530 return oci_build
1531
1532 def makeOCIFile(self, build=None, library_file=None,
1533- layer_file_digest=None):
1534+ layer_file_digest=None, content=None, filename=None):
1535 """Make a new OCIFile."""
1536 if build is None:
1537 build = self.makeOCIRecipeBuild()
1538 if library_file is None:
1539- library_file = self.makeLibraryFileAlias()
1540+ library_file = self.makeLibraryFileAlias(
1541+ content=content, filename=filename)
1542 return OCIFile(build=build, library_file=library_file,
1543 layer_file_digest=layer_file_digest)
1544
1545diff --git a/lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py b/lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py
1546index b387251..32dd553 100644
1547--- a/lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py
1548+++ b/lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py
1549@@ -138,7 +138,7 @@ class TestTranslationTemplatesBuildBehaviour(
1550 buildqueue = FakeBuildQueue(behaviour)
1551 path = behaviour.templates_tarball_path
1552 # Poke the file we're expecting into the mock slave.
1553- behaviour._slave.valid_file_hashes.append(path)
1554+ behaviour._slave.valid_files[path] = ''
1555
1556 def got_tarball(filename):
1557 tarball = open(filename, 'r')

Subscribers

People subscribed via source and target branches

to status/vote changes: