Merge ~twom/launchpad:oci-buildbehaviour into launchpad:master

Proposed by Tom Wardill on 2019-12-12
Status: Superseded
Proposed branch: ~twom/launchpad:oci-buildbehaviour
Merge into: launchpad:master
Prerequisite: ~twom/launchpad:oci-buildjob
Diff against target: 877 lines (+473/-79)
11 files modified
dev/null (+0/-49)
lib/lp/buildmaster/model/buildfarmjobbehaviour.py (+17/-13)
lib/lp/buildmaster/tests/mock_slaves.py (+9/-3)
lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py (+1/-1)
lib/lp/oci/configure.zcml (+6/-0)
lib/lp/oci/model/ocirecipe.py (+0/-10)
lib/lp/oci/model/ocirecipebuild.py (+9/-0)
lib/lp/oci/model/ocirecipebuildbehaviour.py (+115/-0)
lib/lp/oci/tests/test_ocirecipebuildbehaviour.py (+312/-0)
lib/lp/testing/factory.py (+3/-2)
lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py (+1/-1)
Reviewer Review Type Date Requested Status
Colin Watson 2019-12-12 Needs Fixing on 2019-12-12
Review via email: mp+376678@code.launchpad.net

This proposal has been superseded by a proposal from 2020-02-14.

Commit message

Add OCIRecipeBuildBehaviour

Description of the change

Add a buildbehaviour to drive OCI image builds.
Use mixin tests where sensible, override for places where the OCI methods differ.

To post a comment you must log in.
Colin Watson (cjwatson) :
review: Needs Fixing
Tom Wardill (twom) :
Colin Watson (cjwatson) :
~twom/launchpad:oci-buildbehaviour updated on 2020-02-14
d964f9c... by Tom Wardill on 2019-12-13

Refactor and tidy up in ocirecipebuildbehaviour

e58dcda... by Tom Wardill on 2019-12-13

Test file cleanups and config handling

5692697... by Tom Wardill on 2019-12-16

Remove WaitingSlaveWithFiles in favour of changes to WaitingSlave

7e2a207... by Tom Wardill on 2020-02-14

Fix merge oops

Unmerged commits

7e2a207... by Tom Wardill on 2020-02-14

Fix merge oops

5692697... by Tom Wardill on 2019-12-16

Remove WaitingSlaveWithFiles in favour of changes to WaitingSlave

e58dcda... by Tom Wardill on 2019-12-13

Test file cleanups and config handling

d964f9c... by Tom Wardill on 2019-12-13

Refactor and tidy up in ocirecipebuildbehaviour

7a9ce0c... by Tom Wardill on 2019-12-12

Add tests for ocirecipebuildbehaviour

af66ca2... by Tom Wardill on 2019-12-10

Add mock slave with files

09d0fdc... by Tom Wardill on 2019-12-10

Allow downloadfiles to be overriden

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/lib/lp/buildmaster/model/buildfarmjobbehaviour.py b/lib/lp/buildmaster/model/buildfarmjobbehaviour.py
2index 2f3515a..4b28693 100644
3--- a/lib/lp/buildmaster/model/buildfarmjobbehaviour.py
4+++ b/lib/lp/buildmaster/model/buildfarmjobbehaviour.py
5@@ -301,6 +301,22 @@ class BuildFarmJobBehaviourBase:
6 transaction.commit()
7
8 @defer.inlineCallbacks
9+ def _downloadFiles(self, filemap, upload_path, logger):
10+ filenames_to_download = []
11+ for filename, sha1 in filemap.items():
12+ logger.info("Grabbing file: %s (%s)" % (
13+ filename, self._slave.getURL(sha1)))
14+ out_file_name = os.path.join(upload_path, filename)
15+ # If the evaluated output file name is not within our
16+ # upload path, then we don't try to copy this or any
17+ # subsequent files.
18+ if not os.path.realpath(out_file_name).startswith(upload_path):
19+ raise BuildDaemonError(
20+ "Build returned a file named %r." % filename)
21+ filenames_to_download.append((sha1, out_file_name))
22+ yield self._slave.getFiles(filenames_to_download, logger=logger)
23+
24+ @defer.inlineCallbacks
25 def handleSuccess(self, slave_status, logger):
26 """Handle a package that built successfully.
27
28@@ -337,19 +353,7 @@ class BuildFarmJobBehaviourBase:
29 grab_dir, str(build.archive.id), build.distribution.name)
30 os.makedirs(upload_path)
31
32- filenames_to_download = []
33- for filename, sha1 in filemap.items():
34- logger.info("Grabbing file: %s (%s)" % (
35- filename, self._slave.getURL(sha1)))
36- out_file_name = os.path.join(upload_path, filename)
37- # If the evaluated output file name is not within our
38- # upload path, then we don't try to copy this or any
39- # subsequent files.
40- if not os.path.realpath(out_file_name).startswith(upload_path):
41- raise BuildDaemonError(
42- "Build returned a file named %r." % filename)
43- filenames_to_download.append((sha1, out_file_name))
44- yield self._slave.getFiles(filenames_to_download, logger=logger)
45+ yield self._downloadFiles(filemap, upload_path, logger)
46
47 transaction.commit()
48
49diff --git a/lib/lp/buildmaster/tests/mock_slaves.py b/lib/lp/buildmaster/tests/mock_slaves.py
50index 8b00c32..eca171a 100644
51--- a/lib/lp/buildmaster/tests/mock_slaves.py
52+++ b/lib/lp/buildmaster/tests/mock_slaves.py
53@@ -194,7 +194,8 @@ class WaitingSlave(OkSlave):
54
55 # By default, the slave only has a buildlog, but callsites
56 # can update this list as needed.
57- self.valid_file_hashes = ['buildlog']
58+ self.valid_files = {'buildlog': ''}
59+ self._got_file_record = []
60
61 def status(self):
62 self.call_log.append('status')
63@@ -208,12 +209,17 @@ class WaitingSlave(OkSlave):
64
65 def getFile(self, hash, file_to_write):
66 self.call_log.append('getFile')
67- if hash in self.valid_file_hashes:
68- content = "This is a %s" % hash
69+ if hash in self.valid_files:
70 if isinstance(file_to_write, types.StringTypes):
71 file_to_write = open(file_to_write, 'wb')
72+ if not self.valid_files[hash]:
73+ content = "This is a %s" % hash
74+ else:
75+ with open(self.valid_files[hash], 'rb') as source:
76+ content = source.read()
77 file_to_write.write(content)
78 file_to_write.close()
79+ self._got_file_record.append(hash)
80 return defer.succeed(None)
81
82
83diff --git a/lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py b/lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py
84index 697ff52..d6ecd85 100644
85--- a/lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py
86+++ b/lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py
87@@ -337,7 +337,7 @@ class TestHandleStatusMixin:
88 self.builder = self.factory.makeBuilder()
89 self.build.buildqueue_record.markAsBuilding(self.builder)
90 self.slave = WaitingSlave('BuildStatus.OK')
91- self.slave.valid_file_hashes.append('test_file_hash')
92+ self.slave.valid_file_hashes['test_file_hash'] = ''
93 self.interactor = BuilderInteractor()
94 self.behaviour = self.interactor.getBuildBehaviour(
95 self.build.buildqueue_record, self.builder, self.slave)
96diff --git a/lib/lp/oci/configure.zcml b/lib/lp/oci/configure.zcml
97index 30212b9..51e1d31 100644
98--- a/lib/lp/oci/configure.zcml
99+++ b/lib/lp/oci/configure.zcml
100@@ -56,4 +56,10 @@
101 <allow interface="lp.buildmaster.interfaces.buildfarmjob.ISpecificBuildFarmJobSource" />
102 </securedutility>
103
104+ <adapter
105+ for="lp.oci.interfaces.ocirecipebuild.IOCIRecipeBuild"
106+ provides="lp.buildmaster.interfaces.buildfarmjobbehaviour.IBuildFarmJobBehaviour"
107+ factory="lp.oci.model.ocirecipebuildbehaviour.OCIRecipeBuildBehaviour"
108+ permission="zope.Public" />
109+
110 </configure>
111diff --git a/lib/lp/oci/interfaces/ocirecipebuildjob.py b/lib/lp/oci/interfaces/ocirecipebuildjob.py
112deleted file mode 100644
113index cd72177..0000000
114--- a/lib/lp/oci/interfaces/ocirecipebuildjob.py
115+++ /dev/null
116@@ -1,38 +0,0 @@
117-# Copyright 2019 Canonical Ltd. This software is licensed under the
118-# GNU Affero General Public License version 3 (see the file LICENSE).
119-
120-"""OCIRecipe build job interfaces"""
121-
122-from __future__ import absolute_import, print_function, unicode_literals
123-
124-__metaclass__ = type
125-__all__ = [
126- 'IOCIRecipeBuildJob',
127- ]
128-
129-from lazr.restful.fields import Reference
130-from zope.interface import (
131- Attribute,
132- Interface,
133- )
134-from zope.schema import TextLine
135-
136-from lp import _
137-from lp.oci.interfaces.ocirecipebuild import IOCIRecipeBuild
138-from lp.services.job.interfaces.job import (
139- IJob,
140- IJobSource,
141- IRunnableJob,
142- )
143-
144-
145-class IOCIRecipeBuildJob(Interface):
146- job = Reference(
147- title=_("The common Job attributes."), schema=IJob,
148- required=True, readonly=True)
149-
150- build = Reference(
151- title=_("The OCI Recipe Build to use for this job."),
152- schema=IOCIRecipeBuild, required=True, readonly=True)
153-
154- json_data = Attribute(_("A dict of data about the job."))
155diff --git a/lib/lp/oci/model/ocirecipe.py b/lib/lp/oci/model/ocirecipe.py
156index 300f335..8a01b2d 100644
157--- a/lib/lp/oci/model/ocirecipe.py
158+++ b/lib/lp/oci/model/ocirecipe.py
159@@ -308,15 +308,5 @@ class OCIRecipeSet:
160 person_ids.add(recipe.registrant_id)
161 person_ids.add(recipe.owner_id)
162
163- repositories = load_related(
164- GitRepository, recipes, ["git_repository_id"])
165- if repositories:
166- GenericGitCollection.preloadDataForRepositories(repositories)
167- GenericGitCollection.preloadVisibleRepositories(repositories, user)
168-
169- # We need the target repository owner as well; unlike branches,
170- # repository unique names aren't trigger-maintained.
171- person_ids.update(repository.owner_id for repository in repositories)
172-
173 list(getUtility(IPersonSet).getPrecachedPersonsFromIDs(
174 person_ids, need_validity=True))
175diff --git a/lib/lp/oci/model/ocirecipebuild.py b/lib/lp/oci/model/ocirecipebuild.py
176index 307fbd2..5d2d21d 100644
177--- a/lib/lp/oci/model/ocirecipebuild.py
178+++ b/lib/lp/oci/model/ocirecipebuild.py
179@@ -44,6 +44,7 @@ from lp.oci.interfaces.ocirecipebuild import (
180 IOCIRecipeBuildSet,
181 )
182 from lp.registry.model.person import Person
183+from lp.services.config import config
184 from lp.services.database.bulk import load_related
185 from lp.services.database.constants import DEFAULT
186 from lp.services.database.decoratedresultset import DecoratedResultSet
187@@ -208,6 +209,14 @@ class OCIRecipeBuild(PackageBuildMixin, Storm):
188 # pillar isn't just a distribution
189 return self.recipe.oci_project.distribution
190
191+ def notify(self, extra_info=None):
192+ """See `IPackageBuild`."""
193+ if not config.builddmaster.send_build_notification:
194+ return
195+ if self.status == BuildStatus.FULLYBUILT:
196+ return
197+ # XXX twom 2019-12-11 This should send mail
198+
199
200 @implementer(IOCIRecipeBuildSet)
201 class OCIRecipeBuildSet(SpecificBuildFarmJobSourceMixin):
202diff --git a/lib/lp/oci/model/ocirecipebuildbehaviour.py b/lib/lp/oci/model/ocirecipebuildbehaviour.py
203new file mode 100644
204index 0000000..e1a1215
205--- /dev/null
206+++ b/lib/lp/oci/model/ocirecipebuildbehaviour.py
207@@ -0,0 +1,115 @@
208+# Copyright 2019 Canonical Ltd. This software is licensed under the
209+# GNU Affero General Public License version 3 (see the file LICENSE).
210+
211+"""An `IBuildFarmJobBehaviour` for `OCIRecipeBuild`.
212+
213+Dispatches OCI image build jobs to build-farm slaves.
214+"""
215+
216+from __future__ import absolute_import, print_function, unicode_literals
217+
218+__metaclass__ = type
219+__all__ = [
220+ 'OCIRecipeBuildBehaviour',
221+ ]
222+
223+
224+import json
225+import os
226+
227+from twisted.internet import defer
228+from zope.interface import implementer
229+
230+from lp.app.errors import NotFoundError
231+from lp.buildmaster.enums import BuildBaseImageType
232+from lp.buildmaster.interfaces.builder import BuildDaemonError
233+from lp.buildmaster.interfaces.buildfarmjobbehaviour import (
234+ IBuildFarmJobBehaviour,
235+ )
236+from lp.buildmaster.model.buildfarmjobbehaviour import (
237+ BuildFarmJobBehaviourBase,
238+ )
239+from lp.services.librarian.utils import copy_and_close
240+
241+
242+@implementer(IBuildFarmJobBehaviour)
243+class OCIRecipeBuildBehaviour(BuildFarmJobBehaviourBase):
244+
245+ builder_type = "oci"
246+ image_types = [BuildBaseImageType.LXD]
247+
248+ # These attributes are defined in `IOCIBuildFarmJobBehaviour`,
249+ # but are not used in this implementation.
250+ distro_arch_series = None
251+
252+ def _ensureFilePath(self, file_name, file_path, upload_path):
253+ # If the evaluated output file name is not within our
254+ # upload path, then we don't try to copy this or any
255+ # subsequent files.
256+ if not os.path.normpath(file_path).startswith(upload_path):
257+ raise BuildDaemonError(
258+ "Build returned a file named '%s'." % file_name)
259+
260+ def _fetchIntermediaryFile(self, name, filemap, upload_path):
261+ file_hash = filemap[name]
262+ file_path = os.path.join(upload_path, name)
263+ self._ensureFilePath(name, file_path, upload_path)
264+ self._slave.getFile(file_hash, file_path)
265+
266+ with open(file_path, 'r') as file_fp:
267+ contents = json.load(file_fp)
268+ return contents
269+
270+ def _extractLayerFiles(self, upload_path, section, config, digests, files):
271+ # These are different sets of ids, in the same order
272+ # layer_id is the filename, diff_id is the internal (docker) id
273+ for diff_id in config['rootfs']['diff_ids']:
274+ layer_id = digests[diff_id]['layer_id']
275+ # This is in the form '<id>/layer.tar', we only need the first
276+ layer_filename = "{}.tar.gz".format(layer_id.split('/')[0])
277+ digest = digests[diff_id]['digest']
278+ try:
279+ _, librarian_layer_file, _ = self.build.getLayerFileByDigest(
280+ digest)
281+ except NotFoundError:
282+ files.add(layer_filename)
283+ continue
284+ layer_path = os.path.join(upload_path, layer_filename)
285+ librarian_layer_file.open()
286+ with open(layer_path, 'wb') as layer_fp:
287+ copy_and_close(librarian_layer_file, layer_fp)
288+
289+ def _convertToRetrievableFile(self, upload_path, file_name, filemap):
290+ file_path = os.path.join(upload_path, file_name)
291+ self._ensureFilePath(file_name, file_path, upload_path)
292+ return (filemap[file_name], file_path)
293+
294+ @defer.inlineCallbacks
295+ def _downloadFiles(self, filemap, upload_path, logger):
296+ """Download required artifact files."""
297+ # We don't want to download all of the files that have been created,
298+ # just the ones that are mentioned in the manifest and config.
299+
300+ manifest = self._fetchIntermediaryFile(
301+ 'manifest.json', filemap, upload_path)
302+ digests = self._fetchIntermediaryFile(
303+ 'digests.json', filemap, upload_path)
304+
305+ files = set()
306+ for section in manifest:
307+ config = self._fetchIntermediaryFile(
308+ section['Config'], filemap, upload_path)
309+ self._extractLayerFiles(
310+ upload_path, section, config, digests, files)
311+
312+ files_to_download = [
313+ self._convertToRetrievableFile(upload_path, filename, filemap)
314+ for filename in files]
315+ yield self._slave.getFiles(files_to_download, logger=logger)
316+
317+ def verifySuccessfulBuild(self):
318+ """See `IBuildFarmJobBehaviour`."""
319+ # The implementation in BuildFarmJobBehaviourBase checks whether the
320+ # target suite is modifiable in the target archive. However, an
321+ # `OCIRecipeBuild` does not use an archive in this manner.
322+ return True
323diff --git a/lib/lp/oci/model/ocirecipebuildjob.py b/lib/lp/oci/model/ocirecipebuildjob.py
324deleted file mode 100644
325index 6df00cf..0000000
326--- a/lib/lp/oci/model/ocirecipebuildjob.py
327+++ /dev/null
328@@ -1,143 +0,0 @@
329-# Copyright 2019 Canonical Ltd. This software is licensed under the
330-# GNU Affero General Public License version 3 (see the file LICENSE).
331-
332-"""OCIRecipe build jobs."""
333-
334-from __future__ import absolute_import, print_function, unicode_literals
335-
336-__metaclass__ = type
337-__all__ = [
338- 'OCIRecipeBuildJob',
339- 'OCIRecipeBuildJobType',
340- ]
341-
342-import json
343-
344-from lazr.delegates import delegate_to
345-from lazr.enum import (
346- DBEnumeratedType,
347- DBItem,
348- )
349-from storm.locals import (
350- Int,
351- JSON,
352- Reference,
353- )
354-from zope.interface import (
355- implementer,
356- provider,
357- )
358-
359-from lp.app.errors import NotFoundError
360-from lp.oci.interfaces.ocirecipebuildjob import IOCIRecipeBuildJob
361-from lp.services.database.enumcol import DBEnum
362-from lp.services.database.interfaces import (
363- IMasterStore,
364- IStore,
365- )
366-from lp.services.database.stormbase import StormBase
367-from lp.services.job.model.job import (
368- EnumeratedSubclass,
369- Job,
370- )
371-from lp.services.job.runner import BaseRunnableJob
372-from lp.services.propertycache import get_property_cache
373-
374-
375-class OCIRecipeBuildJobType(DBEnumeratedType):
376- """Values that `OCIBuildJobType.job_type` can take."""
377-
378- REGISTRY_UPLOAD = DBItem(0, """
379- Registry upload
380-
381- This job uploads an OCI Image to registry.
382- """)
383-
384-
385-@implementer(IOCIRecipeBuildJob)
386-class OCIRecipeBuildJob(StormBase):
387- """See `IOCIRecipeBuildJob`."""
388-
389- __storm_table__ = 'OCIRecipeBuildJob'
390-
391- job_id = Int(name='job', primary=True, allow_none=False)
392- job = Reference(job_id, 'Job.id')
393-
394- build_id = Int(name='build', allow_none=False)
395- build = Reference(build_id, 'OCIRecipeBuild.id')
396-
397- job_type = DBEnum(enum=OCIRecipeBuildJobType, allow_none=True)
398-
399- json_data = JSON('json_data', allow_none=False)
400-
401- def __init__(self, build, job_type, json_data, **job_args):
402- """Constructor.
403-
404- Extra keyword arguments are used to construct the underlying Job
405- object.
406-
407- :param build: The `IOCIRecipeBuild` this job relates to.
408- :param job_type: The `OCIRecipeBuildJobType` of this job.
409- :param json_data: The type-specific variables, as a JSON-compatible
410- dict.
411- """
412- super(OCIRecipeBuildJob, self).__init__()
413- self.job = Job(**job_args)
414- self.build = build
415- self.job_type = job_type
416- self.json_data = json_data
417-
418- def makeDerived(self):
419- return OCIRecipeBuildJob.makeSubclass(self)
420-
421-
422-@delegate_to(IOCIRecipeBuildJob)
423-class OCIRecipeBuildJobDerived(BaseRunnableJob):
424-
425- __metaclass__ = EnumeratedSubclass
426-
427- def __init__(self, oci_build_job):
428- self.context = oci_build_job
429-
430- def __repr__(self):
431- """An informative representation of the job."""
432- return "<%s for %s>" % (
433- self.__class__.__name__, self.build.id)
434-
435- @classmethod
436- def get(cls, job_id):
437- """Get a job by id.
438-
439- :return: The `OCIBuildJob` with the specified id, as the current
440- `OCIBuildJobDerived` subclass.
441- :raises: `NotFoundError` if there is no job with the specified id,
442- or its `job_type` does not match the desired subclass.
443- """
444- oci_build_job = IStore(OCIRecipeBuildJob).get(
445- OCIRecipeBuildJob, job_id)
446- if oci_build_job.job_type != cls.class_job_type:
447- raise NotFoundError(
448- "No object found with id %d and type %s" %
449- (job_id, cls.class_job_type.title))
450- return cls(oci_build_job)
451-
452- @classmethod
453- def iterReady(cls):
454- """See `IJobSource`."""
455- jobs = IMasterStore(OCIRecipeBuildJob).find(
456- OCIRecipeBuildJob,
457- OCIRecipeBuildJob.job_type == cls.class_job_type,
458- OCIRecipeBuildJob.job == Job.id,
459- Job.id.is_in(Job.ready_jobs))
460- return (cls(job) for job in jobs)
461-
462- def getOopsVars(self):
463- """See `IRunnableJob`."""
464- oops_vars = super(OCIRecipeBuildJobDerived, self).getOopsVars()
465- oops_vars.extend([
466- ('job_type', self.context.job_type.title),
467- ('build_id', self.context.build.id),
468- ('owner_id', self.context.build.recipe.owner.id),
469- ('project_name', self.context.build.recipe.ociproject.name)
470- ])
471- return oops_vars
472diff --git a/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py b/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py
473new file mode 100644
474index 0000000..56d3592
475--- /dev/null
476+++ b/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py
477@@ -0,0 +1,312 @@
478+# Copyright 2015-2019 Canonical Ltd. This software is licensed under the
479+# GNU Affero General Public License version 3 (see the file LICENSE).
480+
481+"""Tests for `OCIRecipeBuildBehavior`."""
482+
483+from __future__ import absolute_import, print_function, unicode_literals
484+
485+__metaclass__ = type
486+
487+import json
488+import os
489+import shutil
490+import tempfile
491+
492+from testtools import ExpectedException
493+from twisted.internet import defer
494+from zope.security.proxy import removeSecurityProxy
495+
496+from lp.buildmaster.enums import BuildStatus
497+from lp.buildmaster.interactor import BuilderInteractor
498+from lp.buildmaster.interfaces.builder import BuildDaemonError
499+from lp.buildmaster.interfaces.buildfarmjobbehaviour import (
500+ IBuildFarmJobBehaviour,
501+ )
502+from lp.buildmaster.tests.mock_slaves import WaitingSlave
503+from lp.buildmaster.tests.test_buildfarmjobbehaviour import (
504+ TestGetUploadMethodsMixin,
505+ )
506+from lp.oci.model.ocirecipebuildbehaviour import OCIRecipeBuildBehaviour
507+from lp.services.config import config
508+from lp.testing import TestCaseWithFactory
509+from lp.testing.dbuser import dbuser
510+from lp.testing.factory import LaunchpadObjectFactory
511+from lp.testing.fakemethod import FakeMethod
512+from lp.testing.layers import LaunchpadZopelessLayer
513+from lp.testing.mail_helpers import pop_notifications
514+
515+
516+class MakeOCIBuildMixin:
517+
518+ def makeBuild(self):
519+ build = self.factory.makeOCIRecipeBuild()
520+ build.queueBuild()
521+ return build
522+
523+ def makeUnmodifiableBuild(self):
524+ build = self.factory.makeOCIRecipeBuild()
525+ build.distro_arch_series = 'failed'
526+ build.queueBuild()
527+ return build
528+
529+
530+class TestOCIBuildBehaviour(TestCaseWithFactory):
531+
532+ layer = LaunchpadZopelessLayer
533+
534+ def test_provides_interface(self):
535+ # OCIRecipeBuildBehaviour provides IBuildFarmJobBehaviour.
536+ job = OCIRecipeBuildBehaviour(None)
537+ self.assertProvides(job, IBuildFarmJobBehaviour)
538+
539+ def test_adapts_IOCIRecipeBuild(self):
540+ # IBuildFarmJobBehaviour adapts an IOCIRecipeBuild.
541+ build = self.factory.makeOCIRecipeBuild()
542+ job = IBuildFarmJobBehaviour(build)
543+ self.assertProvides(job, IBuildFarmJobBehaviour)
544+
545+
546+class TestHandleStatusForOCIRecipeBuild(MakeOCIBuildMixin,
547+ TestCaseWithFactory):
548+ # This is mostly copied from TestHandleStatusMixin, however
549+ # we can't use all of those tests, due to the way OCIRecipeBuildBehaviour
550+ # parses the file contents, rather than just retrieving all that are
551+ # available. There's also some differences in the filemap handling, as
552+ # we need a much more complex filemap here.
553+
554+ layer = LaunchpadZopelessLayer
555+
556+ def _createTestFile(self, name, content, hash):
557+ path = os.path.join(self.test_files_dir, name)
558+ with open(path, 'wb') as fp:
559+ fp.write(content)
560+ self.slave.valid_files[hash] = path
561+
562+ def setUp(self):
563+ super(TestHandleStatusForOCIRecipeBuild, self).setUp()
564+ self.factory = LaunchpadObjectFactory()
565+ self.build = self.makeBuild()
566+ # For the moment, we require a builder for the build so that
567+ # handleStatus_OK can get a reference to the slave.
568+ self.builder = self.factory.makeBuilder()
569+ self.build.buildqueue_record.markAsBuilding(self.builder)
570+ self.slave = WaitingSlave('BuildStatus.OK')
571+ self.slave.valid_files['test_file_hash'] = ''
572+ self.interactor = BuilderInteractor()
573+ self.behaviour = self.interactor.getBuildBehaviour(
574+ self.build.buildqueue_record, self.builder, self.slave)
575+
576+ # We overwrite the buildmaster root to use a temp directory.
577+ tempdir = tempfile.mkdtemp()
578+ self.addCleanup(shutil.rmtree, tempdir)
579+ self.upload_root = tempdir
580+ self.pushConfig('builddmaster', root=self.upload_root)
581+
582+ # We stub out our builds getUploaderCommand() method so
583+ # we can check whether it was called as well as
584+ # verifySuccessfulUpload().
585+ removeSecurityProxy(self.build).verifySuccessfulUpload = FakeMethod(
586+ result=True)
587+
588+ digests = {
589+ "diff_id_1": {
590+ "digest": "digest_1",
591+ "source": "test/base_1",
592+ "layer_id": "layer_1"
593+ },
594+ "diff_id_2": {
595+ "digest": "digest_2",
596+ "source": "",
597+ "layer_id": "layer_2"
598+ }
599+ }
600+
601+ self.test_files_dir = tempfile.mkdtemp()
602+ self._createTestFile('buildlog', '', 'buildlog')
603+ self._createTestFile(
604+ 'manifest.json',
605+ '[{"Config": "config_file_1.json", '
606+ '"Layers": ["layer_1/layer.tar", "layer_2/layer.tar"]}]',
607+ 'manifest_hash')
608+ self._createTestFile(
609+ 'digests.json',
610+ json.dumps(digests),
611+ 'digests_hash')
612+ self._createTestFile(
613+ 'config_file_1.json',
614+ '{"rootfs": {"diff_ids": ["diff_id_1", "diff_id_2"]}}',
615+ 'config_1_hash')
616+ self._createTestFile(
617+ 'layer_2.tar.gz',
618+ '',
619+ 'layer_2_hash'
620+ )
621+
622+ self.filemap = {
623+ 'manifest.json': 'manifest_hash',
624+ 'digests.json': 'digests_hash',
625+ 'config_file_1.json': 'config_1_hash',
626+ 'layer_1.tar.gz': 'layer_1_hash',
627+ 'layer_2.tar.gz': 'layer_2_hash'
628+ }
629+ self.factory.makeOCIFile(
630+ build=self.build, layer_file_digest=u'digest_1',
631+ content="retrieved from librarian")
632+
633+ def assertResultCount(self, count, result):
634+ self.assertEqual(
635+ 1, len(os.listdir(os.path.join(self.upload_root, result))))
636+
637+ @defer.inlineCallbacks
638+ def test_handleStatus_OK_normal_image(self):
639+ with dbuser(config.builddmaster.dbuser):
640+ yield self.behaviour.handleStatus(
641+ self.build.buildqueue_record, 'OK',
642+ {'filemap': self.filemap})
643+ self.assertEqual(
644+ ['buildlog', 'manifest_hash', 'digests_hash', 'config_1_hash',
645+ 'layer_2_hash'],
646+ self.slave._got_file_record)
647+ # This hash should not appear as it is already in the librarian
648+ self.assertNotIn('layer_1_hash', self.slave._got_file_record)
649+ self.assertEqual(BuildStatus.UPLOADING, self.build.status)
650+ self.assertResultCount(1, "incoming")
651+
652+ # layer_1 should have been retrieved from the librarian
653+ layer_1_path = os.path.join(
654+ self.upload_root,
655+ "incoming",
656+ self.behaviour.getUploadDirLeaf(self.build.build_cookie),
657+ str(self.build.archive.id),
658+ self.build.distribution.name,
659+ "layer_1.tar.gz"
660+ )
661+ with open(layer_1_path, 'rb') as layer_1_fp:
662+ contents = layer_1_fp.read()
663+ self.assertEqual(contents, b'retrieved from librarian')
664+
665+ @defer.inlineCallbacks
666+ def test_handleStatus_OK_absolute_filepath(self):
667+
668+ self._createTestFile(
669+ 'manifest.json',
670+ '[{"Config": "/notvalid/config_file_1.json", '
671+ '"Layers": ["layer_1/layer.tar", "layer_2/layer.tar"]}]',
672+ 'manifest_hash')
673+
674+ self.filemap['/notvalid/config_file_1.json'] = 'config_1_hash'
675+
676+ # A filemap that tries to write to files outside of the upload
677+ # directory will not be collected.
678+ with ExpectedException(
679+ BuildDaemonError,
680+ "Build returned a file named "
681+ "'/notvalid/config_file_1.json'."):
682+ with dbuser(config.builddmaster.dbuser):
683+ yield self.behaviour.handleStatus(
684+ self.build.buildqueue_record, 'OK',
685+ {'filemap': self.filemap})
686+
687+ @defer.inlineCallbacks
688+ def test_handleStatus_OK_relative_filepath(self):
689+
690+ self._createTestFile(
691+ 'manifest.json',
692+ '[{"Config": "../config_file_1.json", '
693+ '"Layers": ["layer_1/layer.tar", "layer_2/layer.tar"]}]',
694+ 'manifest_hash')
695+
696+ self.filemap['../config_file_1.json'] = 'config_1_hash'
697+ # A filemap that tries to write to files outside of
698+ # the upload directory will not be collected.
699+ with ExpectedException(
700+ BuildDaemonError,
701+ "Build returned a file named '../config_file_1.json'."):
702+ with dbuser(config.builddmaster.dbuser):
703+ yield self.behaviour.handleStatus(
704+ self.build.buildqueue_record, 'OK',
705+ {'filemap': self.filemap})
706+
707+ @defer.inlineCallbacks
708+ def test_handleStatus_OK_sets_build_log(self):
709+ # The build log is set during handleStatus.
710+ self.assertEqual(None, self.build.log)
711+ with dbuser(config.builddmaster.dbuser):
712+ yield self.behaviour.handleStatus(
713+ self.build.buildqueue_record, 'OK',
714+ {'filemap': self.filemap})
715+ self.assertNotEqual(None, self.build.log)
716+
717+ @defer.inlineCallbacks
718+ def test_handleStatus_ABORTED_cancels_cancelling(self):
719+ with dbuser(config.builddmaster.dbuser):
720+ self.build.updateStatus(BuildStatus.CANCELLING)
721+ yield self.behaviour.handleStatus(
722+ self.build.buildqueue_record, "ABORTED", {})
723+ self.assertEqual(0, len(pop_notifications()), "Notifications received")
724+ self.assertEqual(BuildStatus.CANCELLED, self.build.status)
725+
726+ @defer.inlineCallbacks
727+ def test_handleStatus_ABORTED_illegal_when_building(self):
728+ self.builder.vm_host = "fake_vm_host"
729+ self.behaviour = self.interactor.getBuildBehaviour(
730+ self.build.buildqueue_record, self.builder, self.slave)
731+ with dbuser(config.builddmaster.dbuser):
732+ self.build.updateStatus(BuildStatus.BUILDING)
733+ with ExpectedException(
734+ BuildDaemonError,
735+ "Build returned unexpected status: u'ABORTED'"):
736+ yield self.behaviour.handleStatus(
737+ self.build.buildqueue_record, "ABORTED", {})
738+
739+ @defer.inlineCallbacks
740+ def test_handleStatus_ABORTED_cancelling_sets_build_log(self):
741+ # If a build is intentionally cancelled, the build log is set.
742+ self.assertEqual(None, self.build.log)
743+ with dbuser(config.builddmaster.dbuser):
744+ self.build.updateStatus(BuildStatus.CANCELLING)
745+ yield self.behaviour.handleStatus(
746+ self.build.buildqueue_record, "ABORTED", {})
747+ self.assertNotEqual(None, self.build.log)
748+
749+ @defer.inlineCallbacks
750+ def test_date_finished_set(self):
751+ # The date finished is updated during handleStatus_OK.
752+ self.assertEqual(None, self.build.date_finished)
753+ with dbuser(config.builddmaster.dbuser):
754+ yield self.behaviour.handleStatus(
755+ self.build.buildqueue_record, 'OK',
756+ {'filemap': self.filemap})
757+ self.assertNotEqual(None, self.build.date_finished)
758+
759+ @defer.inlineCallbacks
760+ def test_givenback_collection(self):
761+ with ExpectedException(
762+ BuildDaemonError,
763+ "Build returned unexpected status: u'GIVENBACK'"):
764+ with dbuser(config.builddmaster.dbuser):
765+ yield self.behaviour.handleStatus(
766+ self.build.buildqueue_record, "GIVENBACK", {})
767+
768+ @defer.inlineCallbacks
769+ def test_builderfail_collection(self):
770+ with ExpectedException(
771+ BuildDaemonError,
772+ "Build returned unexpected status: u'BUILDERFAIL'"):
773+ with dbuser(config.builddmaster.dbuser):
774+ yield self.behaviour.handleStatus(
775+ self.build.buildqueue_record, "BUILDERFAIL", {})
776+
777+ @defer.inlineCallbacks
778+ def test_invalid_status_collection(self):
779+ with ExpectedException(
780+ BuildDaemonError,
781+ "Build returned unexpected status: u'BORKED'"):
782+ with dbuser(config.builddmaster.dbuser):
783+ yield self.behaviour.handleStatus(
784+ self.build.buildqueue_record, "BORKED", {})
785+
786+
787+class TestGetUploadMethodsForOCIRecipeBuild(
788+ MakeOCIBuildMixin, TestGetUploadMethodsMixin, TestCaseWithFactory):
789+ """IPackageBuild.getUpload-related methods work with OCI recipe builds."""
790diff --git a/lib/lp/oci/tests/test_ocirecipebuildjob.py b/lib/lp/oci/tests/test_ocirecipebuildjob.py
791deleted file mode 100644
792index cda3eef..0000000
793--- a/lib/lp/oci/tests/test_ocirecipebuildjob.py
794+++ /dev/null
795@@ -1,49 +0,0 @@
796-# Copyright 2019 Canonical Ltd. This software is licensed under the
797-# GNU Affero General Public License version 3 (see the file LICENSE).
798-
799-"""OCIRecipeBuildJob tests"""
800-
801-from __future__ import absolute_import, print_function, unicode_literals
802-
803-__metaclass__ = type
804-
805-
806-from lp.oci.interfaces.ocirecipebuildjob import IOCIRecipeBuildJob
807-from lp.oci.model.ocirecipebuildjob import (
808- OCIRecipeBuildJob,
809- OCIRecipeBuildJobDerived,
810- OCIRecipeBuildJobType,
811- )
812-from lp.testing import TestCaseWithFactory
813-from lp.testing.layers import DatabaseFunctionalLayer
814-
815-
816-class FakeOCIBuildJob(OCIRecipeBuildJobDerived):
817- """For testing OCIRecipeBuildJobDerived without a child class."""
818-
819-
820-class TestOCIRecipeBuildJob(TestCaseWithFactory):
821-
822- layer = DatabaseFunctionalLayer
823-
824- def test_provides_interface(self):
825- oci_build = self.factory.makeOCIRecipeBuild()
826- self.assertProvides(
827- OCIRecipeBuildJob(
828- oci_build, OCIRecipeBuildJobType.REGISTRY_UPLOAD, {}),
829- IOCIRecipeBuildJob)
830-
831- def test_getOopsVars(self):
832- oci_build = self.factory.makeOCIRecipeBuild()
833- build_job = OCIRecipeBuildJob(
834- oci_build, OCIRecipeBuildJobType.REGISTRY_UPLOAD, {})
835- derived = FakeOCIBuildJob(build_job)
836- oops = derived.getOopsVars()
837- expected = [
838- ('job_id', build_job.job.id),
839- ('job_type', build_job.job_type.title),
840- ('build_id', oci_build.id),
841- ('owner_id', oci_build.recipe.owner.id),
842- ('project_name', oci_build.recipe.ociproject.name),
843- ]
844- self.assertEqual(expected, oops)
845diff --git a/lib/lp/testing/factory.py b/lib/lp/testing/factory.py
846index f0788d8..4a25c88 100644
847--- a/lib/lp/testing/factory.py
848+++ b/lib/lp/testing/factory.py
849@@ -5005,12 +5005,13 @@ class BareLaunchpadObjectFactory(ObjectFactory):
850 return oci_build
851
852 def makeOCIFile(self, build=None, library_file=None,
853- layer_file_digest=None):
854+ layer_file_digest=None, content=None, filename=None):
855 """Make a new OCIFile."""
856 if build is None:
857 build = self.makeOCIRecipeBuild()
858 if library_file is None:
859- library_file = self.makeLibraryFileAlias()
860+ library_file = self.makeLibraryFileAlias(
861+ content=content, filename=filename)
862 return OCIFile(build=build, library_file=library_file,
863 layer_file_digest=layer_file_digest)
864
865diff --git a/lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py b/lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py
866index b387251..32dd553 100644
867--- a/lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py
868+++ b/lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py
869@@ -138,7 +138,7 @@ class TestTranslationTemplatesBuildBehaviour(
870 buildqueue = FakeBuildQueue(behaviour)
871 path = behaviour.templates_tarball_path
872 # Poke the file we're expecting into the mock slave.
873- behaviour._slave.valid_file_hashes.append(path)
874+ behaviour._slave.valid_files[path] = ''
875
876 def got_tarball(filename):
877 tarball = open(filename, 'r')

Subscribers

People subscribed via source and target branches