Merge ~twom/launchpad:oci-buildbehaviour into launchpad:master
- Git
- lp:~twom/launchpad
- oci-buildbehaviour
- Merge into master
Proposed by
Tom Wardill
Status: | Superseded |
---|---|
Proposed branch: | ~twom/launchpad:oci-buildbehaviour |
Merge into: | launchpad:master |
Prerequisite: | ~twom/launchpad:oci-buildjob |
Diff against target: |
877 lines (+473/-79) 11 files modified
dev/null (+0/-49) lib/lp/buildmaster/model/buildfarmjobbehaviour.py (+17/-13) lib/lp/buildmaster/tests/mock_slaves.py (+9/-3) lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py (+1/-1) lib/lp/oci/configure.zcml (+6/-0) lib/lp/oci/model/ocirecipe.py (+0/-10) lib/lp/oci/model/ocirecipebuild.py (+9/-0) lib/lp/oci/model/ocirecipebuildbehaviour.py (+115/-0) lib/lp/oci/tests/test_ocirecipebuildbehaviour.py (+312/-0) lib/lp/testing/factory.py (+3/-2) lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py (+1/-1) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Colin Watson (community) | Needs Fixing | ||
Review via email: mp+376678@code.launchpad.net |
This proposal has been superseded by a proposal from 2020-02-14.
Commit message
Add OCIRecipeBuildB
Description of the change
Add a buildbehaviour to drive OCI image builds.
Use mixin tests where sensible, override for places where the OCI methods differ.
To post a comment you must log in.
Revision history for this message
Colin Watson (cjwatson) : | # |
review:
Needs Fixing
Revision history for this message
Tom Wardill (twom) : | # |
Revision history for this message
Colin Watson (cjwatson) : | # |
There was an error fetching revisions from git servers. Please try again in a few minutes. If the problem persists, contact Launchpad support.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/lib/lp/buildmaster/model/buildfarmjobbehaviour.py b/lib/lp/buildmaster/model/buildfarmjobbehaviour.py |
2 | index 2f3515a..4b28693 100644 |
3 | --- a/lib/lp/buildmaster/model/buildfarmjobbehaviour.py |
4 | +++ b/lib/lp/buildmaster/model/buildfarmjobbehaviour.py |
5 | @@ -301,6 +301,22 @@ class BuildFarmJobBehaviourBase: |
6 | transaction.commit() |
7 | |
8 | @defer.inlineCallbacks |
9 | + def _downloadFiles(self, filemap, upload_path, logger): |
10 | + filenames_to_download = [] |
11 | + for filename, sha1 in filemap.items(): |
12 | + logger.info("Grabbing file: %s (%s)" % ( |
13 | + filename, self._slave.getURL(sha1))) |
14 | + out_file_name = os.path.join(upload_path, filename) |
15 | + # If the evaluated output file name is not within our |
16 | + # upload path, then we don't try to copy this or any |
17 | + # subsequent files. |
18 | + if not os.path.realpath(out_file_name).startswith(upload_path): |
19 | + raise BuildDaemonError( |
20 | + "Build returned a file named %r." % filename) |
21 | + filenames_to_download.append((sha1, out_file_name)) |
22 | + yield self._slave.getFiles(filenames_to_download, logger=logger) |
23 | + |
24 | + @defer.inlineCallbacks |
25 | def handleSuccess(self, slave_status, logger): |
26 | """Handle a package that built successfully. |
27 | |
28 | @@ -337,19 +353,7 @@ class BuildFarmJobBehaviourBase: |
29 | grab_dir, str(build.archive.id), build.distribution.name) |
30 | os.makedirs(upload_path) |
31 | |
32 | - filenames_to_download = [] |
33 | - for filename, sha1 in filemap.items(): |
34 | - logger.info("Grabbing file: %s (%s)" % ( |
35 | - filename, self._slave.getURL(sha1))) |
36 | - out_file_name = os.path.join(upload_path, filename) |
37 | - # If the evaluated output file name is not within our |
38 | - # upload path, then we don't try to copy this or any |
39 | - # subsequent files. |
40 | - if not os.path.realpath(out_file_name).startswith(upload_path): |
41 | - raise BuildDaemonError( |
42 | - "Build returned a file named %r." % filename) |
43 | - filenames_to_download.append((sha1, out_file_name)) |
44 | - yield self._slave.getFiles(filenames_to_download, logger=logger) |
45 | + yield self._downloadFiles(filemap, upload_path, logger) |
46 | |
47 | transaction.commit() |
48 | |
49 | diff --git a/lib/lp/buildmaster/tests/mock_slaves.py b/lib/lp/buildmaster/tests/mock_slaves.py |
50 | index 8b00c32..eca171a 100644 |
51 | --- a/lib/lp/buildmaster/tests/mock_slaves.py |
52 | +++ b/lib/lp/buildmaster/tests/mock_slaves.py |
53 | @@ -194,7 +194,8 @@ class WaitingSlave(OkSlave): |
54 | |
55 | # By default, the slave only has a buildlog, but callsites |
56 | # can update this list as needed. |
57 | - self.valid_file_hashes = ['buildlog'] |
58 | + self.valid_files = {'buildlog': ''} |
59 | + self._got_file_record = [] |
60 | |
61 | def status(self): |
62 | self.call_log.append('status') |
63 | @@ -208,12 +209,17 @@ class WaitingSlave(OkSlave): |
64 | |
65 | def getFile(self, hash, file_to_write): |
66 | self.call_log.append('getFile') |
67 | - if hash in self.valid_file_hashes: |
68 | - content = "This is a %s" % hash |
69 | + if hash in self.valid_files: |
70 | if isinstance(file_to_write, types.StringTypes): |
71 | file_to_write = open(file_to_write, 'wb') |
72 | + if not self.valid_files[hash]: |
73 | + content = "This is a %s" % hash |
74 | + else: |
75 | + with open(self.valid_files[hash], 'rb') as source: |
76 | + content = source.read() |
77 | file_to_write.write(content) |
78 | file_to_write.close() |
79 | + self._got_file_record.append(hash) |
80 | return defer.succeed(None) |
81 | |
82 | |
83 | diff --git a/lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py b/lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py |
84 | index 697ff52..d6ecd85 100644 |
85 | --- a/lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py |
86 | +++ b/lib/lp/buildmaster/tests/test_buildfarmjobbehaviour.py |
87 | @@ -337,7 +337,7 @@ class TestHandleStatusMixin: |
88 | self.builder = self.factory.makeBuilder() |
89 | self.build.buildqueue_record.markAsBuilding(self.builder) |
90 | self.slave = WaitingSlave('BuildStatus.OK') |
91 | - self.slave.valid_file_hashes.append('test_file_hash') |
92 | + self.slave.valid_file_hashes['test_file_hash'] = '' |
93 | self.interactor = BuilderInteractor() |
94 | self.behaviour = self.interactor.getBuildBehaviour( |
95 | self.build.buildqueue_record, self.builder, self.slave) |
96 | diff --git a/lib/lp/oci/configure.zcml b/lib/lp/oci/configure.zcml |
97 | index 30212b9..51e1d31 100644 |
98 | --- a/lib/lp/oci/configure.zcml |
99 | +++ b/lib/lp/oci/configure.zcml |
100 | @@ -56,4 +56,10 @@ |
101 | <allow interface="lp.buildmaster.interfaces.buildfarmjob.ISpecificBuildFarmJobSource" /> |
102 | </securedutility> |
103 | |
104 | + <adapter |
105 | + for="lp.oci.interfaces.ocirecipebuild.IOCIRecipeBuild" |
106 | + provides="lp.buildmaster.interfaces.buildfarmjobbehaviour.IBuildFarmJobBehaviour" |
107 | + factory="lp.oci.model.ocirecipebuildbehaviour.OCIRecipeBuildBehaviour" |
108 | + permission="zope.Public" /> |
109 | + |
110 | </configure> |
111 | diff --git a/lib/lp/oci/interfaces/ocirecipebuildjob.py b/lib/lp/oci/interfaces/ocirecipebuildjob.py |
112 | deleted file mode 100644 |
113 | index cd72177..0000000 |
114 | --- a/lib/lp/oci/interfaces/ocirecipebuildjob.py |
115 | +++ /dev/null |
116 | @@ -1,38 +0,0 @@ |
117 | -# Copyright 2019 Canonical Ltd. This software is licensed under the |
118 | -# GNU Affero General Public License version 3 (see the file LICENSE). |
119 | - |
120 | -"""OCIRecipe build job interfaces""" |
121 | - |
122 | -from __future__ import absolute_import, print_function, unicode_literals |
123 | - |
124 | -__metaclass__ = type |
125 | -__all__ = [ |
126 | - 'IOCIRecipeBuildJob', |
127 | - ] |
128 | - |
129 | -from lazr.restful.fields import Reference |
130 | -from zope.interface import ( |
131 | - Attribute, |
132 | - Interface, |
133 | - ) |
134 | -from zope.schema import TextLine |
135 | - |
136 | -from lp import _ |
137 | -from lp.oci.interfaces.ocirecipebuild import IOCIRecipeBuild |
138 | -from lp.services.job.interfaces.job import ( |
139 | - IJob, |
140 | - IJobSource, |
141 | - IRunnableJob, |
142 | - ) |
143 | - |
144 | - |
145 | -class IOCIRecipeBuildJob(Interface): |
146 | - job = Reference( |
147 | - title=_("The common Job attributes."), schema=IJob, |
148 | - required=True, readonly=True) |
149 | - |
150 | - build = Reference( |
151 | - title=_("The OCI Recipe Build to use for this job."), |
152 | - schema=IOCIRecipeBuild, required=True, readonly=True) |
153 | - |
154 | - json_data = Attribute(_("A dict of data about the job.")) |
155 | diff --git a/lib/lp/oci/model/ocirecipe.py b/lib/lp/oci/model/ocirecipe.py |
156 | index 300f335..8a01b2d 100644 |
157 | --- a/lib/lp/oci/model/ocirecipe.py |
158 | +++ b/lib/lp/oci/model/ocirecipe.py |
159 | @@ -308,15 +308,5 @@ class OCIRecipeSet: |
160 | person_ids.add(recipe.registrant_id) |
161 | person_ids.add(recipe.owner_id) |
162 | |
163 | - repositories = load_related( |
164 | - GitRepository, recipes, ["git_repository_id"]) |
165 | - if repositories: |
166 | - GenericGitCollection.preloadDataForRepositories(repositories) |
167 | - GenericGitCollection.preloadVisibleRepositories(repositories, user) |
168 | - |
169 | - # We need the target repository owner as well; unlike branches, |
170 | - # repository unique names aren't trigger-maintained. |
171 | - person_ids.update(repository.owner_id for repository in repositories) |
172 | - |
173 | list(getUtility(IPersonSet).getPrecachedPersonsFromIDs( |
174 | person_ids, need_validity=True)) |
175 | diff --git a/lib/lp/oci/model/ocirecipebuild.py b/lib/lp/oci/model/ocirecipebuild.py |
176 | index 307fbd2..5d2d21d 100644 |
177 | --- a/lib/lp/oci/model/ocirecipebuild.py |
178 | +++ b/lib/lp/oci/model/ocirecipebuild.py |
179 | @@ -44,6 +44,7 @@ from lp.oci.interfaces.ocirecipebuild import ( |
180 | IOCIRecipeBuildSet, |
181 | ) |
182 | from lp.registry.model.person import Person |
183 | +from lp.services.config import config |
184 | from lp.services.database.bulk import load_related |
185 | from lp.services.database.constants import DEFAULT |
186 | from lp.services.database.decoratedresultset import DecoratedResultSet |
187 | @@ -208,6 +209,14 @@ class OCIRecipeBuild(PackageBuildMixin, Storm): |
188 | # pillar isn't just a distribution |
189 | return self.recipe.oci_project.distribution |
190 | |
191 | + def notify(self, extra_info=None): |
192 | + """See `IPackageBuild`.""" |
193 | + if not config.builddmaster.send_build_notification: |
194 | + return |
195 | + if self.status == BuildStatus.FULLYBUILT: |
196 | + return |
197 | + # XXX twom 2019-12-11 This should send mail |
198 | + |
199 | |
200 | @implementer(IOCIRecipeBuildSet) |
201 | class OCIRecipeBuildSet(SpecificBuildFarmJobSourceMixin): |
202 | diff --git a/lib/lp/oci/model/ocirecipebuildbehaviour.py b/lib/lp/oci/model/ocirecipebuildbehaviour.py |
203 | new file mode 100644 |
204 | index 0000000..e1a1215 |
205 | --- /dev/null |
206 | +++ b/lib/lp/oci/model/ocirecipebuildbehaviour.py |
207 | @@ -0,0 +1,115 @@ |
208 | +# Copyright 2019 Canonical Ltd. This software is licensed under the |
209 | +# GNU Affero General Public License version 3 (see the file LICENSE). |
210 | + |
211 | +"""An `IBuildFarmJobBehaviour` for `OCIRecipeBuild`. |
212 | + |
213 | +Dispatches OCI image build jobs to build-farm slaves. |
214 | +""" |
215 | + |
216 | +from __future__ import absolute_import, print_function, unicode_literals |
217 | + |
218 | +__metaclass__ = type |
219 | +__all__ = [ |
220 | + 'OCIRecipeBuildBehaviour', |
221 | + ] |
222 | + |
223 | + |
224 | +import json |
225 | +import os |
226 | + |
227 | +from twisted.internet import defer |
228 | +from zope.interface import implementer |
229 | + |
230 | +from lp.app.errors import NotFoundError |
231 | +from lp.buildmaster.enums import BuildBaseImageType |
232 | +from lp.buildmaster.interfaces.builder import BuildDaemonError |
233 | +from lp.buildmaster.interfaces.buildfarmjobbehaviour import ( |
234 | + IBuildFarmJobBehaviour, |
235 | + ) |
236 | +from lp.buildmaster.model.buildfarmjobbehaviour import ( |
237 | + BuildFarmJobBehaviourBase, |
238 | + ) |
239 | +from lp.services.librarian.utils import copy_and_close |
240 | + |
241 | + |
242 | +@implementer(IBuildFarmJobBehaviour) |
243 | +class OCIRecipeBuildBehaviour(BuildFarmJobBehaviourBase): |
244 | + |
245 | + builder_type = "oci" |
246 | + image_types = [BuildBaseImageType.LXD] |
247 | + |
248 | + # These attributes are defined in `IOCIBuildFarmJobBehaviour`, |
249 | + # but are not used in this implementation. |
250 | + distro_arch_series = None |
251 | + |
252 | + def _ensureFilePath(self, file_name, file_path, upload_path): |
253 | + # If the evaluated output file name is not within our |
254 | + # upload path, then we don't try to copy this or any |
255 | + # subsequent files. |
256 | + if not os.path.normpath(file_path).startswith(upload_path): |
257 | + raise BuildDaemonError( |
258 | + "Build returned a file named '%s'." % file_name) |
259 | + |
260 | + def _fetchIntermediaryFile(self, name, filemap, upload_path): |
261 | + file_hash = filemap[name] |
262 | + file_path = os.path.join(upload_path, name) |
263 | + self._ensureFilePath(name, file_path, upload_path) |
264 | + self._slave.getFile(file_hash, file_path) |
265 | + |
266 | + with open(file_path, 'r') as file_fp: |
267 | + contents = json.load(file_fp) |
268 | + return contents |
269 | + |
270 | + def _extractLayerFiles(self, upload_path, section, config, digests, files): |
271 | + # These are different sets of ids, in the same order |
272 | + # layer_id is the filename, diff_id is the internal (docker) id |
273 | + for diff_id in config['rootfs']['diff_ids']: |
274 | + layer_id = digests[diff_id]['layer_id'] |
275 | + # This is in the form '<id>/layer.tar', we only need the first |
276 | + layer_filename = "{}.tar.gz".format(layer_id.split('/')[0]) |
277 | + digest = digests[diff_id]['digest'] |
278 | + try: |
279 | + _, librarian_layer_file, _ = self.build.getLayerFileByDigest( |
280 | + digest) |
281 | + except NotFoundError: |
282 | + files.add(layer_filename) |
283 | + continue |
284 | + layer_path = os.path.join(upload_path, layer_filename) |
285 | + librarian_layer_file.open() |
286 | + with open(layer_path, 'wb') as layer_fp: |
287 | + copy_and_close(librarian_layer_file, layer_fp) |
288 | + |
289 | + def _convertToRetrievableFile(self, upload_path, file_name, filemap): |
290 | + file_path = os.path.join(upload_path, file_name) |
291 | + self._ensureFilePath(file_name, file_path, upload_path) |
292 | + return (filemap[file_name], file_path) |
293 | + |
294 | + @defer.inlineCallbacks |
295 | + def _downloadFiles(self, filemap, upload_path, logger): |
296 | + """Download required artifact files.""" |
297 | + # We don't want to download all of the files that have been created, |
298 | + # just the ones that are mentioned in the manifest and config. |
299 | + |
300 | + manifest = self._fetchIntermediaryFile( |
301 | + 'manifest.json', filemap, upload_path) |
302 | + digests = self._fetchIntermediaryFile( |
303 | + 'digests.json', filemap, upload_path) |
304 | + |
305 | + files = set() |
306 | + for section in manifest: |
307 | + config = self._fetchIntermediaryFile( |
308 | + section['Config'], filemap, upload_path) |
309 | + self._extractLayerFiles( |
310 | + upload_path, section, config, digests, files) |
311 | + |
312 | + files_to_download = [ |
313 | + self._convertToRetrievableFile(upload_path, filename, filemap) |
314 | + for filename in files] |
315 | + yield self._slave.getFiles(files_to_download, logger=logger) |
316 | + |
317 | + def verifySuccessfulBuild(self): |
318 | + """See `IBuildFarmJobBehaviour`.""" |
319 | + # The implementation in BuildFarmJobBehaviourBase checks whether the |
320 | + # target suite is modifiable in the target archive. However, an |
321 | + # `OCIRecipeBuild` does not use an archive in this manner. |
322 | + return True |
323 | diff --git a/lib/lp/oci/model/ocirecipebuildjob.py b/lib/lp/oci/model/ocirecipebuildjob.py |
324 | deleted file mode 100644 |
325 | index 6df00cf..0000000 |
326 | --- a/lib/lp/oci/model/ocirecipebuildjob.py |
327 | +++ /dev/null |
328 | @@ -1,143 +0,0 @@ |
329 | -# Copyright 2019 Canonical Ltd. This software is licensed under the |
330 | -# GNU Affero General Public License version 3 (see the file LICENSE). |
331 | - |
332 | -"""OCIRecipe build jobs.""" |
333 | - |
334 | -from __future__ import absolute_import, print_function, unicode_literals |
335 | - |
336 | -__metaclass__ = type |
337 | -__all__ = [ |
338 | - 'OCIRecipeBuildJob', |
339 | - 'OCIRecipeBuildJobType', |
340 | - ] |
341 | - |
342 | -import json |
343 | - |
344 | -from lazr.delegates import delegate_to |
345 | -from lazr.enum import ( |
346 | - DBEnumeratedType, |
347 | - DBItem, |
348 | - ) |
349 | -from storm.locals import ( |
350 | - Int, |
351 | - JSON, |
352 | - Reference, |
353 | - ) |
354 | -from zope.interface import ( |
355 | - implementer, |
356 | - provider, |
357 | - ) |
358 | - |
359 | -from lp.app.errors import NotFoundError |
360 | -from lp.oci.interfaces.ocirecipebuildjob import IOCIRecipeBuildJob |
361 | -from lp.services.database.enumcol import DBEnum |
362 | -from lp.services.database.interfaces import ( |
363 | - IMasterStore, |
364 | - IStore, |
365 | - ) |
366 | -from lp.services.database.stormbase import StormBase |
367 | -from lp.services.job.model.job import ( |
368 | - EnumeratedSubclass, |
369 | - Job, |
370 | - ) |
371 | -from lp.services.job.runner import BaseRunnableJob |
372 | -from lp.services.propertycache import get_property_cache |
373 | - |
374 | - |
375 | -class OCIRecipeBuildJobType(DBEnumeratedType): |
376 | - """Values that `OCIBuildJobType.job_type` can take.""" |
377 | - |
378 | - REGISTRY_UPLOAD = DBItem(0, """ |
379 | - Registry upload |
380 | - |
381 | - This job uploads an OCI Image to registry. |
382 | - """) |
383 | - |
384 | - |
385 | -@implementer(IOCIRecipeBuildJob) |
386 | -class OCIRecipeBuildJob(StormBase): |
387 | - """See `IOCIRecipeBuildJob`.""" |
388 | - |
389 | - __storm_table__ = 'OCIRecipeBuildJob' |
390 | - |
391 | - job_id = Int(name='job', primary=True, allow_none=False) |
392 | - job = Reference(job_id, 'Job.id') |
393 | - |
394 | - build_id = Int(name='build', allow_none=False) |
395 | - build = Reference(build_id, 'OCIRecipeBuild.id') |
396 | - |
397 | - job_type = DBEnum(enum=OCIRecipeBuildJobType, allow_none=True) |
398 | - |
399 | - json_data = JSON('json_data', allow_none=False) |
400 | - |
401 | - def __init__(self, build, job_type, json_data, **job_args): |
402 | - """Constructor. |
403 | - |
404 | - Extra keyword arguments are used to construct the underlying Job |
405 | - object. |
406 | - |
407 | - :param build: The `IOCIRecipeBuild` this job relates to. |
408 | - :param job_type: The `OCIRecipeBuildJobType` of this job. |
409 | - :param json_data: The type-specific variables, as a JSON-compatible |
410 | - dict. |
411 | - """ |
412 | - super(OCIRecipeBuildJob, self).__init__() |
413 | - self.job = Job(**job_args) |
414 | - self.build = build |
415 | - self.job_type = job_type |
416 | - self.json_data = json_data |
417 | - |
418 | - def makeDerived(self): |
419 | - return OCIRecipeBuildJob.makeSubclass(self) |
420 | - |
421 | - |
422 | -@delegate_to(IOCIRecipeBuildJob) |
423 | -class OCIRecipeBuildJobDerived(BaseRunnableJob): |
424 | - |
425 | - __metaclass__ = EnumeratedSubclass |
426 | - |
427 | - def __init__(self, oci_build_job): |
428 | - self.context = oci_build_job |
429 | - |
430 | - def __repr__(self): |
431 | - """An informative representation of the job.""" |
432 | - return "<%s for %s>" % ( |
433 | - self.__class__.__name__, self.build.id) |
434 | - |
435 | - @classmethod |
436 | - def get(cls, job_id): |
437 | - """Get a job by id. |
438 | - |
439 | - :return: The `OCIBuildJob` with the specified id, as the current |
440 | - `OCIBuildJobDerived` subclass. |
441 | - :raises: `NotFoundError` if there is no job with the specified id, |
442 | - or its `job_type` does not match the desired subclass. |
443 | - """ |
444 | - oci_build_job = IStore(OCIRecipeBuildJob).get( |
445 | - OCIRecipeBuildJob, job_id) |
446 | - if oci_build_job.job_type != cls.class_job_type: |
447 | - raise NotFoundError( |
448 | - "No object found with id %d and type %s" % |
449 | - (job_id, cls.class_job_type.title)) |
450 | - return cls(oci_build_job) |
451 | - |
452 | - @classmethod |
453 | - def iterReady(cls): |
454 | - """See `IJobSource`.""" |
455 | - jobs = IMasterStore(OCIRecipeBuildJob).find( |
456 | - OCIRecipeBuildJob, |
457 | - OCIRecipeBuildJob.job_type == cls.class_job_type, |
458 | - OCIRecipeBuildJob.job == Job.id, |
459 | - Job.id.is_in(Job.ready_jobs)) |
460 | - return (cls(job) for job in jobs) |
461 | - |
462 | - def getOopsVars(self): |
463 | - """See `IRunnableJob`.""" |
464 | - oops_vars = super(OCIRecipeBuildJobDerived, self).getOopsVars() |
465 | - oops_vars.extend([ |
466 | - ('job_type', self.context.job_type.title), |
467 | - ('build_id', self.context.build.id), |
468 | - ('owner_id', self.context.build.recipe.owner.id), |
469 | - ('project_name', self.context.build.recipe.ociproject.name) |
470 | - ]) |
471 | - return oops_vars |
472 | diff --git a/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py b/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py |
473 | new file mode 100644 |
474 | index 0000000..56d3592 |
475 | --- /dev/null |
476 | +++ b/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py |
477 | @@ -0,0 +1,312 @@ |
478 | +# Copyright 2015-2019 Canonical Ltd. This software is licensed under the |
479 | +# GNU Affero General Public License version 3 (see the file LICENSE). |
480 | + |
481 | +"""Tests for `OCIRecipeBuildBehavior`.""" |
482 | + |
483 | +from __future__ import absolute_import, print_function, unicode_literals |
484 | + |
485 | +__metaclass__ = type |
486 | + |
487 | +import json |
488 | +import os |
489 | +import shutil |
490 | +import tempfile |
491 | + |
492 | +from testtools import ExpectedException |
493 | +from twisted.internet import defer |
494 | +from zope.security.proxy import removeSecurityProxy |
495 | + |
496 | +from lp.buildmaster.enums import BuildStatus |
497 | +from lp.buildmaster.interactor import BuilderInteractor |
498 | +from lp.buildmaster.interfaces.builder import BuildDaemonError |
499 | +from lp.buildmaster.interfaces.buildfarmjobbehaviour import ( |
500 | + IBuildFarmJobBehaviour, |
501 | + ) |
502 | +from lp.buildmaster.tests.mock_slaves import WaitingSlave |
503 | +from lp.buildmaster.tests.test_buildfarmjobbehaviour import ( |
504 | + TestGetUploadMethodsMixin, |
505 | + ) |
506 | +from lp.oci.model.ocirecipebuildbehaviour import OCIRecipeBuildBehaviour |
507 | +from lp.services.config import config |
508 | +from lp.testing import TestCaseWithFactory |
509 | +from lp.testing.dbuser import dbuser |
510 | +from lp.testing.factory import LaunchpadObjectFactory |
511 | +from lp.testing.fakemethod import FakeMethod |
512 | +from lp.testing.layers import LaunchpadZopelessLayer |
513 | +from lp.testing.mail_helpers import pop_notifications |
514 | + |
515 | + |
516 | +class MakeOCIBuildMixin: |
517 | + |
518 | + def makeBuild(self): |
519 | + build = self.factory.makeOCIRecipeBuild() |
520 | + build.queueBuild() |
521 | + return build |
522 | + |
523 | + def makeUnmodifiableBuild(self): |
524 | + build = self.factory.makeOCIRecipeBuild() |
525 | + build.distro_arch_series = 'failed' |
526 | + build.queueBuild() |
527 | + return build |
528 | + |
529 | + |
530 | +class TestOCIBuildBehaviour(TestCaseWithFactory): |
531 | + |
532 | + layer = LaunchpadZopelessLayer |
533 | + |
534 | + def test_provides_interface(self): |
535 | + # OCIRecipeBuildBehaviour provides IBuildFarmJobBehaviour. |
536 | + job = OCIRecipeBuildBehaviour(None) |
537 | + self.assertProvides(job, IBuildFarmJobBehaviour) |
538 | + |
539 | + def test_adapts_IOCIRecipeBuild(self): |
540 | + # IBuildFarmJobBehaviour adapts an IOCIRecipeBuild. |
541 | + build = self.factory.makeOCIRecipeBuild() |
542 | + job = IBuildFarmJobBehaviour(build) |
543 | + self.assertProvides(job, IBuildFarmJobBehaviour) |
544 | + |
545 | + |
546 | +class TestHandleStatusForOCIRecipeBuild(MakeOCIBuildMixin, |
547 | + TestCaseWithFactory): |
548 | + # This is mostly copied from TestHandleStatusMixin, however |
549 | + # we can't use all of those tests, due to the way OCIRecipeBuildBehaviour |
550 | + # parses the file contents, rather than just retrieving all that are |
551 | + # available. There's also some differences in the filemap handling, as |
552 | + # we need a much more complex filemap here. |
553 | + |
554 | + layer = LaunchpadZopelessLayer |
555 | + |
556 | + def _createTestFile(self, name, content, hash): |
557 | + path = os.path.join(self.test_files_dir, name) |
558 | + with open(path, 'wb') as fp: |
559 | + fp.write(content) |
560 | + self.slave.valid_files[hash] = path |
561 | + |
562 | + def setUp(self): |
563 | + super(TestHandleStatusForOCIRecipeBuild, self).setUp() |
564 | + self.factory = LaunchpadObjectFactory() |
565 | + self.build = self.makeBuild() |
566 | + # For the moment, we require a builder for the build so that |
567 | + # handleStatus_OK can get a reference to the slave. |
568 | + self.builder = self.factory.makeBuilder() |
569 | + self.build.buildqueue_record.markAsBuilding(self.builder) |
570 | + self.slave = WaitingSlave('BuildStatus.OK') |
571 | + self.slave.valid_files['test_file_hash'] = '' |
572 | + self.interactor = BuilderInteractor() |
573 | + self.behaviour = self.interactor.getBuildBehaviour( |
574 | + self.build.buildqueue_record, self.builder, self.slave) |
575 | + |
576 | + # We overwrite the buildmaster root to use a temp directory. |
577 | + tempdir = tempfile.mkdtemp() |
578 | + self.addCleanup(shutil.rmtree, tempdir) |
579 | + self.upload_root = tempdir |
580 | + self.pushConfig('builddmaster', root=self.upload_root) |
581 | + |
582 | + # We stub out our builds getUploaderCommand() method so |
583 | + # we can check whether it was called as well as |
584 | + # verifySuccessfulUpload(). |
585 | + removeSecurityProxy(self.build).verifySuccessfulUpload = FakeMethod( |
586 | + result=True) |
587 | + |
588 | + digests = { |
589 | + "diff_id_1": { |
590 | + "digest": "digest_1", |
591 | + "source": "test/base_1", |
592 | + "layer_id": "layer_1" |
593 | + }, |
594 | + "diff_id_2": { |
595 | + "digest": "digest_2", |
596 | + "source": "", |
597 | + "layer_id": "layer_2" |
598 | + } |
599 | + } |
600 | + |
601 | + self.test_files_dir = tempfile.mkdtemp() |
602 | + self._createTestFile('buildlog', '', 'buildlog') |
603 | + self._createTestFile( |
604 | + 'manifest.json', |
605 | + '[{"Config": "config_file_1.json", ' |
606 | + '"Layers": ["layer_1/layer.tar", "layer_2/layer.tar"]}]', |
607 | + 'manifest_hash') |
608 | + self._createTestFile( |
609 | + 'digests.json', |
610 | + json.dumps(digests), |
611 | + 'digests_hash') |
612 | + self._createTestFile( |
613 | + 'config_file_1.json', |
614 | + '{"rootfs": {"diff_ids": ["diff_id_1", "diff_id_2"]}}', |
615 | + 'config_1_hash') |
616 | + self._createTestFile( |
617 | + 'layer_2.tar.gz', |
618 | + '', |
619 | + 'layer_2_hash' |
620 | + ) |
621 | + |
622 | + self.filemap = { |
623 | + 'manifest.json': 'manifest_hash', |
624 | + 'digests.json': 'digests_hash', |
625 | + 'config_file_1.json': 'config_1_hash', |
626 | + 'layer_1.tar.gz': 'layer_1_hash', |
627 | + 'layer_2.tar.gz': 'layer_2_hash' |
628 | + } |
629 | + self.factory.makeOCIFile( |
630 | + build=self.build, layer_file_digest=u'digest_1', |
631 | + content="retrieved from librarian") |
632 | + |
633 | + def assertResultCount(self, count, result): |
634 | + self.assertEqual( |
635 | + 1, len(os.listdir(os.path.join(self.upload_root, result)))) |
636 | + |
637 | + @defer.inlineCallbacks |
638 | + def test_handleStatus_OK_normal_image(self): |
639 | + with dbuser(config.builddmaster.dbuser): |
640 | + yield self.behaviour.handleStatus( |
641 | + self.build.buildqueue_record, 'OK', |
642 | + {'filemap': self.filemap}) |
643 | + self.assertEqual( |
644 | + ['buildlog', 'manifest_hash', 'digests_hash', 'config_1_hash', |
645 | + 'layer_2_hash'], |
646 | + self.slave._got_file_record) |
647 | + # This hash should not appear as it is already in the librarian |
648 | + self.assertNotIn('layer_1_hash', self.slave._got_file_record) |
649 | + self.assertEqual(BuildStatus.UPLOADING, self.build.status) |
650 | + self.assertResultCount(1, "incoming") |
651 | + |
652 | + # layer_1 should have been retrieved from the librarian |
653 | + layer_1_path = os.path.join( |
654 | + self.upload_root, |
655 | + "incoming", |
656 | + self.behaviour.getUploadDirLeaf(self.build.build_cookie), |
657 | + str(self.build.archive.id), |
658 | + self.build.distribution.name, |
659 | + "layer_1.tar.gz" |
660 | + ) |
661 | + with open(layer_1_path, 'rb') as layer_1_fp: |
662 | + contents = layer_1_fp.read() |
663 | + self.assertEqual(contents, b'retrieved from librarian') |
664 | + |
665 | + @defer.inlineCallbacks |
666 | + def test_handleStatus_OK_absolute_filepath(self): |
667 | + |
668 | + self._createTestFile( |
669 | + 'manifest.json', |
670 | + '[{"Config": "/notvalid/config_file_1.json", ' |
671 | + '"Layers": ["layer_1/layer.tar", "layer_2/layer.tar"]}]', |
672 | + 'manifest_hash') |
673 | + |
674 | + self.filemap['/notvalid/config_file_1.json'] = 'config_1_hash' |
675 | + |
676 | + # A filemap that tries to write to files outside of the upload |
677 | + # directory will not be collected. |
678 | + with ExpectedException( |
679 | + BuildDaemonError, |
680 | + "Build returned a file named " |
681 | + "'/notvalid/config_file_1.json'."): |
682 | + with dbuser(config.builddmaster.dbuser): |
683 | + yield self.behaviour.handleStatus( |
684 | + self.build.buildqueue_record, 'OK', |
685 | + {'filemap': self.filemap}) |
686 | + |
687 | + @defer.inlineCallbacks |
688 | + def test_handleStatus_OK_relative_filepath(self): |
689 | + |
690 | + self._createTestFile( |
691 | + 'manifest.json', |
692 | + '[{"Config": "../config_file_1.json", ' |
693 | + '"Layers": ["layer_1/layer.tar", "layer_2/layer.tar"]}]', |
694 | + 'manifest_hash') |
695 | + |
696 | + self.filemap['../config_file_1.json'] = 'config_1_hash' |
697 | + # A filemap that tries to write to files outside of |
698 | + # the upload directory will not be collected. |
699 | + with ExpectedException( |
700 | + BuildDaemonError, |
701 | + "Build returned a file named '../config_file_1.json'."): |
702 | + with dbuser(config.builddmaster.dbuser): |
703 | + yield self.behaviour.handleStatus( |
704 | + self.build.buildqueue_record, 'OK', |
705 | + {'filemap': self.filemap}) |
706 | + |
707 | + @defer.inlineCallbacks |
708 | + def test_handleStatus_OK_sets_build_log(self): |
709 | + # The build log is set during handleStatus. |
710 | + self.assertEqual(None, self.build.log) |
711 | + with dbuser(config.builddmaster.dbuser): |
712 | + yield self.behaviour.handleStatus( |
713 | + self.build.buildqueue_record, 'OK', |
714 | + {'filemap': self.filemap}) |
715 | + self.assertNotEqual(None, self.build.log) |
716 | + |
717 | + @defer.inlineCallbacks |
718 | + def test_handleStatus_ABORTED_cancels_cancelling(self): |
719 | + with dbuser(config.builddmaster.dbuser): |
720 | + self.build.updateStatus(BuildStatus.CANCELLING) |
721 | + yield self.behaviour.handleStatus( |
722 | + self.build.buildqueue_record, "ABORTED", {}) |
723 | + self.assertEqual(0, len(pop_notifications()), "Notifications received") |
724 | + self.assertEqual(BuildStatus.CANCELLED, self.build.status) |
725 | + |
726 | + @defer.inlineCallbacks |
727 | + def test_handleStatus_ABORTED_illegal_when_building(self): |
728 | + self.builder.vm_host = "fake_vm_host" |
729 | + self.behaviour = self.interactor.getBuildBehaviour( |
730 | + self.build.buildqueue_record, self.builder, self.slave) |
731 | + with dbuser(config.builddmaster.dbuser): |
732 | + self.build.updateStatus(BuildStatus.BUILDING) |
733 | + with ExpectedException( |
734 | + BuildDaemonError, |
735 | + "Build returned unexpected status: u'ABORTED'"): |
736 | + yield self.behaviour.handleStatus( |
737 | + self.build.buildqueue_record, "ABORTED", {}) |
738 | + |
739 | + @defer.inlineCallbacks |
740 | + def test_handleStatus_ABORTED_cancelling_sets_build_log(self): |
741 | + # If a build is intentionally cancelled, the build log is set. |
742 | + self.assertEqual(None, self.build.log) |
743 | + with dbuser(config.builddmaster.dbuser): |
744 | + self.build.updateStatus(BuildStatus.CANCELLING) |
745 | + yield self.behaviour.handleStatus( |
746 | + self.build.buildqueue_record, "ABORTED", {}) |
747 | + self.assertNotEqual(None, self.build.log) |
748 | + |
749 | + @defer.inlineCallbacks |
750 | + def test_date_finished_set(self): |
751 | + # The date finished is updated during handleStatus_OK. |
752 | + self.assertEqual(None, self.build.date_finished) |
753 | + with dbuser(config.builddmaster.dbuser): |
754 | + yield self.behaviour.handleStatus( |
755 | + self.build.buildqueue_record, 'OK', |
756 | + {'filemap': self.filemap}) |
757 | + self.assertNotEqual(None, self.build.date_finished) |
758 | + |
759 | + @defer.inlineCallbacks |
760 | + def test_givenback_collection(self): |
761 | + with ExpectedException( |
762 | + BuildDaemonError, |
763 | + "Build returned unexpected status: u'GIVENBACK'"): |
764 | + with dbuser(config.builddmaster.dbuser): |
765 | + yield self.behaviour.handleStatus( |
766 | + self.build.buildqueue_record, "GIVENBACK", {}) |
767 | + |
768 | + @defer.inlineCallbacks |
769 | + def test_builderfail_collection(self): |
770 | + with ExpectedException( |
771 | + BuildDaemonError, |
772 | + "Build returned unexpected status: u'BUILDERFAIL'"): |
773 | + with dbuser(config.builddmaster.dbuser): |
774 | + yield self.behaviour.handleStatus( |
775 | + self.build.buildqueue_record, "BUILDERFAIL", {}) |
776 | + |
777 | + @defer.inlineCallbacks |
778 | + def test_invalid_status_collection(self): |
779 | + with ExpectedException( |
780 | + BuildDaemonError, |
781 | + "Build returned unexpected status: u'BORKED'"): |
782 | + with dbuser(config.builddmaster.dbuser): |
783 | + yield self.behaviour.handleStatus( |
784 | + self.build.buildqueue_record, "BORKED", {}) |
785 | + |
786 | + |
787 | +class TestGetUploadMethodsForOCIRecipeBuild( |
788 | + MakeOCIBuildMixin, TestGetUploadMethodsMixin, TestCaseWithFactory): |
789 | + """IPackageBuild.getUpload-related methods work with OCI recipe builds.""" |
790 | diff --git a/lib/lp/oci/tests/test_ocirecipebuildjob.py b/lib/lp/oci/tests/test_ocirecipebuildjob.py |
791 | deleted file mode 100644 |
792 | index cda3eef..0000000 |
793 | --- a/lib/lp/oci/tests/test_ocirecipebuildjob.py |
794 | +++ /dev/null |
795 | @@ -1,49 +0,0 @@ |
796 | -# Copyright 2019 Canonical Ltd. This software is licensed under the |
797 | -# GNU Affero General Public License version 3 (see the file LICENSE). |
798 | - |
799 | -"""OCIRecipeBuildJob tests""" |
800 | - |
801 | -from __future__ import absolute_import, print_function, unicode_literals |
802 | - |
803 | -__metaclass__ = type |
804 | - |
805 | - |
806 | -from lp.oci.interfaces.ocirecipebuildjob import IOCIRecipeBuildJob |
807 | -from lp.oci.model.ocirecipebuildjob import ( |
808 | - OCIRecipeBuildJob, |
809 | - OCIRecipeBuildJobDerived, |
810 | - OCIRecipeBuildJobType, |
811 | - ) |
812 | -from lp.testing import TestCaseWithFactory |
813 | -from lp.testing.layers import DatabaseFunctionalLayer |
814 | - |
815 | - |
816 | -class FakeOCIBuildJob(OCIRecipeBuildJobDerived): |
817 | - """For testing OCIRecipeBuildJobDerived without a child class.""" |
818 | - |
819 | - |
820 | -class TestOCIRecipeBuildJob(TestCaseWithFactory): |
821 | - |
822 | - layer = DatabaseFunctionalLayer |
823 | - |
824 | - def test_provides_interface(self): |
825 | - oci_build = self.factory.makeOCIRecipeBuild() |
826 | - self.assertProvides( |
827 | - OCIRecipeBuildJob( |
828 | - oci_build, OCIRecipeBuildJobType.REGISTRY_UPLOAD, {}), |
829 | - IOCIRecipeBuildJob) |
830 | - |
831 | - def test_getOopsVars(self): |
832 | - oci_build = self.factory.makeOCIRecipeBuild() |
833 | - build_job = OCIRecipeBuildJob( |
834 | - oci_build, OCIRecipeBuildJobType.REGISTRY_UPLOAD, {}) |
835 | - derived = FakeOCIBuildJob(build_job) |
836 | - oops = derived.getOopsVars() |
837 | - expected = [ |
838 | - ('job_id', build_job.job.id), |
839 | - ('job_type', build_job.job_type.title), |
840 | - ('build_id', oci_build.id), |
841 | - ('owner_id', oci_build.recipe.owner.id), |
842 | - ('project_name', oci_build.recipe.ociproject.name), |
843 | - ] |
844 | - self.assertEqual(expected, oops) |
845 | diff --git a/lib/lp/testing/factory.py b/lib/lp/testing/factory.py |
846 | index f0788d8..4a25c88 100644 |
847 | --- a/lib/lp/testing/factory.py |
848 | +++ b/lib/lp/testing/factory.py |
849 | @@ -5005,12 +5005,13 @@ class BareLaunchpadObjectFactory(ObjectFactory): |
850 | return oci_build |
851 | |
852 | def makeOCIFile(self, build=None, library_file=None, |
853 | - layer_file_digest=None): |
854 | + layer_file_digest=None, content=None, filename=None): |
855 | """Make a new OCIFile.""" |
856 | if build is None: |
857 | build = self.makeOCIRecipeBuild() |
858 | if library_file is None: |
859 | - library_file = self.makeLibraryFileAlias() |
860 | + library_file = self.makeLibraryFileAlias( |
861 | + content=content, filename=filename) |
862 | return OCIFile(build=build, library_file=library_file, |
863 | layer_file_digest=layer_file_digest) |
864 | |
865 | diff --git a/lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py b/lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py |
866 | index b387251..32dd553 100644 |
867 | --- a/lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py |
868 | +++ b/lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py |
869 | @@ -138,7 +138,7 @@ class TestTranslationTemplatesBuildBehaviour( |
870 | buildqueue = FakeBuildQueue(behaviour) |
871 | path = behaviour.templates_tarball_path |
872 | # Poke the file we're expecting into the mock slave. |
873 | - behaviour._slave.valid_file_hashes.append(path) |
874 | + behaviour._slave.valid_files[path] = '' |
875 | |
876 | def got_tarball(filename): |
877 | tarball = open(filename, 'r') |