Merge ~twom/launchpad:oci-upload-job into launchpad:master
- Git
- lp:~twom/launchpad
- oci-upload-job
- Merge into master
Proposed by
Tom Wardill
Status: | Merged | ||||
---|---|---|---|---|---|
Approved by: | Tom Wardill | ||||
Approved revision: | 2d7644b069e88d91c44e527190debc2833babdef | ||||
Merge reported by: | Otto Co-Pilot | ||||
Merged at revision: | not available | ||||
Proposed branch: | ~twom/launchpad:oci-upload-job | ||||
Merge into: | launchpad:master | ||||
Diff against target: |
622 lines (+514/-17) 8 files modified
database/schema/security.cfg (+1/-0) lib/lp/archiveuploader/ocirecipeupload.py (+100/-0) lib/lp/archiveuploader/tests/test_ocirecipeupload.py (+112/-0) lib/lp/archiveuploader/uploadprocessor.py (+30/-0) lib/lp/oci/interfaces/ocirecipebuildjob.py (+34/-0) lib/lp/oci/model/ocirecipebuild.py (+48/-17) lib/lp/oci/model/ocirecipebuildjob.py (+140/-0) lib/lp/oci/tests/test_ocirecipebuildjob.py (+49/-0) |
||||
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Colin Watson (community) | Approve | ||
Thiago F. Pappacena (community) | Approve | ||
Review via email: mp+381522@code.launchpad.net |
Commit message
Process completed OCI builds for upload to librarian
Description of the change
To post a comment you must log in.
~twom/launchpad:oci-upload-job
updated
- 9fdbbad... by Tom Wardill
-
Use cachedproperty
- f78988c... by Tom Wardill
-
Clear property caches in tests
- 82fb2d2... by Tom Wardill
-
Use IStore for read operations
Revision history for this message
Tom Wardill (twom) : | # |
Revision history for this message
Colin Watson (cjwatson) : | # |
Revision history for this message
Colin Watson (cjwatson) : | # |
review:
Needs Information
~twom/launchpad:oci-upload-job
updated
- 426c9ef... by Tom Wardill
-
Remove UPDATE
- b935fe4... by Tom Wardill
-
Don't need the import patch
- 6a796ef... by Tom Wardill
-
Header updates
- 92d7a2a... by Tom Wardill
-
More debugging
- ba659f5... by Tom Wardill
-
Rename processOCIRecip
eBuild to processOCIRecipe - 7b81b28... by Tom Wardill
-
Small typo fixes
- 460078c... by Tom Wardill
-
Future comments
- 8a63981... by Tom Wardill
-
Better repr, oops vars
Revision history for this message
Tom Wardill (twom) : | # |
Revision history for this message
Colin Watson (cjwatson) wrote : | # |
Looks reasonable to me now, thanks.
review:
Approve
~twom/launchpad:oci-upload-job
updated
- 2d7644b... by Tom Wardill
-
Fix typo
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/database/schema/security.cfg b/database/schema/security.cfg |
2 | index f5cca8d..d3ee546 100644 |
3 | --- a/database/schema/security.cfg |
4 | +++ b/database/schema/security.cfg |
5 | @@ -1432,6 +1432,7 @@ public.message = SELECT, INSERT |
6 | public.messagechunk = SELECT, INSERT |
7 | public.milestone = SELECT |
8 | public.milestonetag = SELECT |
9 | +public.ocifile = SELECT, INSERT |
10 | public.ociproject = SELECT |
11 | public.ociprojectname = SELECT |
12 | public.ociprojectseries = SELECT |
13 | diff --git a/lib/lp/archiveuploader/ocirecipeupload.py b/lib/lp/archiveuploader/ocirecipeupload.py |
14 | new file mode 100644 |
15 | index 0000000..c3db6cb |
16 | --- /dev/null |
17 | +++ b/lib/lp/archiveuploader/ocirecipeupload.py |
18 | @@ -0,0 +1,100 @@ |
19 | +# Copyright 2020 Canonical Ltd. This software is licensed under the |
20 | +# GNU Affero General Public License version 3 (see the file LICENSE). |
21 | + |
22 | +"""Upload OCI build artifacts to the librarian.""" |
23 | + |
24 | +from __future__ import absolute_import, print_function, unicode_literals |
25 | + |
26 | +__metaclass__ = type |
27 | +__all__ = ['OCIRecipeUpload'] |
28 | + |
29 | + |
30 | +import json |
31 | +import os |
32 | + |
33 | +import scandir |
34 | +from zope.component import getUtility |
35 | + |
36 | +from lp.archiveuploader.utils import UploadError |
37 | +from lp.buildmaster.enums import BuildStatus |
38 | +from lp.services.helpers import filenameToContentType |
39 | +from lp.services.librarian.interfaces import ILibraryFileAliasSet |
40 | + |
41 | + |
42 | +class OCIRecipeUpload: |
43 | + """An OCI image upload.""" |
44 | + |
45 | + def __init__(self, upload_path, logger): |
46 | + """Create a `OCIRecipeUpload`. |
47 | + |
48 | + :param upload_path: A directory containing files to upload. |
49 | + :param logger: The logger to be used. |
50 | + """ |
51 | + self.upload_path = upload_path |
52 | + self.logger = logger |
53 | + |
54 | + self.librarian = getUtility(ILibraryFileAliasSet) |
55 | + |
56 | + def process(self, build): |
57 | + """Process this upload, loading it into the database.""" |
58 | + self.logger.debug("Beginning processing.") |
59 | + |
60 | + # Find digest file |
61 | + for dirpath, _, filenames in scandir.walk(self.upload_path): |
62 | + if dirpath == self.upload_path: |
63 | + # All relevant files will be in a subdirectory. |
64 | + continue |
65 | + if 'digests.json' not in filenames: |
66 | + continue |
67 | + # Open the digest file |
68 | + digest_path = os.path.join(dirpath, 'digests.json') |
69 | + self.logger.debug("Digest path: {}".format(digest_path)) |
70 | + with open(digest_path, 'r') as digest_fp: |
71 | + digests = json.load(digest_fp) |
72 | + |
73 | + # Foreach id in digest file, find matching layer |
74 | + for single_digest in digests: |
75 | + for diff_id, data in single_digest.items(): |
76 | + digest = data["digest"] |
77 | + layer_id = data["layer_id"] |
78 | + layer_path = os.path.join( |
79 | + dirpath, |
80 | + "{}.tar.gz".format(layer_id) |
81 | + ) |
82 | + self.logger.debug("Layer path: {}".format(layer_path)) |
83 | + if not os.path.exists(layer_path): |
84 | + raise UploadError( |
85 | + "Missing layer file: {}.".format(layer_id)) |
86 | + # Upload layer |
87 | + libraryfile = self.librarian.create( |
88 | + os.path.basename(layer_path), |
89 | + os.stat(layer_path).st_size, |
90 | + open(layer_path, "rb"), |
91 | + filenameToContentType(layer_path), |
92 | + restricted=build.is_private) |
93 | + build.addFile(libraryfile, layer_file_digest=digest) |
94 | + # Upload all json files |
95 | + for filename in filenames: |
96 | + if filename.endswith('.json'): |
97 | + file_path = os.path.join(dirpath, filename) |
98 | + self.logger.debug("JSON file: {}".format(file_path)) |
99 | + libraryfile = self.librarian.create( |
100 | + os.path.basename(file_path), |
101 | + os.stat(file_path).st_size, |
102 | + open(file_path, "rb"), |
103 | + filenameToContentType(file_path), |
104 | + restricted=build.is_private) |
105 | + # This doesn't have a digest as it's not a layer file. |
106 | + build.addFile(libraryfile, layer_file_digest=None) |
107 | + # We've found digest, we can stop now |
108 | + break |
109 | + else: |
110 | + # If we get here, we've not got a digests.json, |
111 | + # something has gone wrong |
112 | + raise UploadError("Build did not produce a digests.json.") |
113 | + |
114 | + # The master verifies the status to confirm successful upload. |
115 | + self.logger.debug("Updating %s" % build.title) |
116 | + build.updateStatus(BuildStatus.FULLYBUILT) |
117 | + |
118 | + self.logger.debug("Finished upload.") |
119 | diff --git a/lib/lp/archiveuploader/tests/test_ocirecipeupload.py b/lib/lp/archiveuploader/tests/test_ocirecipeupload.py |
120 | new file mode 100644 |
121 | index 0000000..d80be3a |
122 | --- /dev/null |
123 | +++ b/lib/lp/archiveuploader/tests/test_ocirecipeupload.py |
124 | @@ -0,0 +1,112 @@ |
125 | +# Copyright 2020 Canonical Ltd. This software is licensed under the |
126 | +# GNU Affero General Public License version 3 (see the file LICENSE). |
127 | + |
128 | +"""Tests for `OCIRecipeUpload`.""" |
129 | + |
130 | +from __future__ import absolute_import, print_function, unicode_literals |
131 | + |
132 | +__metaclass__ = type |
133 | + |
134 | +import json |
135 | +import os |
136 | + |
137 | +from storm.store import Store |
138 | + |
139 | +from lp.archiveuploader.tests.test_uploadprocessor import ( |
140 | + TestUploadProcessorBase, |
141 | + ) |
142 | +from lp.archiveuploader.uploadprocessor import ( |
143 | + UploadHandler, |
144 | + UploadStatusEnum, |
145 | + ) |
146 | +from lp.buildmaster.enums import BuildStatus |
147 | +from lp.services.propertycache import get_property_cache |
148 | +from lp.services.osutils import write_file |
149 | + |
150 | + |
151 | +class TestOCIRecipeUploads(TestUploadProcessorBase): |
152 | + |
153 | + def setUp(self): |
154 | + super(TestOCIRecipeUploads, self).setUp() |
155 | + |
156 | + self.setupBreezy() |
157 | + |
158 | + self.switchToAdmin() |
159 | + self.build = self.factory.makeOCIRecipeBuild() |
160 | + Store.of(self.build).flush() |
161 | + self.switchToUploader() |
162 | + self.options.context = "buildd" |
163 | + |
164 | + self.uploadprocessor = self.getUploadProcessor( |
165 | + self.layer.txn, builds=True) |
166 | + |
167 | + self.digests = [{ |
168 | + "diff_id_1": { |
169 | + "digest": "digest_1", |
170 | + "source": "test/base_1", |
171 | + "layer_id": "layer_1" |
172 | + }, |
173 | + "diff_id_2": { |
174 | + "digest": "digest_2", |
175 | + "source": "", |
176 | + "layer_id": "layer_2" |
177 | + } |
178 | + }] |
179 | + |
180 | + def test_sets_build_and_state(self): |
181 | + # The upload processor uploads files and sets the correct status. |
182 | + self.assertFalse(self.build.verifySuccessfulUpload()) |
183 | + del get_property_cache(self.build).manifest |
184 | + del get_property_cache(self.build).digests |
185 | + upload_dir = os.path.join( |
186 | + self.incoming_folder, "test", str(self.build.id), "ubuntu") |
187 | + write_file(os.path.join(upload_dir, "layer_1.tar.gz"), b"layer_1") |
188 | + write_file(os.path.join(upload_dir, "layer_2.tar.gz"), b"layer_2") |
189 | + write_file( |
190 | + os.path.join(upload_dir, "digests.json"), json.dumps(self.digests)) |
191 | + write_file(os.path.join(upload_dir, "manifest.json"), b"manifest") |
192 | + handler = UploadHandler.forProcessor( |
193 | + self.uploadprocessor, self.incoming_folder, "test", self.build) |
194 | + result = handler.processOCIRecipe(self.log) |
195 | + self.assertEqual( |
196 | + UploadStatusEnum.ACCEPTED, result, |
197 | + "OCI upload failed\nGot: %s" % self.log.getLogBuffer()) |
198 | + self.assertEqual(BuildStatus.FULLYBUILT, self.build.status) |
199 | + self.assertTrue(self.build.verifySuccessfulUpload()) |
200 | + |
201 | + def test_requires_digests(self): |
202 | + # The upload processor fails if the upload does not contain the |
203 | + # digests file |
204 | + self.assertFalse(self.build.verifySuccessfulUpload()) |
205 | + del get_property_cache(self.build).manifest |
206 | + del get_property_cache(self.build).digests |
207 | + upload_dir = os.path.join( |
208 | + self.incoming_folder, "test", str(self.build.id), "ubuntu") |
209 | + write_file(os.path.join(upload_dir, "layer_1.tar.gz"), b"layer_1") |
210 | + handler = UploadHandler.forProcessor( |
211 | + self.uploadprocessor, self.incoming_folder, "test", self.build) |
212 | + result = handler.processOCIRecipe(self.log) |
213 | + self.assertEqual(UploadStatusEnum.REJECTED, result) |
214 | + self.assertIn( |
215 | + "ERROR Build did not produce a digests.json.", |
216 | + self.log.getLogBuffer()) |
217 | + self.assertFalse(self.build.verifySuccessfulUpload()) |
218 | + |
219 | + def test_missing_layer_file(self): |
220 | + # The digests.json specifies a layer file that is missing |
221 | + self.assertFalse(self.build.verifySuccessfulUpload()) |
222 | + del get_property_cache(self.build).manifest |
223 | + del get_property_cache(self.build).digests |
224 | + upload_dir = os.path.join( |
225 | + self.incoming_folder, "test", str(self.build.id), "ubuntu") |
226 | + write_file(os.path.join(upload_dir, "layer_1.tar.gz"), b"layer_1") |
227 | + write_file( |
228 | + os.path.join(upload_dir, "digests.json"), json.dumps(self.digests)) |
229 | + handler = UploadHandler.forProcessor( |
230 | + self.uploadprocessor, self.incoming_folder, "test", self.build) |
231 | + result = handler.processOCIRecipe(self.log) |
232 | + self.assertEqual(UploadStatusEnum.REJECTED, result) |
233 | + self.assertIn( |
234 | + "ERROR Missing layer file: layer_2.", |
235 | + self.log.getLogBuffer()) |
236 | + self.assertFalse(self.build.verifySuccessfulUpload()) |
237 | diff --git a/lib/lp/archiveuploader/uploadprocessor.py b/lib/lp/archiveuploader/uploadprocessor.py |
238 | index 3aee65c..752df0a 100644 |
239 | --- a/lib/lp/archiveuploader/uploadprocessor.py |
240 | +++ b/lib/lp/archiveuploader/uploadprocessor.py |
241 | @@ -61,6 +61,7 @@ from lp.archiveuploader.nascentupload import ( |
242 | EarlyReturnUploadError, |
243 | NascentUpload, |
244 | ) |
245 | +from lp.archiveuploader.ocirecipeupload import OCIRecipeUpload |
246 | from lp.archiveuploader.snapupload import SnapUpload |
247 | from lp.archiveuploader.uploadpolicy import ( |
248 | BuildDaemonUploadPolicy, |
249 | @@ -72,6 +73,7 @@ from lp.buildmaster.interfaces.buildfarmjob import ISpecificBuildFarmJobSource |
250 | from lp.code.interfaces.sourcepackagerecipebuild import ( |
251 | ISourcePackageRecipeBuild, |
252 | ) |
253 | +from lp.oci.interfaces.ocirecipebuild import IOCIRecipeBuild |
254 | from lp.registry.interfaces.distribution import IDistributionSet |
255 | from lp.registry.interfaces.person import IPersonSet |
256 | from lp.services.log.logger import BufferLogger |
257 | @@ -630,6 +632,32 @@ class BuildUploadHandler(UploadHandler): |
258 | self.processor.ztm.abort() |
259 | raise |
260 | |
261 | + def processOCIRecipe(self, logger=None): |
262 | + """Process an OCI image upload.""" |
263 | + assert IOCIRecipeBuild.providedBy(self.build) |
264 | + if logger is None: |
265 | + logger = self.processor.log |
266 | + try: |
267 | + logger.info( |
268 | + "Processing OCI Image upload {}".format(self.upload_path)) |
269 | + OCIRecipeUpload(self.upload_path, logger).process(self.build) |
270 | + |
271 | + if self.processor.dry_run: |
272 | + logger.info("Dry run, aborting transaction.") |
273 | + self.processor.ztm.abort() |
274 | + else: |
275 | + logger.info( |
276 | + "Committing the transaction and any mails associated " |
277 | + "with this upload.") |
278 | + self.processor.ztm.commit() |
279 | + return UploadStatusEnum.ACCEPTED |
280 | + except UploadError as e: |
281 | + logger.error(str(e)) |
282 | + return UploadStatusEnum.REJECTED |
283 | + except BaseException: |
284 | + self.processor.ztm.abort() |
285 | + raise |
286 | + |
287 | def process(self): |
288 | """Process an upload that is the result of a build. |
289 | |
290 | @@ -671,6 +699,8 @@ class BuildUploadHandler(UploadHandler): |
291 | result = self.processLiveFS(logger) |
292 | elif ISnapBuild.providedBy(self.build): |
293 | result = self.processSnap(logger) |
294 | + elif IOCIRecipeBuild.providedBy(self.build): |
295 | + result = self.processOCIRecipe(logger) |
296 | else: |
297 | self.processor.log.debug("Build %s found" % self.build.id) |
298 | [changes_file] = self.locateChangesFiles() |
299 | diff --git a/lib/lp/oci/interfaces/ocirecipebuildjob.py b/lib/lp/oci/interfaces/ocirecipebuildjob.py |
300 | new file mode 100644 |
301 | index 0000000..6c25b73 |
302 | --- /dev/null |
303 | +++ b/lib/lp/oci/interfaces/ocirecipebuildjob.py |
304 | @@ -0,0 +1,34 @@ |
305 | +# Copyright 2019 Canonical Ltd. This software is licensed under the |
306 | +# GNU Affero General Public License version 3 (see the file LICENSE). |
307 | + |
308 | +"""OCIRecipe build job interfaces""" |
309 | + |
310 | +from __future__ import absolute_import, print_function, unicode_literals |
311 | + |
312 | +__metaclass__ = type |
313 | +__all__ = [ |
314 | + 'IOCIRecipeBuildJob', |
315 | + ] |
316 | + |
317 | +from lazr.restful.fields import Reference |
318 | +from zope.interface import ( |
319 | + Attribute, |
320 | + Interface, |
321 | + ) |
322 | + |
323 | +from lp import _ |
324 | +from lp.oci.interfaces.ocirecipebuild import IOCIRecipeBuild |
325 | +from lp.services.job.interfaces.job import IJob |
326 | + |
327 | + |
328 | +class IOCIRecipeBuildJob(Interface): |
329 | + """A job related to an OCI image.""" |
330 | + job = Reference( |
331 | + title=_("The common Job attributes."), schema=IJob, |
332 | + required=True, readonly=True) |
333 | + |
334 | + build = Reference( |
335 | + title=_("The OCI Recipe Build to use for this job."), |
336 | + schema=IOCIRecipeBuild, required=True, readonly=True) |
337 | + |
338 | + json_data = Attribute(_("A dict of data about the job.")) |
339 | diff --git a/lib/lp/oci/model/ocirecipebuild.py b/lib/lp/oci/model/ocirecipebuild.py |
340 | index c36fce4..12046d6 100644 |
341 | --- a/lib/lp/oci/model/ocirecipebuild.py |
342 | +++ b/lib/lp/oci/model/ocirecipebuild.py |
343 | @@ -208,23 +208,6 @@ class OCIRecipeBuild(PackageBuildMixin, Storm): |
344 | return result |
345 | raise NotFoundError(filename) |
346 | |
347 | - def getLayerFileByDigest(self, layer_file_digest): |
348 | - file_object = Store.of(self).find( |
349 | - (OCIFile, LibraryFileAlias, LibraryFileContent), |
350 | - OCIFile.build == self.id, |
351 | - LibraryFileAlias.id == OCIFile.library_file_id, |
352 | - LibraryFileContent.id == LibraryFileAlias.contentID, |
353 | - OCIFile.layer_file_digest == layer_file_digest).one() |
354 | - if file_object is not None: |
355 | - return file_object |
356 | - raise NotFoundError(layer_file_digest) |
357 | - |
358 | - def addFile(self, lfa, layer_file_digest=None): |
359 | - oci_file = OCIFile( |
360 | - build=self, library_file=lfa, layer_file_digest=layer_file_digest) |
361 | - IMasterStore(OCIFile).add(oci_file) |
362 | - return oci_file |
363 | - |
364 | @cachedproperty |
365 | def eta(self): |
366 | """The datetime when the build job is estimated to complete. |
367 | @@ -308,6 +291,54 @@ class OCIRecipeBuild(PackageBuildMixin, Storm): |
368 | return |
369 | # XXX twom 2019-12-11 This should send mail |
370 | |
371 | + def getLayerFileByDigest(self, layer_file_digest): |
372 | + file_object = Store.of(self).find( |
373 | + (OCIFile, LibraryFileAlias, LibraryFileContent), |
374 | + OCIFile.build == self.id, |
375 | + LibraryFileAlias.id == OCIFile.library_file_id, |
376 | + LibraryFileContent.id == LibraryFileAlias.contentID, |
377 | + OCIFile.layer_file_digest == layer_file_digest).one() |
378 | + if file_object is not None: |
379 | + return file_object |
380 | + raise NotFoundError(layer_file_digest) |
381 | + |
382 | + def addFile(self, lfa, layer_file_digest=None): |
383 | + oci_file = OCIFile( |
384 | + build=self, library_file=lfa, layer_file_digest=layer_file_digest) |
385 | + IMasterStore(OCIFile).add(oci_file) |
386 | + return oci_file |
387 | + |
388 | + @cachedproperty |
389 | + def manifest(self): |
390 | + result = Store.of(self).find( |
391 | + (OCIFile, LibraryFileAlias, LibraryFileContent), |
392 | + OCIFile.build == self.id, |
393 | + LibraryFileAlias.id == OCIFile.library_file_id, |
394 | + LibraryFileContent.id == LibraryFileAlias.contentID, |
395 | + LibraryFileAlias.filename == 'manifest.json') |
396 | + return result.one() |
397 | + |
398 | + @cachedproperty |
399 | + def digests(self): |
400 | + result = Store.of(self).find( |
401 | + (OCIFile, LibraryFileAlias, LibraryFileContent), |
402 | + OCIFile.build == self.id, |
403 | + LibraryFileAlias.id == OCIFile.library_file_id, |
404 | + LibraryFileContent.id == LibraryFileAlias.contentID, |
405 | + LibraryFileAlias.filename == 'digests.json') |
406 | + return result.one() |
407 | + |
408 | + def verifySuccessfulUpload(self): |
409 | + """See `IPackageBuild`.""" |
410 | + layer_files = Store.of(self).find( |
411 | + OCIFile, |
412 | + OCIFile.build == self.id, |
413 | + OCIFile.layer_file_digest is not None) |
414 | + layer_files_present = not layer_files.is_empty() |
415 | + metadata_present = (self.manifest is not None |
416 | + and self.digests is not None) |
417 | + return layer_files_present and metadata_present |
418 | + |
419 | |
420 | @implementer(IOCIRecipeBuildSet) |
421 | class OCIRecipeBuildSet(SpecificBuildFarmJobSourceMixin): |
422 | diff --git a/lib/lp/oci/model/ocirecipebuildjob.py b/lib/lp/oci/model/ocirecipebuildjob.py |
423 | new file mode 100644 |
424 | index 0000000..bb6fc95 |
425 | --- /dev/null |
426 | +++ b/lib/lp/oci/model/ocirecipebuildjob.py |
427 | @@ -0,0 +1,140 @@ |
428 | +# Copyright 2020 Canonical Ltd. This software is licensed under the |
429 | +# GNU Affero General Public License version 3 (see the file LICENSE). |
430 | + |
431 | +"""OCIRecipe build jobs.""" |
432 | + |
433 | +from __future__ import absolute_import, print_function, unicode_literals |
434 | + |
435 | +__metaclass__ = type |
436 | +__all__ = [ |
437 | + 'OCIRecipeBuildJob', |
438 | + 'OCIRecipeBuildJobType', |
439 | + ] |
440 | + |
441 | +from lazr.delegates import delegate_to |
442 | +from lazr.enum import ( |
443 | + DBEnumeratedType, |
444 | + DBItem, |
445 | + ) |
446 | +from storm.locals import ( |
447 | + Int, |
448 | + JSON, |
449 | + Reference, |
450 | + ) |
451 | +from zope.interface import implementer |
452 | + |
453 | +from lp.app.errors import NotFoundError |
454 | +from lp.oci.interfaces.ocirecipebuildjob import IOCIRecipeBuildJob |
455 | +from lp.services.database.enumcol import DBEnum |
456 | +from lp.services.database.interfaces import IStore |
457 | +from lp.services.database.stormbase import StormBase |
458 | +from lp.services.job.model.job import ( |
459 | + EnumeratedSubclass, |
460 | + Job, |
461 | + ) |
462 | +from lp.services.job.runner import BaseRunnableJob |
463 | + |
464 | + |
465 | +class OCIRecipeBuildJobType(DBEnumeratedType): |
466 | + """Values that `OCIBuildJobType.job_type` can take.""" |
467 | + |
468 | + # XXX twom (2020-04-02) This does not currently have a concrete |
469 | + # implementation, awaiting registry upload. |
470 | + |
471 | + REGISTRY_UPLOAD = DBItem(0, """ |
472 | + Registry upload |
473 | + |
474 | + This job uploads an OCI Image to the registry. |
475 | + """) |
476 | + |
477 | + |
478 | +@implementer(IOCIRecipeBuildJob) |
479 | +class OCIRecipeBuildJob(StormBase): |
480 | + """See `IOCIRecipeBuildJob`.""" |
481 | + |
482 | + __storm_table__ = 'OCIRecipeBuildJob' |
483 | + |
484 | + job_id = Int(name='job', primary=True, allow_none=False) |
485 | + job = Reference(job_id, 'Job.id') |
486 | + |
487 | + build_id = Int(name='build', allow_none=False) |
488 | + build = Reference(build_id, 'OCIRecipeBuild.id') |
489 | + |
490 | + job_type = DBEnum(enum=OCIRecipeBuildJobType, allow_none=True) |
491 | + |
492 | + json_data = JSON('json_data', allow_none=False) |
493 | + |
494 | + def __init__(self, build, job_type, json_data, **job_args): |
495 | + """Constructor. |
496 | + |
497 | + Extra keyword arguments are used to construct the underlying Job |
498 | + object. |
499 | + |
500 | + :param build: The `IOCIRecipeBuild` this job relates to. |
501 | + :param job_type: The `OCIRecipeBuildJobType` of this job. |
502 | + :param json_data: The type-specific variables, as a JSON-compatible |
503 | + dict. |
504 | + """ |
505 | + super(OCIRecipeBuildJob, self).__init__() |
506 | + self.job = Job(**job_args) |
507 | + self.build = build |
508 | + self.job_type = job_type |
509 | + self.json_data = json_data |
510 | + |
511 | + def makeDerived(self): |
512 | + return OCIRecipeBuildJob.makeSubclass(self) |
513 | + |
514 | + |
515 | +@delegate_to(IOCIRecipeBuildJob) |
516 | +class OCIRecipeBuildJobDerived(BaseRunnableJob): |
517 | + |
518 | + __metaclass__ = EnumeratedSubclass |
519 | + |
520 | + def __init__(self, oci_build_job): |
521 | + self.context = oci_build_job |
522 | + |
523 | + def __repr__(self): |
524 | + """An informative representation of the job.""" |
525 | + build = self.build |
526 | + return "<%s for ~%s/%s/+oci/%s/+recipe/%s/+build/%d>" % ( |
527 | + self.__class__.__name__, build.recipe.owner.name, |
528 | + build.recipe.oci_project.pillar.name, |
529 | + build.recipe.oci_project.name, build.recipe.name, build.id) |
530 | + |
531 | + @classmethod |
532 | + def get(cls, job_id): |
533 | + """Get a job by id. |
534 | + |
535 | + :return: The `OCIRecipeBuildJob` with the specified id, as the current |
536 | + `OCIRecipeBuildJobDerived` subclass. |
537 | + :raises: `NotFoundError` if there is no job with the specified id, |
538 | + or its `job_type` does not match the desired subclass. |
539 | + """ |
540 | + oci_build_job = IStore(OCIRecipeBuildJob).get( |
541 | + OCIRecipeBuildJob, job_id) |
542 | + if oci_build_job.job_type != cls.class_job_type: |
543 | + raise NotFoundError( |
544 | + "No object found with id %d and type %s" % |
545 | + (job_id, cls.class_job_type.title)) |
546 | + return cls(oci_build_job) |
547 | + |
548 | + @classmethod |
549 | + def iterReady(cls): |
550 | + """See `IJobSource`.""" |
551 | + jobs = IStore(OCIRecipeBuildJob).find( |
552 | + OCIRecipeBuildJob, |
553 | + OCIRecipeBuildJob.job_type == cls.class_job_type, |
554 | + OCIRecipeBuildJob.job == Job.id, |
555 | + Job.id.is_in(Job.ready_jobs)) |
556 | + return (cls(job) for job in jobs) |
557 | + |
558 | + def getOopsVars(self): |
559 | + """See `IRunnableJob`.""" |
560 | + oops_vars = super(OCIRecipeBuildJobDerived, self).getOopsVars() |
561 | + oops_vars.extend([ |
562 | + ('job_type', self.context.job_type.title), |
563 | + ('build_id', self.context.build.id), |
564 | + ('recipe_owner_id', self.context.build.recipe.owner.id), |
565 | + ('oci_project_name', self.context.build.recipe.oci_project.name) |
566 | + ]) |
567 | + return oops_vars |
568 | diff --git a/lib/lp/oci/tests/test_ocirecipebuildjob.py b/lib/lp/oci/tests/test_ocirecipebuildjob.py |
569 | new file mode 100644 |
570 | index 0000000..2b2d45c |
571 | --- /dev/null |
572 | +++ b/lib/lp/oci/tests/test_ocirecipebuildjob.py |
573 | @@ -0,0 +1,49 @@ |
574 | +# Copyright 2020 Canonical Ltd. This software is licensed under the |
575 | +# GNU Affero General Public License version 3 (see the file LICENSE). |
576 | + |
577 | +"""OCIRecipeBuildJob tests""" |
578 | + |
579 | +from __future__ import absolute_import, print_function, unicode_literals |
580 | + |
581 | +__metaclass__ = type |
582 | + |
583 | + |
584 | +from lp.oci.interfaces.ocirecipebuildjob import IOCIRecipeBuildJob |
585 | +from lp.oci.model.ocirecipebuildjob import ( |
586 | + OCIRecipeBuildJob, |
587 | + OCIRecipeBuildJobDerived, |
588 | + OCIRecipeBuildJobType, |
589 | + ) |
590 | +from lp.testing import TestCaseWithFactory |
591 | +from lp.testing.layers import DatabaseFunctionalLayer |
592 | + |
593 | + |
594 | +class FakeOCIBuildJob(OCIRecipeBuildJobDerived): |
595 | + """For testing OCIRecipeBuildJobDerived without a child class.""" |
596 | + |
597 | + |
598 | +class TestOCIRecipeBuildJob(TestCaseWithFactory): |
599 | + |
600 | + layer = DatabaseFunctionalLayer |
601 | + |
602 | + def test_provides_interface(self): |
603 | + oci_build = self.factory.makeOCIRecipeBuild() |
604 | + self.assertProvides( |
605 | + OCIRecipeBuildJob( |
606 | + oci_build, OCIRecipeBuildJobType.REGISTRY_UPLOAD, {}), |
607 | + IOCIRecipeBuildJob) |
608 | + |
609 | + def test_getOopsVars(self): |
610 | + oci_build = self.factory.makeOCIRecipeBuild() |
611 | + build_job = OCIRecipeBuildJob( |
612 | + oci_build, OCIRecipeBuildJobType.REGISTRY_UPLOAD, {}) |
613 | + derived = FakeOCIBuildJob(build_job) |
614 | + oops = derived.getOopsVars() |
615 | + expected = [ |
616 | + ('job_id', build_job.job.id), |
617 | + ('job_type', build_job.job_type.title), |
618 | + ('build_id', oci_build.id), |
619 | + ('recipe_owner_id', oci_build.recipe.owner.id), |
620 | + ('oci_project_name', oci_build.recipe.oci_project.name), |
621 | + ] |
622 | + self.assertEqual(expected, oops) |
Mostly good to me. Added just a few minor comments, but maybe we need another opinion too.