Merge ~twom/launchpad:oci-registry-upload into launchpad:master
- Git
- lp:~twom/launchpad
- oci-registry-upload
- Merge into master
Proposed by
Tom Wardill
Status: | Merged |
---|---|
Approved by: | Tom Wardill |
Approved revision: | 8215dfdeeafb07ed27722bcf51b93450df6bbd92 |
Merge reported by: | Otto Co-Pilot |
Merged at revision: | not available |
Proposed branch: | ~twom/launchpad:oci-registry-upload |
Merge into: | launchpad:master |
Diff against target: |
1332 lines (+986/-33) 17 files modified
database/schema/security.cfg (+41/-0) lib/lp/oci/configure.zcml (+19/-1) lib/lp/oci/interfaces/ocirecipe.py (+6/-0) lib/lp/oci/interfaces/ocirecipebuild.py (+66/-2) lib/lp/oci/interfaces/ocirecipebuildjob.py (+25/-1) lib/lp/oci/interfaces/ociregistryclient.py (+33/-0) lib/lp/oci/model/ocirecipe.py (+4/-0) lib/lp/oci/model/ocirecipebuild.py (+46/-14) lib/lp/oci/model/ocirecipebuildjob.py (+116/-6) lib/lp/oci/model/ociregistryclient.py (+252/-0) lib/lp/oci/model/ociregistrycredentials.py (+2/-2) lib/lp/oci/subscribers/ocirecipebuild.py (+11/-4) lib/lp/oci/tests/test_ocirecipe.py (+1/-0) lib/lp/oci/tests/test_ocirecipebuild.py (+1/-0) lib/lp/oci/tests/test_ocirecipebuildjob.py (+153/-3) lib/lp/oci/tests/test_ociregistryclient.py (+204/-0) lib/lp/services/config/schema-lazr.conf (+6/-0) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Colin Watson (community) | Approve | ||
Review via email: mp+381981@code.launchpad.net |
Commit message
Upload built images to an OCI registry
Description of the change
To post a comment you must log in.
Revision history for this message
Colin Watson (cjwatson) : | # |
Revision history for this message
Colin Watson (cjwatson) : | # |
review:
Needs Fixing
Revision history for this message
Tom Wardill (twom) : | # |
Revision history for this message
Tom Wardill (twom) : | # |
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/database/schema/security.cfg b/database/schema/security.cfg | |||
2 | index 942c5b4..55226e1 100644 | |||
3 | --- a/database/schema/security.cfg | |||
4 | +++ b/database/schema/security.cfg | |||
5 | @@ -999,6 +999,7 @@ public.livefsfile = SELECT | |||
6 | 999 | public.ocifile = SELECT | 999 | public.ocifile = SELECT |
7 | 1000 | public.ociproject = SELECT | 1000 | public.ociproject = SELECT |
8 | 1001 | public.ociprojectname = SELECT | 1001 | public.ociprojectname = SELECT |
9 | 1002 | public.ocipushrule = SELECT | ||
10 | 1002 | public.ocirecipe = SELECT | 1003 | public.ocirecipe = SELECT |
11 | 1003 | public.ocirecipebuild = SELECT, UPDATE | 1004 | public.ocirecipebuild = SELECT, UPDATE |
12 | 1004 | public.ocirecipebuildjob = SELECT, INSERT | 1005 | public.ocirecipebuildjob = SELECT, INSERT |
13 | @@ -1439,6 +1440,7 @@ public.ocifile = SELECT, INSERT | |||
14 | 1439 | public.ociproject = SELECT | 1440 | public.ociproject = SELECT |
15 | 1440 | public.ociprojectname = SELECT | 1441 | public.ociprojectname = SELECT |
16 | 1441 | public.ociprojectseries = SELECT | 1442 | public.ociprojectseries = SELECT |
17 | 1443 | public.ocipushrule = SELECT | ||
18 | 1442 | public.ocirecipe = SELECT, UPDATE | 1444 | public.ocirecipe = SELECT, UPDATE |
19 | 1443 | public.ocirecipebuild = SELECT, UPDATE | 1445 | public.ocirecipebuild = SELECT, UPDATE |
20 | 1444 | public.ocirecipebuildjob = SELECT, INSERT, UPDATE | 1446 | public.ocirecipebuildjob = SELECT, INSERT, UPDATE |
21 | @@ -2678,3 +2680,42 @@ public.teammembership = SELECT | |||
22 | 2678 | public.teamparticipation = SELECT | 2680 | public.teamparticipation = SELECT |
23 | 2679 | public.webhook = SELECT | 2681 | public.webhook = SELECT |
24 | 2680 | public.webhookjob = SELECT, INSERT | 2682 | public.webhookjob = SELECT, INSERT |
25 | 2683 | |||
26 | 2684 | [oci-build-job] | ||
27 | 2685 | type=user | ||
28 | 2686 | groups=script | ||
29 | 2687 | public.account = SELECT | ||
30 | 2688 | public.archive = SELECT | ||
31 | 2689 | public.branch = SELECT | ||
32 | 2690 | public.builder = SELECT | ||
33 | 2691 | public.buildfarmjob = SELECT, INSERT | ||
34 | 2692 | public.buildqueue = SELECT, INSERT, UPDATE | ||
35 | 2693 | public.distribution = SELECT | ||
36 | 2694 | public.distroarchseries = SELECT | ||
37 | 2695 | public.distroseries = SELECT | ||
38 | 2696 | public.emailaddress = SELECT | ||
39 | 2697 | public.gitref = SELECT | ||
40 | 2698 | public.gitrepository = SELECT | ||
41 | 2699 | public.job = SELECT, INSERT, UPDATE | ||
42 | 2700 | public.libraryfilealias = SELECT | ||
43 | 2701 | public.libraryfilecontent = SELECT | ||
44 | 2702 | public.person = SELECT | ||
45 | 2703 | public.personsettings = SELECT | ||
46 | 2704 | public.pocketchroot = SELECT | ||
47 | 2705 | public.processor = SELECT | ||
48 | 2706 | public.product = SELECT | ||
49 | 2707 | public.ocirecipe = SELECT, UPDATE | ||
50 | 2708 | public.ocirecipearch = SELECT | ||
51 | 2709 | public.ocirecipebuild = SELECT, INSERT, UPDATE | ||
52 | 2710 | public.ocirecipebuildjob = SELECT, UPDATE | ||
53 | 2711 | public.ocifile = SELECT | ||
54 | 2712 | public.ociproject = SELECT | ||
55 | 2713 | public.ociprojectname = SELECT | ||
56 | 2714 | public.ociprojectseries = SELECT | ||
57 | 2715 | public.ocipushrule = SELECT | ||
58 | 2716 | public.ociregistrycredentials = SELECT | ||
59 | 2717 | public.sourcepackagename = SELECT | ||
60 | 2718 | public.teammembership = SELECT | ||
61 | 2719 | public.teamparticipation = SELECT | ||
62 | 2720 | public.webhook = SELECT | ||
63 | 2721 | public.webhookjob = SELECT, INSERT | ||
64 | diff --git a/lib/lp/oci/configure.zcml b/lib/lp/oci/configure.zcml | |||
65 | index 565412d..5836861 100644 | |||
66 | --- a/lib/lp/oci/configure.zcml | |||
67 | +++ b/lib/lp/oci/configure.zcml | |||
68 | @@ -60,7 +60,7 @@ | |||
69 | 60 | <subscriber | 60 | <subscriber |
70 | 61 | for="lp.oci.interfaces.ocirecipebuild.IOCIRecipeBuild | 61 | for="lp.oci.interfaces.ocirecipebuild.IOCIRecipeBuild |
71 | 62 | lazr.lifecycle.interfaces.IObjectModifiedEvent" | 62 | lazr.lifecycle.interfaces.IObjectModifiedEvent" |
73 | 63 | handler="lp.oci.subscribers.ocirecipebuild.oci_recipe_build_status_changed" /> | 63 | handler="lp.oci.subscribers.ocirecipebuild.oci_recipe_build_modified" /> |
74 | 64 | 64 | ||
75 | 65 | <!-- OCIRecipeBuildSet --> | 65 | <!-- OCIRecipeBuildSet --> |
76 | 66 | <securedutility | 66 | <securedutility |
77 | @@ -127,4 +127,22 @@ | |||
78 | 127 | interface="lp.oci.interfaces.ocipushrule.IOCIPushRuleSet"/> | 127 | interface="lp.oci.interfaces.ocipushrule.IOCIPushRuleSet"/> |
79 | 128 | </securedutility> | 128 | </securedutility> |
80 | 129 | 129 | ||
81 | 130 | <!-- OCI related jobs --> | ||
82 | 131 | <securedutility | ||
83 | 132 | component="lp.oci.model.ocirecipebuildjob.OCIRegistryUploadJob" | ||
84 | 133 | provides="lp.oci.interfaces.ocirecipebuildjob.IOCIRegistryUploadJobSource"> | ||
85 | 134 | <allow interface="lp.oci.interfaces.ocirecipebuildjob.IOCIRegistryUploadJobSource" /> | ||
86 | 135 | </securedutility> | ||
87 | 136 | <class class="lp.oci.model.ocirecipebuildjob.OCIRegistryUploadJob"> | ||
88 | 137 | <allow interface="lp.oci.interfaces.ocirecipebuildjob.IOCIRecipeBuildJob" /> | ||
89 | 138 | <allow interface="lp.oci.interfaces.ocirecipebuildjob.IOCIRegistryUploadJob" /> | ||
90 | 139 | </class> | ||
91 | 140 | |||
92 | 141 | <!-- Registry interaction --> | ||
93 | 142 | <securedutility | ||
94 | 143 | class="lp.oci.model.ociregistryclient.OCIRegistryClient" | ||
95 | 144 | provides="lp.oci.interfaces.ociregistryclient.IOCIRegistryClient"> | ||
96 | 145 | <allow interface="lp.oci.interfaces.ociregistryclient.IOCIRegistryClient" /> | ||
97 | 146 | </securedutility> | ||
98 | 147 | |||
99 | 130 | </configure> | 148 | </configure> |
100 | diff --git a/lib/lp/oci/interfaces/ocirecipe.py b/lib/lp/oci/interfaces/ocirecipe.py | |||
101 | index 25a8967..eda2782 100644 | |||
102 | --- a/lib/lp/oci/interfaces/ocirecipe.py | |||
103 | +++ b/lib/lp/oci/interfaces/ocirecipe.py | |||
104 | @@ -214,6 +214,12 @@ class IOCIRecipeView(Interface): | |||
105 | 214 | # Really IOCIPushRule, patched in _schema_cirular_imports. | 214 | # Really IOCIPushRule, patched in _schema_cirular_imports. |
106 | 215 | value_type=Reference(schema=Interface), readonly=True) | 215 | value_type=Reference(schema=Interface), readonly=True) |
107 | 216 | 216 | ||
108 | 217 | can_upload_to_registry = Bool( | ||
109 | 218 | title=_("Can upload to registry"), required=True, readonly=True, | ||
110 | 219 | description=_( | ||
111 | 220 | "Whether everything is set up to allow uploading builds of " | ||
112 | 221 | "this OCI recipe to a registry.")) | ||
113 | 222 | |||
114 | 217 | 223 | ||
115 | 218 | class IOCIRecipeEdit(IWebhookTarget): | 224 | class IOCIRecipeEdit(IWebhookTarget): |
116 | 219 | """`IOCIRecipe` methods that require launchpad.Edit permission.""" | 225 | """`IOCIRecipe` methods that require launchpad.Edit permission.""" |
117 | diff --git a/lib/lp/oci/interfaces/ocirecipebuild.py b/lib/lp/oci/interfaces/ocirecipebuild.py | |||
118 | index 90bb829..a1fbfc0 100644 | |||
119 | --- a/lib/lp/oci/interfaces/ocirecipebuild.py | |||
120 | +++ b/lib/lp/oci/interfaces/ocirecipebuild.py | |||
121 | @@ -10,12 +10,24 @@ __all__ = [ | |||
122 | 10 | 'IOCIFile', | 10 | 'IOCIFile', |
123 | 11 | 'IOCIRecipeBuild', | 11 | 'IOCIRecipeBuild', |
124 | 12 | 'IOCIRecipeBuildSet', | 12 | 'IOCIRecipeBuildSet', |
125 | 13 | 'OCIRecipeBuildRegistryUploadStatus', | ||
126 | 13 | ] | 14 | ] |
127 | 14 | 15 | ||
130 | 15 | from lazr.restful.fields import Reference | 16 | from lazr.enum import ( |
131 | 16 | from zope.interface import Interface | 17 | EnumeratedType, |
132 | 18 | Item, | ||
133 | 19 | ) | ||
134 | 20 | from lazr.restful.fields import ( | ||
135 | 21 | CollectionField, | ||
136 | 22 | Reference, | ||
137 | 23 | ) | ||
138 | 24 | from zope.interface import ( | ||
139 | 25 | Attribute, | ||
140 | 26 | Interface, | ||
141 | 27 | ) | ||
142 | 17 | from zope.schema import ( | 28 | from zope.schema import ( |
143 | 18 | Bool, | 29 | Bool, |
144 | 30 | Choice, | ||
145 | 19 | Datetime, | 31 | Datetime, |
146 | 20 | Int, | 32 | Int, |
147 | 21 | TextLine, | 33 | TextLine, |
148 | @@ -31,6 +43,38 @@ from lp.services.librarian.interfaces import ILibraryFileAlias | |||
149 | 31 | from lp.soyuz.interfaces.distroarchseries import IDistroArchSeries | 43 | from lp.soyuz.interfaces.distroarchseries import IDistroArchSeries |
150 | 32 | 44 | ||
151 | 33 | 45 | ||
152 | 46 | class OCIRecipeBuildRegistryUploadStatus(EnumeratedType): | ||
153 | 47 | """OCI build registry upload status type | ||
154 | 48 | |||
155 | 49 | OCI builds may be uploaded to a registry. This represents the state of | ||
156 | 50 | that process. | ||
157 | 51 | """ | ||
158 | 52 | |||
159 | 53 | UNSCHEDULED = Item(""" | ||
160 | 54 | Unscheduled | ||
161 | 55 | |||
162 | 56 | No upload of this OCI build to a registry is scheduled. | ||
163 | 57 | """) | ||
164 | 58 | |||
165 | 59 | PENDING = Item(""" | ||
166 | 60 | Pending | ||
167 | 61 | |||
168 | 62 | This OCI build is queued for upload to a registry. | ||
169 | 63 | """) | ||
170 | 64 | |||
171 | 65 | FAILEDTOUPLOAD = Item(""" | ||
172 | 66 | Failed to upload | ||
173 | 67 | |||
174 | 68 | The last attempt to upload this OCI build to a registry failed. | ||
175 | 69 | """) | ||
176 | 70 | |||
177 | 71 | UPLOADED = Item(""" | ||
178 | 72 | Uploaded | ||
179 | 73 | |||
180 | 74 | This OCI build was successfully uploaded to a registry. | ||
181 | 75 | """) | ||
182 | 76 | |||
183 | 77 | |||
184 | 34 | class IOCIRecipeBuildView(IPackageBuild): | 78 | class IOCIRecipeBuildView(IPackageBuild): |
185 | 35 | """`IOCIRecipeBuild` attributes that require launchpad.View permission.""" | 79 | """`IOCIRecipeBuild` attributes that require launchpad.View permission.""" |
186 | 36 | 80 | ||
187 | @@ -102,6 +146,26 @@ class IOCIRecipeBuildView(IPackageBuild): | |||
188 | 102 | required=True, readonly=True, | 146 | required=True, readonly=True, |
189 | 103 | description=_("Whether this build record can be cancelled.")) | 147 | description=_("Whether this build record can be cancelled.")) |
190 | 104 | 148 | ||
191 | 149 | manifest = Attribute(_("The manifest of the image.")) | ||
192 | 150 | |||
193 | 151 | digests = Attribute(_("File containing the image digests.")) | ||
194 | 152 | |||
195 | 153 | registry_upload_jobs = CollectionField( | ||
196 | 154 | title=_("Registry upload jobs for this build."), | ||
197 | 155 | # Really IOCIRegistryUploadJob. | ||
198 | 156 | value_type=Reference(schema=Interface), | ||
199 | 157 | readonly=True) | ||
200 | 158 | |||
201 | 159 | # Really IOCIRegistryUploadJob | ||
202 | 160 | last_registry_upload_job = Reference( | ||
203 | 161 | title=_("Last registry upload job for this build."), schema=Interface) | ||
204 | 162 | |||
205 | 163 | registry_upload_status = Choice( | ||
206 | 164 | title=_("Registry upload status"), | ||
207 | 165 | vocabulary=OCIRecipeBuildRegistryUploadStatus, | ||
208 | 166 | required=True, readonly=False | ||
209 | 167 | ) | ||
210 | 168 | |||
211 | 105 | 169 | ||
212 | 106 | class IOCIRecipeBuildEdit(Interface): | 170 | class IOCIRecipeBuildEdit(Interface): |
213 | 107 | """`IOCIRecipeBuild` attributes that require launchpad.Edit permission.""" | 171 | """`IOCIRecipeBuild` attributes that require launchpad.Edit permission.""" |
214 | diff --git a/lib/lp/oci/interfaces/ocirecipebuildjob.py b/lib/lp/oci/interfaces/ocirecipebuildjob.py | |||
215 | index 6c25b73..7f2537b 100644 | |||
216 | --- a/lib/lp/oci/interfaces/ocirecipebuildjob.py | |||
217 | +++ b/lib/lp/oci/interfaces/ocirecipebuildjob.py | |||
218 | @@ -8,17 +8,25 @@ from __future__ import absolute_import, print_function, unicode_literals | |||
219 | 8 | __metaclass__ = type | 8 | __metaclass__ = type |
220 | 9 | __all__ = [ | 9 | __all__ = [ |
221 | 10 | 'IOCIRecipeBuildJob', | 10 | 'IOCIRecipeBuildJob', |
222 | 11 | 'IOCIRegistryUploadJob', | ||
223 | 12 | 'IOCIRegistryUploadJobSource', | ||
224 | 11 | ] | 13 | ] |
225 | 12 | 14 | ||
226 | 13 | from lazr.restful.fields import Reference | 15 | from lazr.restful.fields import Reference |
227 | 16 | from zope.component.interfaces import IObjectEvent | ||
228 | 14 | from zope.interface import ( | 17 | from zope.interface import ( |
229 | 15 | Attribute, | 18 | Attribute, |
230 | 16 | Interface, | 19 | Interface, |
231 | 17 | ) | 20 | ) |
232 | 21 | from zope.schema import TextLine | ||
233 | 18 | 22 | ||
234 | 19 | from lp import _ | 23 | from lp import _ |
235 | 20 | from lp.oci.interfaces.ocirecipebuild import IOCIRecipeBuild | 24 | from lp.oci.interfaces.ocirecipebuild import IOCIRecipeBuild |
237 | 21 | from lp.services.job.interfaces.job import IJob | 25 | from lp.services.job.interfaces.job import ( |
238 | 26 | IJob, | ||
239 | 27 | IJobSource, | ||
240 | 28 | IRunnableJob, | ||
241 | 29 | ) | ||
242 | 22 | 30 | ||
243 | 23 | 31 | ||
244 | 24 | class IOCIRecipeBuildJob(Interface): | 32 | class IOCIRecipeBuildJob(Interface): |
245 | @@ -32,3 +40,19 @@ class IOCIRecipeBuildJob(Interface): | |||
246 | 32 | schema=IOCIRecipeBuild, required=True, readonly=True) | 40 | schema=IOCIRecipeBuild, required=True, readonly=True) |
247 | 33 | 41 | ||
248 | 34 | json_data = Attribute(_("A dict of data about the job.")) | 42 | json_data = Attribute(_("A dict of data about the job.")) |
249 | 43 | |||
250 | 44 | |||
251 | 45 | class IOCIRegistryUploadJob(IRunnableJob): | ||
252 | 46 | """A Job that uploads an OCI image to a registry.""" | ||
253 | 47 | |||
254 | 48 | error_message = TextLine( | ||
255 | 49 | title=_("Error message"), required=False, readonly=True) | ||
256 | 50 | |||
257 | 51 | |||
258 | 52 | class IOCIRegistryUploadJobSource(IJobSource): | ||
259 | 53 | |||
260 | 54 | def create(build): | ||
261 | 55 | """Upload an OCI image to a registry. | ||
262 | 56 | |||
263 | 57 | :param build: The OCI recipe build to upload. | ||
264 | 58 | """ | ||
265 | diff --git a/lib/lp/oci/interfaces/ociregistryclient.py b/lib/lp/oci/interfaces/ociregistryclient.py | |||
266 | 35 | new file mode 100644 | 59 | new file mode 100644 |
267 | index 0000000..8ac2aa6 | |||
268 | --- /dev/null | |||
269 | +++ b/lib/lp/oci/interfaces/ociregistryclient.py | |||
270 | @@ -0,0 +1,33 @@ | |||
271 | 1 | # Copyright 2020 Canonical Ltd. This software is licensed under the | ||
272 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | ||
273 | 3 | |||
274 | 4 | """Interface for communication with an OCI registry.""" | ||
275 | 5 | |||
276 | 6 | from __future__ import absolute_import, print_function, unicode_literals | ||
277 | 7 | |||
278 | 8 | __metaclass__ = type | ||
279 | 9 | __all__ = [ | ||
280 | 10 | 'BlobUploadFailed', | ||
281 | 11 | 'IOCIRegistryClient', | ||
282 | 12 | 'ManifestUploadFailed', | ||
283 | 13 | ] | ||
284 | 14 | |||
285 | 15 | from zope.interface import Interface | ||
286 | 16 | |||
287 | 17 | |||
288 | 18 | class BlobUploadFailed(Exception): | ||
289 | 19 | pass | ||
290 | 20 | |||
291 | 21 | |||
292 | 22 | class ManifestUploadFailed(Exception): | ||
293 | 23 | pass | ||
294 | 24 | |||
295 | 25 | |||
296 | 26 | class IOCIRegistryClient(Interface): | ||
297 | 27 | """Interface for the API provided by an OCI registry.""" | ||
298 | 28 | |||
299 | 29 | def upload(build): | ||
300 | 30 | """Upload an OCI image to a registry. | ||
301 | 31 | |||
302 | 32 | :param build: The `IOCIRecipeBuild` to upload. | ||
303 | 33 | """ | ||
304 | diff --git a/lib/lp/oci/model/ocirecipe.py b/lib/lp/oci/model/ocirecipe.py | |||
305 | index 4e3273a..aa69900 100644 | |||
306 | --- a/lib/lp/oci/model/ocirecipe.py | |||
307 | +++ b/lib/lp/oci/model/ocirecipe.py | |||
308 | @@ -377,6 +377,10 @@ class OCIRecipe(Storm, WebhookTargetMixin): | |||
309 | 377 | order_by = Desc(OCIRecipeBuild.id) | 377 | order_by = Desc(OCIRecipeBuild.id) |
310 | 378 | return self._getBuilds(filter_term, order_by) | 378 | return self._getBuilds(filter_term, order_by) |
311 | 379 | 379 | ||
312 | 380 | @property | ||
313 | 381 | def can_upload_to_registry(self): | ||
314 | 382 | return not self.push_rules.is_empty() | ||
315 | 383 | |||
316 | 380 | 384 | ||
317 | 381 | class OCIRecipeArch(Storm): | 385 | class OCIRecipeArch(Storm): |
318 | 382 | """Link table to back `OCIRecipe.processors`.""" | 386 | """Link table to back `OCIRecipe.processors`.""" |
319 | diff --git a/lib/lp/oci/model/ocirecipebuild.py b/lib/lp/oci/model/ocirecipebuild.py | |||
320 | index 595da53..c69e348 100644 | |||
321 | --- a/lib/lp/oci/model/ocirecipebuild.py | |||
322 | +++ b/lib/lp/oci/model/ocirecipebuild.py | |||
323 | @@ -45,6 +45,11 @@ from lp.oci.interfaces.ocirecipebuild import ( | |||
324 | 45 | IOCIFile, | 45 | IOCIFile, |
325 | 46 | IOCIRecipeBuild, | 46 | IOCIRecipeBuild, |
326 | 47 | IOCIRecipeBuildSet, | 47 | IOCIRecipeBuildSet, |
327 | 48 | OCIRecipeBuildRegistryUploadStatus, | ||
328 | 49 | ) | ||
329 | 50 | from lp.oci.model.ocirecipebuildjob import ( | ||
330 | 51 | OCIRecipeBuildJob, | ||
331 | 52 | OCIRecipeBuildJobType, | ||
332 | 48 | ) | 53 | ) |
333 | 49 | from lp.registry.interfaces.pocket import PackagePublishingPocket | 54 | from lp.registry.interfaces.pocket import PackagePublishingPocket |
334 | 50 | from lp.registry.model.person import Person | 55 | from lp.registry.model.person import Person |
335 | @@ -57,6 +62,8 @@ from lp.services.database.interfaces import ( | |||
336 | 57 | IMasterStore, | 62 | IMasterStore, |
337 | 58 | IStore, | 63 | IStore, |
338 | 59 | ) | 64 | ) |
339 | 65 | from lp.services.job.interfaces.job import JobStatus | ||
340 | 66 | from lp.services.job.model.job import Job | ||
341 | 60 | from lp.services.librarian.model import ( | 67 | from lp.services.librarian.model import ( |
342 | 61 | LibraryFileAlias, | 68 | LibraryFileAlias, |
343 | 62 | LibraryFileContent, | 69 | LibraryFileContent, |
344 | @@ -354,23 +361,17 @@ class OCIRecipeBuild(PackageBuildMixin, Storm): | |||
345 | 354 | 361 | ||
346 | 355 | @cachedproperty | 362 | @cachedproperty |
347 | 356 | def manifest(self): | 363 | def manifest(self): |
355 | 357 | result = Store.of(self).find( | 364 | try: |
356 | 358 | (OCIFile, LibraryFileAlias, LibraryFileContent), | 365 | return self.getFileByName("manifest.json") |
357 | 359 | OCIFile.build == self.id, | 366 | except NotFoundError: |
358 | 360 | LibraryFileAlias.id == OCIFile.library_file_id, | 367 | return None |
352 | 361 | LibraryFileContent.id == LibraryFileAlias.contentID, | ||
353 | 362 | LibraryFileAlias.filename == 'manifest.json') | ||
354 | 363 | return result.one() | ||
359 | 364 | 368 | ||
360 | 365 | @cachedproperty | 369 | @cachedproperty |
361 | 366 | def digests(self): | 370 | def digests(self): |
369 | 367 | result = Store.of(self).find( | 371 | try: |
370 | 368 | (OCIFile, LibraryFileAlias, LibraryFileContent), | 372 | return self.getFileByName("digests.json") |
371 | 369 | OCIFile.build == self.id, | 373 | except NotFoundError: |
372 | 370 | LibraryFileAlias.id == OCIFile.library_file_id, | 374 | return None |
366 | 371 | LibraryFileContent.id == LibraryFileAlias.contentID, | ||
367 | 372 | LibraryFileAlias.filename == 'digests.json') | ||
368 | 373 | return result.one() | ||
373 | 374 | 375 | ||
374 | 375 | def verifySuccessfulUpload(self): | 376 | def verifySuccessfulUpload(self): |
375 | 376 | """See `IPackageBuild`.""" | 377 | """See `IPackageBuild`.""" |
376 | @@ -383,6 +384,37 @@ class OCIRecipeBuild(PackageBuildMixin, Storm): | |||
377 | 383 | and self.digests is not None) | 384 | and self.digests is not None) |
378 | 384 | return layer_files_present and metadata_present | 385 | return layer_files_present and metadata_present |
379 | 385 | 386 | ||
380 | 387 | @property | ||
381 | 388 | def registry_upload_jobs(self): | ||
382 | 389 | jobs = Store.of(self).find( | ||
383 | 390 | OCIRecipeBuildJob, | ||
384 | 391 | OCIRecipeBuildJob.build == self, | ||
385 | 392 | OCIRecipeBuildJob.job_type == OCIRecipeBuildJobType.REGISTRY_UPLOAD | ||
386 | 393 | ) | ||
387 | 394 | jobs.order_by(Desc(OCIRecipeBuildJob.job_id)) | ||
388 | 395 | |||
389 | 396 | def preload_jobs(rows): | ||
390 | 397 | load_related(Job, rows, ["job_id"]) | ||
391 | 398 | |||
392 | 399 | return DecoratedResultSet( | ||
393 | 400 | jobs, lambda job: job.makeDerived(), pre_iter_hook=preload_jobs) | ||
394 | 401 | |||
395 | 402 | @cachedproperty | ||
396 | 403 | def last_registry_upload_job(self): | ||
397 | 404 | return self.registry_upload_jobs.first() | ||
398 | 405 | |||
399 | 406 | @property | ||
400 | 407 | def registry_upload_status(self): | ||
401 | 408 | job = self.last_registry_upload_job | ||
402 | 409 | if job is None or job.job.status == JobStatus.SUSPENDED: | ||
403 | 410 | return OCIRecipeBuildRegistryUploadStatus.UNSCHEDULED | ||
404 | 411 | elif job.job.status in (JobStatus.WAITING, JobStatus.RUNNING): | ||
405 | 412 | return OCIRecipeBuildRegistryUploadStatus.PENDING | ||
406 | 413 | elif job.job.status == JobStatus.COMPLETED: | ||
407 | 414 | return OCIRecipeBuildRegistryUploadStatus.UPLOADED | ||
408 | 415 | else: | ||
409 | 416 | return OCIRecipeBuildRegistryUploadStatus.FAILEDTOUPLOAD | ||
410 | 417 | |||
411 | 386 | 418 | ||
412 | 387 | @implementer(IOCIRecipeBuildSet) | 419 | @implementer(IOCIRecipeBuildSet) |
413 | 388 | class OCIRecipeBuildSet(SpecificBuildFarmJobSourceMixin): | 420 | class OCIRecipeBuildSet(SpecificBuildFarmJobSourceMixin): |
414 | diff --git a/lib/lp/oci/model/ocirecipebuildjob.py b/lib/lp/oci/model/ocirecipebuildjob.py | |||
415 | index 7d8699c..c17ecf3 100644 | |||
416 | --- a/lib/lp/oci/model/ocirecipebuildjob.py | |||
417 | +++ b/lib/lp/oci/model/ocirecipebuildjob.py | |||
418 | @@ -11,20 +11,33 @@ __all__ = [ | |||
419 | 11 | 'OCIRecipeBuildJobType', | 11 | 'OCIRecipeBuildJobType', |
420 | 12 | ] | 12 | ] |
421 | 13 | 13 | ||
422 | 14 | |||
423 | 14 | from lazr.delegates import delegate_to | 15 | from lazr.delegates import delegate_to |
424 | 15 | from lazr.enum import ( | 16 | from lazr.enum import ( |
425 | 16 | DBEnumeratedType, | 17 | DBEnumeratedType, |
426 | 17 | DBItem, | 18 | DBItem, |
427 | 18 | ) | 19 | ) |
428 | 20 | from lazr.lifecycle.event import ObjectCreatedEvent | ||
429 | 19 | from storm.databases.postgres import JSON | 21 | from storm.databases.postgres import JSON |
430 | 20 | from storm.locals import ( | 22 | from storm.locals import ( |
431 | 21 | Int, | 23 | Int, |
432 | 22 | Reference, | 24 | Reference, |
433 | 23 | ) | 25 | ) |
435 | 24 | from zope.interface import implementer | 26 | import transaction |
436 | 27 | from zope.component import getUtility | ||
437 | 28 | from zope.event import notify | ||
438 | 29 | from zope.interface import ( | ||
439 | 30 | implementer, | ||
440 | 31 | provider, | ||
441 | 32 | ) | ||
442 | 25 | 33 | ||
443 | 26 | from lp.app.errors import NotFoundError | 34 | from lp.app.errors import NotFoundError |
445 | 27 | from lp.oci.interfaces.ocirecipebuildjob import IOCIRecipeBuildJob | 35 | from lp.oci.interfaces.ocirecipebuildjob import ( |
446 | 36 | IOCIRecipeBuildJob, | ||
447 | 37 | IOCIRegistryUploadJob, | ||
448 | 38 | IOCIRegistryUploadJobSource, | ||
449 | 39 | ) | ||
450 | 40 | from lp.oci.interfaces.ociregistryclient import IOCIRegistryClient | ||
451 | 28 | from lp.services.database.enumcol import DBEnum | 41 | from lp.services.database.enumcol import DBEnum |
452 | 29 | from lp.services.database.interfaces import IStore | 42 | from lp.services.database.interfaces import IStore |
453 | 30 | from lp.services.database.stormbase import StormBase | 43 | from lp.services.database.stormbase import StormBase |
454 | @@ -33,14 +46,13 @@ from lp.services.job.model.job import ( | |||
455 | 33 | Job, | 46 | Job, |
456 | 34 | ) | 47 | ) |
457 | 35 | from lp.services.job.runner import BaseRunnableJob | 48 | from lp.services.job.runner import BaseRunnableJob |
458 | 49 | from lp.services.propertycache import get_property_cache | ||
459 | 50 | from lp.services.webapp.snapshot import notify_modified | ||
460 | 36 | 51 | ||
461 | 37 | 52 | ||
462 | 38 | class OCIRecipeBuildJobType(DBEnumeratedType): | 53 | class OCIRecipeBuildJobType(DBEnumeratedType): |
463 | 39 | """Values that `OCIBuildJobType.job_type` can take.""" | 54 | """Values that `OCIBuildJobType.job_type` can take.""" |
464 | 40 | 55 | ||
465 | 41 | # XXX twom (2020-04-02) This does not currently have a concrete | ||
466 | 42 | # implementation, awaiting registry upload. | ||
467 | 43 | |||
468 | 44 | REGISTRY_UPLOAD = DBItem(0, """ | 56 | REGISTRY_UPLOAD = DBItem(0, """ |
469 | 45 | Registry upload | 57 | Registry upload |
470 | 46 | 58 | ||
471 | @@ -82,7 +94,7 @@ class OCIRecipeBuildJob(StormBase): | |||
472 | 82 | self.json_data = json_data | 94 | self.json_data = json_data |
473 | 83 | 95 | ||
474 | 84 | def makeDerived(self): | 96 | def makeDerived(self): |
476 | 85 | return OCIRecipeBuildJob.makeSubclass(self) | 97 | return OCIRecipeBuildJobDerived.makeSubclass(self) |
477 | 86 | 98 | ||
478 | 87 | 99 | ||
479 | 88 | @delegate_to(IOCIRecipeBuildJob) | 100 | @delegate_to(IOCIRecipeBuildJob) |
480 | @@ -138,3 +150,101 @@ class OCIRecipeBuildJobDerived(BaseRunnableJob): | |||
481 | 138 | ('oci_project_name', self.context.build.recipe.oci_project.name) | 150 | ('oci_project_name', self.context.build.recipe.oci_project.name) |
482 | 139 | ]) | 151 | ]) |
483 | 140 | return oops_vars | 152 | return oops_vars |
484 | 153 | |||
485 | 154 | |||
486 | 155 | @implementer(IOCIRegistryUploadJob) | ||
487 | 156 | @provider(IOCIRegistryUploadJobSource) | ||
488 | 157 | class OCIRegistryUploadJob(OCIRecipeBuildJobDerived): | ||
489 | 158 | |||
490 | 159 | class_job_type = OCIRecipeBuildJobType.REGISTRY_UPLOAD | ||
491 | 160 | |||
492 | 161 | @classmethod | ||
493 | 162 | def create(cls, build): | ||
494 | 163 | """See `IOCIRegistryUploadJobSource`""" | ||
495 | 164 | oci_build_job = OCIRecipeBuildJob( | ||
496 | 165 | build, cls.class_job_type, {}) | ||
497 | 166 | job = cls(oci_build_job) | ||
498 | 167 | job.celeryRunOnCommit() | ||
499 | 168 | del get_property_cache(build).last_registry_upload_job | ||
500 | 169 | notify(ObjectCreatedEvent(build)) | ||
501 | 170 | return job | ||
502 | 171 | |||
503 | 172 | # Ideally we'd just override Job._set_status or similar, but | ||
504 | 173 | # lazr.delegates makes that difficult, so we use this to override all | ||
505 | 174 | # the individual Job lifecycle methods instead. | ||
506 | 175 | def _do_lifecycle(self, method_name, manage_transaction=False, | ||
507 | 176 | *args, **kwargs): | ||
508 | 177 | edited_fields = set() | ||
509 | 178 | with notify_modified(self.build, edited_fields) as before_modification: | ||
510 | 179 | getattr(super(OCIRegistryUploadJob, self), method_name)( | ||
511 | 180 | *args, manage_transaction=manage_transaction, **kwargs) | ||
512 | 181 | upload_status = self.build.registry_upload_status | ||
513 | 182 | if upload_status != before_modification.registry_upload_status: | ||
514 | 183 | edited_fields.add('registry_upload_status') | ||
515 | 184 | if edited_fields and manage_transaction: | ||
516 | 185 | transaction.commit() | ||
517 | 186 | |||
518 | 187 | def start(self, *args, **kwargs): | ||
519 | 188 | self._do_lifecycle("start", *args, **kwargs) | ||
520 | 189 | |||
521 | 190 | def complete(self, *args, **kwargs): | ||
522 | 191 | self._do_lifecycle("complete", *args, **kwargs) | ||
523 | 192 | |||
524 | 193 | def fail(self, *args, **kwargs): | ||
525 | 194 | self._do_lifecycle("fail", *args, **kwargs) | ||
526 | 195 | |||
527 | 196 | def queue(self, *args, **kwargs): | ||
528 | 197 | self._do_lifecycle("queue", *args, **kwargs) | ||
529 | 198 | |||
530 | 199 | def suspend(self, *args, **kwargs): | ||
531 | 200 | self._do_lifecycle("suspend", *args, **kwargs) | ||
532 | 201 | |||
533 | 202 | def resume(self, *args, **kwargs): | ||
534 | 203 | self._do_lifecycle("resume", *args, **kwargs) | ||
535 | 204 | |||
536 | 205 | @property | ||
537 | 206 | def error_message(self): | ||
538 | 207 | """See `IOCIRegistryUploadJob`.""" | ||
539 | 208 | return self.json_data.get("error_message") | ||
540 | 209 | |||
541 | 210 | @error_message.setter | ||
542 | 211 | def error_message(self, message): | ||
543 | 212 | """See `IOCIRegistryUploadJob`.""" | ||
544 | 213 | self.json_data["error_message"] = message | ||
545 | 214 | |||
546 | 215 | @property | ||
547 | 216 | def error_detail(self): | ||
548 | 217 | """See `IOCIRegistryUploadJob`.""" | ||
549 | 218 | return self.json_data.get("error_detail") | ||
550 | 219 | |||
551 | 220 | @error_detail.setter | ||
552 | 221 | def error_detail(self, detail): | ||
553 | 222 | """See `IOCIRegistryUploadJob`.""" | ||
554 | 223 | self.json_data["error_detail"] = detail | ||
555 | 224 | |||
556 | 225 | @property | ||
557 | 226 | def error_messages(self): | ||
558 | 227 | """See `IOCIRegistryUploadJob`.""" | ||
559 | 228 | return self.json_data.get("error_messages") | ||
560 | 229 | |||
561 | 230 | @error_messages.setter | ||
562 | 231 | def error_messages(self, messages): | ||
563 | 232 | """See `IOCIRegistryUploadJob`.""" | ||
564 | 233 | self.json_data["error_messages"] = messages | ||
565 | 234 | |||
566 | 235 | def run(self): | ||
567 | 236 | """See `IRunnableJob`.""" | ||
568 | 237 | client = getUtility(IOCIRegistryClient) | ||
569 | 238 | # XXX twom 2020-04-16 This is taken from SnapStoreUploadJob | ||
570 | 239 | # it will need to gain retry support. | ||
571 | 240 | try: | ||
572 | 241 | try: | ||
573 | 242 | client.upload(self.build) | ||
574 | 243 | except Exception as e: | ||
575 | 244 | self.error_message = str(e) | ||
576 | 245 | self.error_messages = getattr(e, "messages", None) | ||
577 | 246 | self.error_detail = getattr(e, "detail", None) | ||
578 | 247 | raise | ||
579 | 248 | except Exception: | ||
580 | 249 | transaction.commit() | ||
581 | 250 | raise | ||
582 | diff --git a/lib/lp/oci/model/ociregistryclient.py b/lib/lp/oci/model/ociregistryclient.py | |||
583 | 141 | new file mode 100644 | 251 | new file mode 100644 |
584 | index 0000000..2152ab8 | |||
585 | --- /dev/null | |||
586 | +++ b/lib/lp/oci/model/ociregistryclient.py | |||
587 | @@ -0,0 +1,252 @@ | |||
588 | 1 | # Copyright 2020 Canonical Ltd. This software is licensed under the | ||
589 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | ||
590 | 3 | |||
591 | 4 | """Client for talking to an OCI registry.""" | ||
592 | 5 | |||
593 | 6 | from __future__ import absolute_import, print_function, unicode_literals | ||
594 | 7 | |||
595 | 8 | __metaclass__ = type | ||
596 | 9 | __all__ = [ | ||
597 | 10 | 'OCIRegistryClient' | ||
598 | 11 | ] | ||
599 | 12 | |||
600 | 13 | |||
601 | 14 | import hashlib | ||
602 | 15 | from io import BytesIO | ||
603 | 16 | import json | ||
604 | 17 | import logging | ||
605 | 18 | import tarfile | ||
606 | 19 | |||
607 | 20 | from requests.exceptions import HTTPError | ||
608 | 21 | from zope.interface import implementer | ||
609 | 22 | |||
610 | 23 | from lp.oci.interfaces.ociregistryclient import ( | ||
611 | 24 | BlobUploadFailed, | ||
612 | 25 | IOCIRegistryClient, | ||
613 | 26 | ManifestUploadFailed, | ||
614 | 27 | ) | ||
615 | 28 | from lp.services.timeout import urlfetch | ||
616 | 29 | |||
617 | 30 | |||
618 | 31 | log = logging.getLogger(__name__) | ||
619 | 32 | |||
620 | 33 | |||
621 | 34 | @implementer(IOCIRegistryClient) | ||
622 | 35 | class OCIRegistryClient: | ||
623 | 36 | |||
624 | 37 | @classmethod | ||
625 | 38 | def _getJSONfile(cls, reference): | ||
626 | 39 | """Read JSON out of a `LibraryFileAlias`.""" | ||
627 | 40 | try: | ||
628 | 41 | reference.open() | ||
629 | 42 | return json.loads(reference.read()) | ||
630 | 43 | finally: | ||
631 | 44 | reference.close() | ||
632 | 45 | |||
633 | 46 | @classmethod | ||
634 | 47 | def _upload(cls, digest, push_rule, name, fileobj): | ||
635 | 48 | """Upload a blob to the registry, using a given digest. | ||
636 | 49 | |||
637 | 50 | :param digest: The digest to store the file under. | ||
638 | 51 | :param push_rule: `OCIPushRule` to use for the URL and credentials. | ||
639 | 52 | :param name: Name of the image the blob is part of. | ||
640 | 53 | :param fileobj: An object that looks like a buffer. | ||
641 | 54 | |||
642 | 55 | :raises BlobUploadFailed: if the registry does not accept the blob. | ||
643 | 56 | """ | ||
644 | 57 | # Check if it already exists | ||
645 | 58 | try: | ||
646 | 59 | head_response = urlfetch( | ||
647 | 60 | "{}/v2/{}/blobs/{}".format( | ||
648 | 61 | push_rule.registry_credentials.url, name, digest), | ||
649 | 62 | method="HEAD") | ||
650 | 63 | if head_response.status_code == 200: | ||
651 | 64 | log.info("{} already found".format(digest)) | ||
652 | 65 | return | ||
653 | 66 | except HTTPError as http_error: | ||
654 | 67 | # A 404 is fine, we're about to upload the layer anyway | ||
655 | 68 | if http_error.response.status_code != 404: | ||
656 | 69 | raise http_error | ||
657 | 70 | |||
658 | 71 | post_response = urlfetch( | ||
659 | 72 | "{}/v2/{}/blobs/uploads/".format( | ||
660 | 73 | push_rule.registry_credentials.url, name), | ||
661 | 74 | method="POST") | ||
662 | 75 | |||
663 | 76 | post_location = post_response.headers["Location"] | ||
664 | 77 | query_parsed = {"digest": digest} | ||
665 | 78 | |||
666 | 79 | put_response = urlfetch( | ||
667 | 80 | post_location, | ||
668 | 81 | params=query_parsed, | ||
669 | 82 | data=fileobj, | ||
670 | 83 | method="PUT") | ||
671 | 84 | |||
672 | 85 | if put_response.status_code != 201: | ||
673 | 86 | raise BlobUploadFailed( | ||
674 | 87 | "Upload of {} for {} failed".format(digest, name)) | ||
675 | 88 | |||
676 | 89 | @classmethod | ||
677 | 90 | def _upload_layer(cls, digest, push_rule, name, lfa): | ||
678 | 91 | """Upload a layer blob to the registry. | ||
679 | 92 | |||
680 | 93 | Uses _upload, but opens the LFA and extracts the necessary files | ||
681 | 94 | from the .tar.gz first. | ||
682 | 95 | |||
683 | 96 | :param digest: The digest to store the file under. | ||
684 | 97 | :param push_rule: `OCIPushRule` to use for the URL and credentials. | ||
685 | 98 | :param name: Name of the image the blob is part of. | ||
686 | 99 | :param lfa: The `LibraryFileAlias` for the layer. | ||
687 | 100 | """ | ||
688 | 101 | lfa.open() | ||
689 | 102 | try: | ||
690 | 103 | un_zipped = tarfile.open(fileobj=lfa, mode='r|gz') | ||
691 | 104 | for tarinfo in un_zipped: | ||
692 | 105 | if tarinfo.name != 'layer.tar': | ||
693 | 106 | continue | ||
694 | 107 | fileobj = un_zipped.extractfile(tarinfo) | ||
695 | 108 | cls._upload(digest, push_rule, name, fileobj) | ||
696 | 109 | finally: | ||
697 | 110 | lfa.close() | ||
698 | 111 | |||
699 | 112 | @classmethod | ||
700 | 113 | def _build_registry_manifest(cls, digests, config, config_json, | ||
701 | 114 | config_sha, preloaded_data): | ||
702 | 115 | """Create an image manifest for the uploading image. | ||
703 | 116 | |||
704 | 117 | This involves nearly everything as digests and lengths are required. | ||
705 | 118 | This method creates a minimal manifest, some fields are missing. | ||
706 | 119 | |||
707 | 120 | :param digests: Dict of the various digests involved. | ||
708 | 121 | :param config: The contents of the manifest config file as a dict. | ||
709 | 122 | :param config_json: The config file as a JSON string. | ||
710 | 123 | :param config_sha: The sha256sum of the config JSON string. | ||
711 | 124 | """ | ||
712 | 125 | # Create the initial manifest data with empty layer information | ||
713 | 126 | manifest = { | ||
714 | 127 | "schemaVersion": 2, | ||
715 | 128 | "mediaType": | ||
716 | 129 | "application/vnd.docker.distribution.manifest.v2+json", | ||
717 | 130 | "config": { | ||
718 | 131 | "mediaType": "application/vnd.docker.container.image.v1+json", | ||
719 | 132 | "size": len(config_json), | ||
720 | 133 | "digest": "sha256:{}".format(config_sha), | ||
721 | 134 | }, | ||
722 | 135 | "layers": []} | ||
723 | 136 | |||
724 | 137 | # Fill in the layer information | ||
725 | 138 | for layer in config["rootfs"]["diff_ids"]: | ||
726 | 139 | manifest["layers"].append({ | ||
727 | 140 | "mediaType": | ||
728 | 141 | "application/vnd.docker.image.rootfs.diff.tar.gzip", | ||
729 | 142 | "size": preloaded_data[layer].content.filesize, | ||
730 | 143 | "digest": layer}) | ||
731 | 144 | return manifest | ||
732 | 145 | |||
733 | 146 | @classmethod | ||
734 | 147 | def _preloadFiles(cls, build, manifest, digests): | ||
735 | 148 | """Preload the data from the librarian to avoid multiple fetches | ||
736 | 149 | if there is more than one push rule for a build. | ||
737 | 150 | |||
738 | 151 | :param build: The referencing `OCIRecipeBuild`. | ||
739 | 152 | :param manifest: The manifest from the built image. | ||
740 | 153 | :param digests: Dict of the various digests involved. | ||
741 | 154 | """ | ||
742 | 155 | data = {} | ||
743 | 156 | for section in manifest: | ||
744 | 157 | # Load the matching config file for this section | ||
745 | 158 | config = cls._getJSONfile( | ||
746 | 159 | build.getFileByName(section['Config'])) | ||
747 | 160 | files = {"config_file": config} | ||
748 | 161 | for diff_id in config["rootfs"]["diff_ids"]: | ||
749 | 162 | # We may have already seen this diff ID. | ||
750 | 163 | if files.get(diff_id): | ||
751 | 164 | continue | ||
752 | 165 | # Retrieve the layer files. | ||
753 | 166 | # This doesn't read the content, so there is potential | ||
754 | 167 | # for multiple fetches, but the files can be arbitrary size | ||
755 | 168 | # Potentially gigabytes. | ||
756 | 169 | files[diff_id] = {} | ||
757 | 170 | source_digest = digests[diff_id]["digest"] | ||
758 | 171 | _, lfa, _ = build.getLayerFileByDigest(source_digest) | ||
759 | 172 | files[diff_id] = lfa | ||
760 | 173 | data[section["Config"]] = files | ||
761 | 174 | return data | ||
762 | 175 | |||
763 | 176 | @classmethod | ||
764 | 177 | def _calculateTag(cls, build, push_rule): | ||
765 | 178 | """Work out the base tag for the image should be. | ||
766 | 179 | |||
767 | 180 | :param build: `OCIRecipeBuild` representing this build. | ||
768 | 181 | :param push_rule: `OCIPushRule` that we are using. | ||
769 | 182 | """ | ||
770 | 183 | # XXX twom 2020-04-17 This needs to include OCIProjectSeries and | ||
771 | 184 | # base image name | ||
772 | 185 | |||
773 | 186 | return "{}".format("edge") | ||
774 | 187 | |||
775 | 188 | @classmethod | ||
776 | 189 | def upload(cls, build): | ||
777 | 190 | """Upload the artifacts from an OCIRecipeBuild to a registry. | ||
778 | 191 | |||
779 | 192 | :param build: `OCIRecipeBuild` representing this build. | ||
780 | 193 | :raises ManifestUploadFailed: If the final registry manifest fails to | ||
781 | 194 | upload due to network or validity. | ||
782 | 195 | """ | ||
783 | 196 | # Get the required metadata files | ||
784 | 197 | manifest = cls._getJSONfile(build.manifest) | ||
785 | 198 | digests_list = cls._getJSONfile(build.digests) | ||
786 | 199 | digests = {} | ||
787 | 200 | for digest_dict in digests_list: | ||
788 | 201 | digests.update(digest_dict) | ||
789 | 202 | |||
790 | 203 | # Preload the requested files | ||
791 | 204 | preloaded_data = cls._preloadFiles(build, manifest, digests) | ||
792 | 205 | |||
793 | 206 | for push_rule in build.recipe.push_rules: | ||
794 | 207 | for section in manifest: | ||
795 | 208 | # Work out names and tags | ||
796 | 209 | image_name = push_rule.image_name | ||
797 | 210 | tag = cls._calculateTag(build, push_rule) | ||
798 | 211 | file_data = preloaded_data[section["Config"]] | ||
799 | 212 | config = file_data["config_file"] | ||
800 | 213 | # Upload the layers involved | ||
801 | 214 | for diff_id in config["rootfs"]["diff_ids"]: | ||
802 | 215 | cls._upload_layer( | ||
803 | 216 | diff_id, | ||
804 | 217 | push_rule, | ||
805 | 218 | image_name, | ||
806 | 219 | file_data[diff_id]) | ||
807 | 220 | # The config file is required in different forms, so we can | ||
808 | 221 | # calculate the sha, work these out and upload | ||
809 | 222 | config_json = json.dumps(config).encode("UTF-8") | ||
810 | 223 | config_sha = hashlib.sha256(config_json).hexdigest() | ||
811 | 224 | cls._upload( | ||
812 | 225 | "sha256:{}".format(config_sha), | ||
813 | 226 | push_rule, | ||
814 | 227 | image_name, | ||
815 | 228 | BytesIO(config_json)) | ||
816 | 229 | |||
817 | 230 | # Build the registry manifest from the image manifest | ||
818 | 231 | # and associated configs | ||
819 | 232 | registry_manifest = cls._build_registry_manifest( | ||
820 | 233 | digests, config, config_json, config_sha, | ||
821 | 234 | preloaded_data[section["Config"]]) | ||
822 | 235 | |||
823 | 236 | # Upload the registry manifest | ||
824 | 237 | manifest_response = urlfetch( | ||
825 | 238 | "{}/v2/{}/manifests/{}".format( | ||
826 | 239 | push_rule.registry_credentials.url, | ||
827 | 240 | image_name, | ||
828 | 241 | tag), | ||
829 | 242 | json=registry_manifest, | ||
830 | 243 | headers={ | ||
831 | 244 | "Content-Type": | ||
832 | 245 | "application/" | ||
833 | 246 | "vnd.docker.distribution.manifest.v2+json" | ||
834 | 247 | }, | ||
835 | 248 | method="PUT") | ||
836 | 249 | if manifest_response.status_code != 201: | ||
837 | 250 | raise ManifestUploadFailed( | ||
838 | 251 | "Failed to upload manifest for {} in {}".format( | ||
839 | 252 | build.recipe.name, build.id)) | ||
840 | diff --git a/lib/lp/oci/model/ociregistrycredentials.py b/lib/lp/oci/model/ociregistrycredentials.py | |||
841 | index 2a70159..03b0f72 100644 | |||
842 | --- a/lib/lp/oci/model/ociregistrycredentials.py | |||
843 | +++ b/lib/lp/oci/model/ociregistrycredentials.py | |||
844 | @@ -80,8 +80,8 @@ class OCIRegistryCredentials(Storm): | |||
845 | 80 | def getCredentials(self): | 80 | def getCredentials(self): |
846 | 81 | container = getUtility(IEncryptedContainer, "oci-registry-secrets") | 81 | container = getUtility(IEncryptedContainer, "oci-registry-secrets") |
847 | 82 | try: | 82 | try: |
850 | 83 | return json.loads(container.decrypt(( | 83 | return json.loads(container.decrypt( |
851 | 84 | self._credentials['credentials_encrypted'])).decode("UTF-8")) | 84 | self._credentials['credentials_encrypted']).decode("UTF-8")) |
852 | 85 | except CryptoError as e: | 85 | except CryptoError as e: |
853 | 86 | # XXX twom 2020-03-18 This needs a better error | 86 | # XXX twom 2020-03-18 This needs a better error |
854 | 87 | # see SnapStoreClient.UnauthorizedUploadResponse | 87 | # see SnapStoreClient.UnauthorizedUploadResponse |
855 | diff --git a/lib/lp/oci/subscribers/ocirecipebuild.py b/lib/lp/oci/subscribers/ocirecipebuild.py | |||
856 | index 3e7f80d..2f3fc01 100644 | |||
857 | --- a/lib/lp/oci/subscribers/ocirecipebuild.py | |||
858 | +++ b/lib/lp/oci/subscribers/ocirecipebuild.py | |||
859 | @@ -9,8 +9,10 @@ __metaclass__ = type | |||
860 | 9 | 9 | ||
861 | 10 | from zope.component import getUtility | 10 | from zope.component import getUtility |
862 | 11 | 11 | ||
863 | 12 | from lp.buildmaster.enums import BuildStatus | ||
864 | 12 | from lp.oci.interfaces.ocirecipe import OCI_RECIPE_WEBHOOKS_FEATURE_FLAG | 13 | from lp.oci.interfaces.ocirecipe import OCI_RECIPE_WEBHOOKS_FEATURE_FLAG |
865 | 13 | from lp.oci.interfaces.ocirecipebuild import IOCIRecipeBuild | 14 | from lp.oci.interfaces.ocirecipebuild import IOCIRecipeBuild |
866 | 15 | from lp.oci.interfaces.ocirecipebuildjob import IOCIRegistryUploadJobSource | ||
867 | 14 | from lp.services.features import getFeatureFlag | 16 | from lp.services.features import getFeatureFlag |
868 | 15 | from lp.services.webapp.publisher import canonical_url | 17 | from lp.services.webapp.publisher import canonical_url |
869 | 16 | from lp.services.webhooks.interfaces import IWebhookSet | 18 | from lp.services.webhooks.interfaces import IWebhookSet |
870 | @@ -25,7 +27,7 @@ def _trigger_oci_recipe_build_webhook(build, action): | |||
871 | 25 | } | 27 | } |
872 | 26 | payload.update(compose_webhook_payload( | 28 | payload.update(compose_webhook_payload( |
873 | 27 | IOCIRecipeBuild, build, | 29 | IOCIRecipeBuild, build, |
875 | 28 | ["recipe", "status"])) | 30 | ["recipe", "status", "registry_upload_status"])) |
876 | 29 | getUtility(IWebhookSet).trigger( | 31 | getUtility(IWebhookSet).trigger( |
877 | 30 | build.recipe, "oci-recipe:build:0.1", payload) | 32 | build.recipe, "oci-recipe:build:0.1", payload) |
878 | 31 | 33 | ||
879 | @@ -34,9 +36,14 @@ def oci_recipe_build_created(build, event): | |||
880 | 34 | """Trigger events when a new OCI recipe build is created.""" | 36 | """Trigger events when a new OCI recipe build is created.""" |
881 | 35 | _trigger_oci_recipe_build_webhook(build, "created") | 37 | _trigger_oci_recipe_build_webhook(build, "created") |
882 | 36 | 38 | ||
885 | 37 | 39 | def oci_recipe_build_modified(build, event): | |
884 | 38 | def oci_recipe_build_status_changed(build, event): | ||
886 | 39 | """Trigger events when OCI recipe build statuses change.""" | 40 | """Trigger events when OCI recipe build statuses change.""" |
887 | 40 | if event.edited_fields is not None: | 41 | if event.edited_fields is not None: |
889 | 41 | if "status" in event.edited_fields: | 42 | status_changed = "status" in event.edited_fields |
890 | 43 | registry_changed = "registry_upload_status" in event.edited_fields | ||
891 | 44 | if status_changed or registry_changed: | ||
892 | 42 | _trigger_oci_recipe_build_webhook(build, "status-changed") | 45 | _trigger_oci_recipe_build_webhook(build, "status-changed") |
893 | 46 | if status_changed: | ||
894 | 47 | if (build.recipe.can_upload_to_registry and | ||
895 | 48 | build.status == BuildStatus.FULLYBUILT): | ||
896 | 49 | getUtility(IOCIRegistryUploadJobSource).create(build) | ||
897 | diff --git a/lib/lp/oci/tests/test_ocirecipe.py b/lib/lp/oci/tests/test_ocirecipe.py | |||
898 | index e158e2e..06ed382 100644 | |||
899 | --- a/lib/lp/oci/tests/test_ocirecipe.py | |||
900 | +++ b/lib/lp/oci/tests/test_ocirecipe.py | |||
901 | @@ -127,6 +127,7 @@ class TestOCIRecipe(TestCaseWithFactory): | |||
902 | 127 | "action": Equals("created"), | 127 | "action": Equals("created"), |
903 | 128 | "recipe": Equals(canonical_url(recipe, force_local_path=True)), | 128 | "recipe": Equals(canonical_url(recipe, force_local_path=True)), |
904 | 129 | "status": Equals("Needs building"), | 129 | "status": Equals("Needs building"), |
905 | 130 | 'registry_upload_status': Equals('Unscheduled'), | ||
906 | 130 | } | 131 | } |
907 | 131 | with person_logged_in(recipe.owner): | 132 | with person_logged_in(recipe.owner): |
908 | 132 | delivery = hook.deliveries.one() | 133 | delivery = hook.deliveries.one() |
909 | diff --git a/lib/lp/oci/tests/test_ocirecipebuild.py b/lib/lp/oci/tests/test_ocirecipebuild.py | |||
910 | index 400d78d..0579a8b 100644 | |||
911 | --- a/lib/lp/oci/tests/test_ocirecipebuild.py | |||
912 | +++ b/lib/lp/oci/tests/test_ocirecipebuild.py | |||
913 | @@ -182,6 +182,7 @@ class TestOCIRecipeBuild(TestCaseWithFactory): | |||
914 | 182 | "recipe": Equals( | 182 | "recipe": Equals( |
915 | 183 | canonical_url(self.build.recipe, force_local_path=True)), | 183 | canonical_url(self.build.recipe, force_local_path=True)), |
916 | 184 | "status": Equals("Successfully built"), | 184 | "status": Equals("Successfully built"), |
917 | 185 | 'registry_upload_status': Equals("Unscheduled"), | ||
918 | 185 | } | 186 | } |
919 | 186 | self.assertThat( | 187 | self.assertThat( |
920 | 187 | logger.output, LogsScheduledWebhooks([ | 188 | logger.output, LogsScheduledWebhooks([ |
921 | diff --git a/lib/lp/oci/tests/test_ocirecipebuildjob.py b/lib/lp/oci/tests/test_ocirecipebuildjob.py | |||
922 | index cf85684..f4628e0 100644 | |||
923 | --- a/lib/lp/oci/tests/test_ocirecipebuildjob.py | |||
924 | +++ b/lib/lp/oci/tests/test_ocirecipebuildjob.py | |||
925 | @@ -7,16 +7,65 @@ from __future__ import absolute_import, print_function, unicode_literals | |||
926 | 7 | 7 | ||
927 | 8 | __metaclass__ = type | 8 | __metaclass__ = type |
928 | 9 | 9 | ||
931 | 10 | from lp.oci.interfaces.ocirecipe import OCI_RECIPE_ALLOW_CREATE | 10 | from fixtures import FakeLogger |
932 | 11 | from lp.oci.interfaces.ocirecipebuildjob import IOCIRecipeBuildJob | 11 | from testtools.matchers import ( |
933 | 12 | Equals, | ||
934 | 13 | MatchesDict, | ||
935 | 14 | MatchesListwise, | ||
936 | 15 | MatchesStructure, | ||
937 | 16 | ) | ||
938 | 17 | import transaction | ||
939 | 18 | from zope.interface import implementer | ||
940 | 19 | |||
941 | 20 | from lp.buildmaster.enums import BuildStatus | ||
942 | 21 | from lp.oci.interfaces.ocirecipe import ( | ||
943 | 22 | OCI_RECIPE_ALLOW_CREATE, | ||
944 | 23 | OCI_RECIPE_WEBHOOKS_FEATURE_FLAG, | ||
945 | 24 | ) | ||
946 | 25 | from lp.oci.interfaces.ocirecipebuildjob import ( | ||
947 | 26 | IOCIRecipeBuildJob, | ||
948 | 27 | IOCIRegistryUploadJob, | ||
949 | 28 | IOCIRegistryUploadJobSource, | ||
950 | 29 | ) | ||
951 | 30 | from lp.oci.interfaces.ociregistryclient import IOCIRegistryClient | ||
952 | 12 | from lp.oci.model.ocirecipebuildjob import ( | 31 | from lp.oci.model.ocirecipebuildjob import ( |
953 | 13 | OCIRecipeBuildJob, | 32 | OCIRecipeBuildJob, |
954 | 14 | OCIRecipeBuildJobDerived, | 33 | OCIRecipeBuildJobDerived, |
955 | 15 | OCIRecipeBuildJobType, | 34 | OCIRecipeBuildJobType, |
956 | 35 | OCIRegistryUploadJob, | ||
957 | 16 | ) | 36 | ) |
958 | 37 | from lp.services.config import config | ||
959 | 17 | from lp.services.features.testing import FeatureFixture | 38 | from lp.services.features.testing import FeatureFixture |
960 | 39 | from lp.services.job.runner import JobRunner | ||
961 | 18 | from lp.testing import TestCaseWithFactory | 40 | from lp.testing import TestCaseWithFactory |
963 | 19 | from lp.testing.layers import DatabaseFunctionalLayer | 41 | from lp.testing.dbuser import dbuser |
964 | 42 | from lp.testing.fakemethod import FakeMethod | ||
965 | 43 | from lp.testing.fixture import ZopeUtilityFixture | ||
966 | 44 | from lp.testing.layers import ( | ||
967 | 45 | DatabaseFunctionalLayer, | ||
968 | 46 | LaunchpadZopelessLayer, | ||
969 | 47 | ) | ||
970 | 48 | from lp.testing.mail_helpers import pop_notifications | ||
971 | 49 | from lp.services.webapp import canonical_url | ||
972 | 50 | from lp.services.webhooks.testing import LogsScheduledWebhooks | ||
973 | 51 | |||
974 | 52 | |||
975 | 53 | def run_isolated_jobs(jobs): | ||
976 | 54 | """Run a sequence of jobs, ensuring transaction isolation. | ||
977 | 55 | |||
978 | 56 | We abort the transaction after each job to make sure that there is no | ||
979 | 57 | relevant uncommitted work. | ||
980 | 58 | """ | ||
981 | 59 | for job in jobs: | ||
982 | 60 | JobRunner([job]).runAll() | ||
983 | 61 | transaction.abort() | ||
984 | 62 | |||
985 | 63 | |||
986 | 64 | @implementer(IOCIRegistryClient) | ||
987 | 65 | class FakeRegistryClient: | ||
988 | 66 | |||
989 | 67 | def __init__(self): | ||
990 | 68 | self.upload = FakeMethod() | ||
991 | 20 | 69 | ||
992 | 21 | 70 | ||
993 | 22 | class FakeOCIBuildJob(OCIRecipeBuildJobDerived): | 71 | class FakeOCIBuildJob(OCIRecipeBuildJobDerived): |
994 | @@ -52,3 +101,104 @@ class TestOCIRecipeBuildJob(TestCaseWithFactory): | |||
995 | 52 | ('oci_project_name', oci_build.recipe.oci_project.name), | 101 | ('oci_project_name', oci_build.recipe.oci_project.name), |
996 | 53 | ] | 102 | ] |
997 | 54 | self.assertEqual(expected, oops) | 103 | self.assertEqual(expected, oops) |
998 | 104 | |||
999 | 105 | |||
1000 | 106 | class TestOCIRegistryUploadJob(TestCaseWithFactory): | ||
1001 | 107 | |||
1002 | 108 | layer = LaunchpadZopelessLayer | ||
1003 | 109 | |||
1004 | 110 | def setUp(self): | ||
1005 | 111 | super(TestOCIRegistryUploadJob, self).setUp() | ||
1006 | 112 | self.useFixture(FeatureFixture({ | ||
1007 | 113 | 'webhooks.new.enabled': 'true', | ||
1008 | 114 | OCI_RECIPE_WEBHOOKS_FEATURE_FLAG: 'on', | ||
1009 | 115 | OCI_RECIPE_ALLOW_CREATE: 'on' | ||
1010 | 116 | })) | ||
1011 | 117 | |||
1012 | 118 | def makeOCIRecipeBuild(self, **kwargs): | ||
1013 | 119 | ocibuild = self.factory.makeOCIRecipeBuild( | ||
1014 | 120 | builder=self.factory.makeBuilder(), **kwargs) | ||
1015 | 121 | ocibuild.updateStatus(BuildStatus.FULLYBUILT) | ||
1016 | 122 | self.factory.makeWebhook( | ||
1017 | 123 | target=ocibuild.recipe, event_types=["oci-recipe:build:0.1"]) | ||
1018 | 124 | return ocibuild | ||
1019 | 125 | |||
1020 | 126 | def assertWebhookDeliveries(self, ocibuild, | ||
1021 | 127 | expected_registry_upload_statuses, logger): | ||
1022 | 128 | hook = ocibuild.recipe.webhooks.one() | ||
1023 | 129 | deliveries = list(hook.deliveries) | ||
1024 | 130 | deliveries.reverse() | ||
1025 | 131 | expected_payloads = [{ | ||
1026 | 132 | "recipe_build": Equals( | ||
1027 | 133 | canonical_url(ocibuild, force_local_path=True)), | ||
1028 | 134 | "action": Equals("created"), | ||
1029 | 135 | "recipe": Equals( | ||
1030 | 136 | canonical_url(ocibuild.recipe, force_local_path=True)), | ||
1031 | 137 | "status": Equals("Successfully built"), | ||
1032 | 138 | "registry_upload_status": Equals("Pending")}] | ||
1033 | 139 | expected_payloads += [{ | ||
1034 | 140 | "recipe_build": Equals( | ||
1035 | 141 | canonical_url(ocibuild, force_local_path=True)), | ||
1036 | 142 | "action": Equals("status-changed"), | ||
1037 | 143 | "recipe": Equals( | ||
1038 | 144 | canonical_url(ocibuild.recipe, force_local_path=True)), | ||
1039 | 145 | "status": Equals("Successfully built"), | ||
1040 | 146 | "registry_upload_status": Equals(expected), | ||
1041 | 147 | } for expected in expected_registry_upload_statuses] | ||
1042 | 148 | matchers = [ | ||
1043 | 149 | MatchesStructure( | ||
1044 | 150 | event_type=Equals("oci-recipe:build:0.1"), | ||
1045 | 151 | payload=MatchesDict(expected_payload)) | ||
1046 | 152 | for expected_payload in expected_payloads] | ||
1047 | 153 | self.assertThat(deliveries, MatchesListwise(matchers)) | ||
1048 | 154 | with dbuser(config.IWebhookDeliveryJobSource.dbuser): | ||
1049 | 155 | for delivery in deliveries: | ||
1050 | 156 | self.assertEqual( | ||
1051 | 157 | "<WebhookDeliveryJob for webhook %d on %r>" % ( | ||
1052 | 158 | hook.id, hook.target), | ||
1053 | 159 | repr(delivery)) | ||
1054 | 160 | self.assertThat( | ||
1055 | 161 | logger.output, LogsScheduledWebhooks([ | ||
1056 | 162 | (hook, "oci-recipe:build:0.1", MatchesDict( | ||
1057 | 163 | expected_payload)) | ||
1058 | 164 | for expected_payload in expected_payloads])) | ||
1059 | 165 | |||
1060 | 166 | def test_provides_interface(self): | ||
1061 | 167 | # `OCIRegistryUploadJob` objects provide `IOCIRegistryUploadJob`. | ||
1062 | 168 | ocibuild = self.factory.makeOCIRecipeBuild() | ||
1063 | 169 | job = OCIRegistryUploadJob.create(ocibuild) | ||
1064 | 170 | self.assertProvides(job, IOCIRegistryUploadJob) | ||
1065 | 171 | |||
1066 | 172 | def test_run(self): | ||
1067 | 173 | logger = self.useFixture(FakeLogger()) | ||
1068 | 174 | ocibuild = self.makeOCIRecipeBuild() | ||
1069 | 175 | self.assertContentEqual([], ocibuild.registry_upload_jobs) | ||
1070 | 176 | job = OCIRegistryUploadJob.create(ocibuild) | ||
1071 | 177 | client = FakeRegistryClient() | ||
1072 | 178 | self.useFixture(ZopeUtilityFixture(client, IOCIRegistryClient)) | ||
1073 | 179 | with dbuser(config.IOCIRegistryUploadJobSource.dbuser): | ||
1074 | 180 | run_isolated_jobs([job]) | ||
1075 | 181 | self.assertEqual([((ocibuild,), {})], client.upload.calls) | ||
1076 | 182 | self.assertContentEqual([job], ocibuild.registry_upload_jobs) | ||
1077 | 183 | self.assertIsNone(job.error_message) | ||
1078 | 184 | self.assertEqual([], pop_notifications()) | ||
1079 | 185 | self.assertWebhookDeliveries( | ||
1080 | 186 | ocibuild, ["Uploaded"], logger) | ||
1081 | 187 | |||
1082 | 188 | def test_run_failed(self): | ||
1083 | 189 | # A failed run sets the registry upload status to FAILED. | ||
1084 | 190 | logger = self.useFixture(FakeLogger()) | ||
1085 | 191 | ocibuild = self.makeOCIRecipeBuild() | ||
1086 | 192 | self.assertContentEqual([], ocibuild.registry_upload_jobs) | ||
1087 | 193 | job = OCIRegistryUploadJob.create(ocibuild) | ||
1088 | 194 | client = FakeRegistryClient() | ||
1089 | 195 | client.upload.failure = ValueError("An upload failure") | ||
1090 | 196 | self.useFixture(ZopeUtilityFixture(client, IOCIRegistryClient)) | ||
1091 | 197 | with dbuser(config.IOCIRegistryUploadJobSource.dbuser): | ||
1092 | 198 | run_isolated_jobs([job]) | ||
1093 | 199 | self.assertEqual([((ocibuild,), {})], client.upload.calls) | ||
1094 | 200 | self.assertContentEqual([job], ocibuild.registry_upload_jobs) | ||
1095 | 201 | self.assertEqual("An upload failure", job.error_message) | ||
1096 | 202 | self.assertEqual([], pop_notifications()) | ||
1097 | 203 | self.assertWebhookDeliveries( | ||
1098 | 204 | ocibuild, ["Failed to upload"], logger) | ||
1099 | diff --git a/lib/lp/oci/tests/test_ociregistryclient.py b/lib/lp/oci/tests/test_ociregistryclient.py | |||
1100 | 55 | new file mode 100644 | 205 | new file mode 100644 |
1101 | index 0000000..5f83e38 | |||
1102 | --- /dev/null | |||
1103 | +++ b/lib/lp/oci/tests/test_ociregistryclient.py | |||
1104 | @@ -0,0 +1,204 @@ | |||
1105 | 1 | # Copyright 2020 Canonical Ltd. This software is licensed under the | ||
1106 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | ||
1107 | 3 | |||
1108 | 4 | """Tests for the OCI Registry client.""" | ||
1109 | 5 | |||
1110 | 6 | from __future__ import absolute_import, print_function, unicode_literals | ||
1111 | 7 | |||
1112 | 8 | __metaclass__ = type | ||
1113 | 9 | |||
1114 | 10 | import json | ||
1115 | 11 | |||
1116 | 12 | from fixtures import MockPatch | ||
1117 | 13 | from requests.exceptions import HTTPError | ||
1118 | 14 | import responses | ||
1119 | 15 | from testtools.matchers import ( | ||
1120 | 16 | Equals, | ||
1121 | 17 | MatchesDict, | ||
1122 | 18 | MatchesListwise, | ||
1123 | 19 | ) | ||
1124 | 20 | import transaction | ||
1125 | 21 | |||
1126 | 22 | from lp.oci.model.ociregistryclient import OCIRegistryClient | ||
1127 | 23 | from lp.oci.tests.helpers import OCIConfigHelperMixin | ||
1128 | 24 | from lp.testing import TestCaseWithFactory | ||
1129 | 25 | from lp.testing.layers import LaunchpadZopelessLayer | ||
1130 | 26 | |||
1131 | 27 | |||
1132 | 28 | class TestOCIRegistryClient(OCIConfigHelperMixin, TestCaseWithFactory): | ||
1133 | 29 | |||
1134 | 30 | layer = LaunchpadZopelessLayer | ||
1135 | 31 | |||
1136 | 32 | def setUp(self): | ||
1137 | 33 | super(TestOCIRegistryClient, self).setUp() | ||
1138 | 34 | self.setConfig() | ||
1139 | 35 | self.manifest = [{ | ||
1140 | 36 | "Config": "config_file_1.json", | ||
1141 | 37 | "Layers": ["layer_1/layer.tar", "layer_2/layer.tar"]}] | ||
1142 | 38 | self.digests = [{ | ||
1143 | 39 | "diff_id_1": { | ||
1144 | 40 | "digest": "digest_1", | ||
1145 | 41 | "source": "test/base_1", | ||
1146 | 42 | "layer_id": "layer_1" | ||
1147 | 43 | }, | ||
1148 | 44 | "diff_id_2": { | ||
1149 | 45 | "digest": "digest_2", | ||
1150 | 46 | "source": "", | ||
1151 | 47 | "layer_id": "layer_2" | ||
1152 | 48 | } | ||
1153 | 49 | }] | ||
1154 | 50 | self.config = {"rootfs": {"diff_ids": ["diff_id_1", "diff_id_2"]}} | ||
1155 | 51 | self.build = self.factory.makeOCIRecipeBuild() | ||
1156 | 52 | self.factory.makeOCIPushRule(recipe=self.build.recipe) | ||
1157 | 53 | self.client = OCIRegistryClient() | ||
1158 | 54 | |||
1159 | 55 | def _makeFiles(self): | ||
1160 | 56 | self.factory.makeOCIFile( | ||
1161 | 57 | build=self.build, | ||
1162 | 58 | content=json.dumps(self.manifest), | ||
1163 | 59 | filename='manifest.json', | ||
1164 | 60 | ) | ||
1165 | 61 | self.factory.makeOCIFile( | ||
1166 | 62 | build=self.build, | ||
1167 | 63 | content=json.dumps(self.digests), | ||
1168 | 64 | filename='digests.json', | ||
1169 | 65 | ) | ||
1170 | 66 | self.factory.makeOCIFile( | ||
1171 | 67 | build=self.build, | ||
1172 | 68 | content=json.dumps(self.config), | ||
1173 | 69 | filename='config_file_1.json' | ||
1174 | 70 | ) | ||
1175 | 71 | |||
1176 | 72 | # make layer files | ||
1177 | 73 | self.layer_1_file = self.factory.makeOCIFile( | ||
1178 | 74 | build=self.build, | ||
1179 | 75 | content="digest_1", | ||
1180 | 76 | filename="digest_1_filename", | ||
1181 | 77 | layer_file_digest="digest_1" | ||
1182 | 78 | ) | ||
1183 | 79 | self.layer_2_file = self.factory.makeOCIFile( | ||
1184 | 80 | build=self.build, | ||
1185 | 81 | content="digest_2", | ||
1186 | 82 | filename="digest_2_filename", | ||
1187 | 83 | layer_file_digest="digest_2" | ||
1188 | 84 | ) | ||
1189 | 85 | |||
1190 | 86 | transaction.commit() | ||
1191 | 87 | |||
1192 | 88 | @responses.activate | ||
1193 | 89 | def test_upload(self): | ||
1194 | 90 | self._makeFiles() | ||
1195 | 91 | self.useFixture(MockPatch( | ||
1196 | 92 | "lp.oci.model.ociregistryclient.OCIRegistryClient._upload")) | ||
1197 | 93 | self.useFixture(MockPatch( | ||
1198 | 94 | "lp.oci.model.ociregistryclient.OCIRegistryClient._upload_layer")) | ||
1199 | 95 | |||
1200 | 96 | manifests_url = "{}/v2/{}/manifests/edge".format( | ||
1201 | 97 | self.build.recipe.push_rules[0].registry_credentials.url, | ||
1202 | 98 | self.build.recipe.push_rules[0].image_name | ||
1203 | 99 | ) | ||
1204 | 100 | responses.add("PUT", manifests_url, status=201) | ||
1205 | 101 | self.client.upload(self.build) | ||
1206 | 102 | |||
1207 | 103 | request = json.loads(responses.calls[0].request.body) | ||
1208 | 104 | |||
1209 | 105 | self.assertThat(request, MatchesDict({ | ||
1210 | 106 | "layers": MatchesListwise([ | ||
1211 | 107 | MatchesDict({ | ||
1212 | 108 | "mediaType": Equals( | ||
1213 | 109 | "application/vnd.docker.image.rootfs.diff.tar.gzip"), | ||
1214 | 110 | "digest": Equals("diff_id_1"), | ||
1215 | 111 | "size": Equals(8)}), | ||
1216 | 112 | MatchesDict({ | ||
1217 | 113 | "mediaType": Equals( | ||
1218 | 114 | "application/vnd.docker.image.rootfs.diff.tar.gzip"), | ||
1219 | 115 | "digest": Equals("diff_id_2"), | ||
1220 | 116 | "size": Equals(8)}) | ||
1221 | 117 | ]), | ||
1222 | 118 | "schemaVersion": Equals(2), | ||
1223 | 119 | "config": MatchesDict({ | ||
1224 | 120 | "mediaType": Equals( | ||
1225 | 121 | "application/vnd.docker.container.image.v1+json"), | ||
1226 | 122 | "digest": Equals( | ||
1227 | 123 | "sha256:33b69b4b6e106f9fc7a8b93409" | ||
1228 | 124 | "36c85cf7f84b2d017e7b55bee6ab214761f6ab"), | ||
1229 | 125 | "size": Equals(52) | ||
1230 | 126 | }), | ||
1231 | 127 | "mediaType": Equals( | ||
1232 | 128 | "application/vnd.docker.distribution.manifest.v2+json") | ||
1233 | 129 | })) | ||
1234 | 130 | |||
1235 | 131 | def test_preloadFiles(self): | ||
1236 | 132 | self._makeFiles() | ||
1237 | 133 | files = self.client._preloadFiles( | ||
1238 | 134 | self.build, self.manifest, self.digests[0]) | ||
1239 | 135 | |||
1240 | 136 | self.assertThat(files, MatchesDict({ | ||
1241 | 137 | 'config_file_1.json': MatchesDict({ | ||
1242 | 138 | 'config_file': Equals(self.config), | ||
1243 | 139 | 'diff_id_1': Equals(self.layer_1_file.library_file), | ||
1244 | 140 | 'diff_id_2': Equals(self.layer_2_file.library_file)})})) | ||
1245 | 141 | |||
1246 | 142 | def test_calculateTag(self): | ||
1247 | 143 | result = self.client._calculateTag( | ||
1248 | 144 | self.build, self.build.recipe.push_rules[0]) | ||
1249 | 145 | self.assertEqual("edge", result) | ||
1250 | 146 | |||
1251 | 147 | def test_build_registry_manifest(self): | ||
1252 | 148 | self._makeFiles() | ||
1253 | 149 | preloaded_data = self.client._preloadFiles( | ||
1254 | 150 | self.build, self.manifest, self.digests[0]) | ||
1255 | 151 | manifest = self.client._build_registry_manifest( | ||
1256 | 152 | self.digests[0], | ||
1257 | 153 | self.config, | ||
1258 | 154 | json.dumps(self.config), | ||
1259 | 155 | "config-sha", | ||
1260 | 156 | preloaded_data["config_file_1.json"]) | ||
1261 | 157 | self.assertThat(manifest, MatchesDict({ | ||
1262 | 158 | "layers": MatchesListwise([ | ||
1263 | 159 | MatchesDict({ | ||
1264 | 160 | "mediaType": Equals( | ||
1265 | 161 | "application/vnd.docker.image.rootfs.diff.tar.gzip"), | ||
1266 | 162 | "digest": Equals("diff_id_1"), | ||
1267 | 163 | "size": Equals(8)}), | ||
1268 | 164 | MatchesDict({ | ||
1269 | 165 | "mediaType": Equals( | ||
1270 | 166 | "application/vnd.docker.image.rootfs.diff.tar.gzip"), | ||
1271 | 167 | "digest": Equals("diff_id_2"), | ||
1272 | 168 | "size": Equals(8)}) | ||
1273 | 169 | ]), | ||
1274 | 170 | "schemaVersion": Equals(2), | ||
1275 | 171 | "config": MatchesDict({ | ||
1276 | 172 | "mediaType": Equals( | ||
1277 | 173 | "application/vnd.docker.container.image.v1+json"), | ||
1278 | 174 | "digest": Equals("sha256:config-sha"), | ||
1279 | 175 | "size": Equals(52) | ||
1280 | 176 | }), | ||
1281 | 177 | "mediaType": Equals( | ||
1282 | 178 | "application/vnd.docker.distribution.manifest.v2+json") | ||
1283 | 179 | })) | ||
1284 | 180 | |||
1285 | 181 | @responses.activate | ||
1286 | 182 | def test_upload_handles_existing(self): | ||
1287 | 183 | blobs_url = "{}/v2/{}/blobs/{}".format( | ||
1288 | 184 | self.build.recipe.push_rules[0].registry_credentials.url, | ||
1289 | 185 | "test-name", | ||
1290 | 186 | "test-digest") | ||
1291 | 187 | responses.add("HEAD", blobs_url, status=200) | ||
1292 | 188 | self.client._upload( | ||
1293 | 189 | "test-digest", self.build.recipe.push_rules[0], "test-name", None) | ||
1294 | 190 | |||
1295 | 191 | @responses.activate | ||
1296 | 192 | def test_upload_raises_non_404(self): | ||
1297 | 193 | blobs_url = "{}/v2/{}/blobs/{}".format( | ||
1298 | 194 | self.build.recipe.push_rules[0].registry_credentials.url, | ||
1299 | 195 | "test-name", | ||
1300 | 196 | "test-digest") | ||
1301 | 197 | responses.add("HEAD", blobs_url, status=500) | ||
1302 | 198 | self.assertRaises( | ||
1303 | 199 | HTTPError, | ||
1304 | 200 | self.client._upload, | ||
1305 | 201 | "test-digest", | ||
1306 | 202 | self.build.recipe.push_rules[0], | ||
1307 | 203 | "test-name", | ||
1308 | 204 | None) | ||
1309 | diff --git a/lib/lp/services/config/schema-lazr.conf b/lib/lp/services/config/schema-lazr.conf | |||
1310 | index 18b452b..ec65b96 100644 | |||
1311 | --- a/lib/lp/services/config/schema-lazr.conf | |||
1312 | +++ b/lib/lp/services/config/schema-lazr.conf | |||
1313 | @@ -1821,6 +1821,7 @@ job_sources: | |||
1314 | 1821 | IExpiringMembershipNotificationJobSource, | 1821 | IExpiringMembershipNotificationJobSource, |
1315 | 1822 | IGitRepositoryModifiedMailJobSource, | 1822 | IGitRepositoryModifiedMailJobSource, |
1316 | 1823 | IMembershipNotificationJobSource, | 1823 | IMembershipNotificationJobSource, |
1317 | 1824 | IOCIRegistryUploadJobSource, | ||
1318 | 1824 | IPackageUploadNotificationJobSource, | 1825 | IPackageUploadNotificationJobSource, |
1319 | 1825 | IPersonDeactivateJobSource, | 1826 | IPersonDeactivateJobSource, |
1320 | 1826 | IPersonMergeJobSource, | 1827 | IPersonMergeJobSource, |
1321 | @@ -1984,6 +1985,11 @@ module: lp.snappy.interfaces.snapbuildjob | |||
1322 | 1984 | dbuser: snap-build-job | 1985 | dbuser: snap-build-job |
1323 | 1985 | crontab_group: MAIN | 1986 | crontab_group: MAIN |
1324 | 1986 | 1987 | ||
1325 | 1988 | [IOCIRegistryUploadJobSource] | ||
1326 | 1989 | module: lp.oci.interfaces.ocirecipebuildjob | ||
1327 | 1990 | dbuser: oci-build-job | ||
1328 | 1991 | crontab_group: MAIN | ||
1329 | 1992 | |||
1330 | 1987 | [ITeamInvitationNotificationJobSource] | 1993 | [ITeamInvitationNotificationJobSource] |
1331 | 1988 | module: lp.registry.interfaces.persontransferjob | 1994 | module: lp.registry.interfaces.persontransferjob |
1332 | 1989 | dbuser: person-transfer-job | 1995 | dbuser: person-transfer-job |
Some things to fix, but mostly looks OK now. As you probably know, landing is blocked until https:/ /portal. admin.canonical .com/C125275 is done.