Merge ~bowenfan/snapstore-client:SN2487b into snapstore-client:main
- Git
- lp:~bowenfan/snapstore-client
- SN2487b
- Merge into main
Proposed by
Bowen Fan
Status: | Merged |
---|---|
Merged at revision: | 9dc36fbdcaa1baa35cfd15e3c8360589453d24c2 |
Proposed branch: | ~bowenfan/snapstore-client:SN2487b |
Merge into: | snapstore-client:main |
Diff against target: |
1177 lines (+727/-144) 5 files modified
store_admin/cli/charms.py (+125/-0) store_admin/cli/runner.py (+4/-1) store_admin/logic/charms.py (+235/-48) store_admin/logic/tests/test_charms.py (+350/-81) store_admin/types.py (+13/-14) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Ubuntu One hackers | Pending | ||
Review via email: mp+460621@code.launchpad.net |
Commit message
Description of the change
To post a comment you must log in.
Updating diff...
An updated diff will be available in a few minutes. Reload to see the changes.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/store_admin/cli/charms.py b/store_admin/cli/charms.py |
2 | new file mode 100644 |
3 | index 0000000..f832896 |
4 | --- /dev/null |
5 | +++ b/store_admin/cli/charms.py |
6 | @@ -0,0 +1,125 @@ |
7 | +import logging |
8 | +import pathlib |
9 | + |
10 | +import click |
11 | + |
12 | +from store_admin.logic import charms |
13 | + |
14 | +logger = logging.getLogger(__name__) |
15 | + |
16 | + |
17 | +@click.command(name="charms") |
18 | +@click.argument( |
19 | + "bundle_filepath", |
20 | + required=True, |
21 | + type=click.Path( |
22 | + exists=True, |
23 | + file_okay=True, |
24 | + dir_okay=False, |
25 | + path_type=pathlib.Path, |
26 | + ), |
27 | +) |
28 | +@click.option( |
29 | + "--export-dir", |
30 | + multiple=False, |
31 | + help="Directory to export charms to", |
32 | + type=click.Path( |
33 | + exists=True, |
34 | + file_okay=False, |
35 | + dir_okay=True, |
36 | + writable=True, |
37 | + path_type=pathlib.Path, |
38 | + ), |
39 | +) |
40 | +@click.option( |
41 | + "--series", |
42 | + multiple=False, |
43 | + default=charms.DEFAULT_SERIES, |
44 | + help="Series of the exported charms (jammy by default)", |
45 | +) |
46 | +@click.option( |
47 | + "--arch", |
48 | + "architecture", |
49 | + multiple=False, |
50 | + default=charms.DEFAULT_ARCHITECTURE, |
51 | + help="Architecture of the exported charms (amd64 by default)", |
52 | +) |
53 | +def export_charms(bundle_filepath, export_dir, series, architecture): |
54 | + """Download charms, resources, and metadata for importing to an offline store.""" |
55 | + export_tar_path = charms.export_charms( |
56 | + bundle_yaml_filepath=bundle_filepath, |
57 | + export_dir_path=export_dir, |
58 | + series=series, |
59 | + architecture=architecture, |
60 | + ) |
61 | + logger.info(f"Successfully exported charms to: {export_tar_path}") |
62 | + |
63 | + |
64 | +@click.command(name="bundle") |
65 | +@click.argument( |
66 | + "bundle_name", |
67 | + required=True, |
68 | +) |
69 | +@click.option( |
70 | + "--export-dir", |
71 | + multiple=False, |
72 | + help="Directory to export the bundle to", |
73 | + type=click.Path( |
74 | + exists=True, |
75 | + file_okay=False, |
76 | + dir_okay=True, |
77 | + writable=True, |
78 | + path_type=pathlib.Path, |
79 | + ), |
80 | +) |
81 | +@click.option( |
82 | + "--channel", |
83 | + multiple=False, |
84 | + default="latest/stable", |
85 | + help="Channel of the charm bundle to export (latest/stable by default)", |
86 | +) |
87 | +@click.option( |
88 | + "--series", |
89 | + multiple=False, |
90 | + default=charms.DEFAULT_SERIES, |
91 | + help="Series for the bundle's charms (jammy by default)", |
92 | +) |
93 | +@click.option( |
94 | + "--arch", |
95 | + "architecture", |
96 | + multiple=False, |
97 | + default=charms.DEFAULT_ARCHITECTURE, |
98 | + help="Architecture for the bundle's charms (amd64 by default)", |
99 | +) |
100 | +def export_bundle(bundle_name, export_dir, channel, series, architecture): |
101 | + """Download a charm bundle and its components for offline store import. |
102 | + |
103 | + This command is for charm-bundle packages in the Store. It will automatically fetch |
104 | + all component charms and their component resources. |
105 | + """ |
106 | + export_bundle_name, export_tar_path = charms.export_charm_bundle( |
107 | + bundle_name=bundle_name, |
108 | + export_dir_path=export_dir, |
109 | + channel=channel, |
110 | + charm_series=series, |
111 | + charm_architecture=architecture, |
112 | + ) |
113 | + logger.info( |
114 | + f"Successfully exported charm bundle {export_bundle_name}: {export_tar_path}" |
115 | + ) |
116 | + |
117 | + |
118 | +@click.command(name="get-bundle-yaml") # TODO decide on a better command name |
119 | +@click.argument( |
120 | + "bundle_name", |
121 | + required=True, |
122 | +) |
123 | +@click.option( |
124 | + "--channel", |
125 | + multiple=False, |
126 | + default="latest/stable", |
127 | + help="Channel of the charm bundle to export (latest/stable by default)", |
128 | +) |
129 | +def get_bundle_yaml(bundle_name, channel): |
130 | + """Fetch a charm bundle and print its bundle.yaml to stdout.""" |
131 | + charms.get_bundle_yaml(bundle_name, channel) |
132 | diff --git a/store_admin/cli/runner.py b/store_admin/cli/runner.py |
133 | index 14d88a4..e4090e8 100644 |
134 | --- a/store_admin/cli/runner.py |
135 | +++ b/store_admin/cli/runner.py |
136 | @@ -3,7 +3,7 @@ import logging |
137 | import click |
138 | |
139 | from store_admin import defaults, log, version |
140 | -from store_admin.cli import common, model_service, overrides, snaps, stores |
141 | +from store_admin.cli import common, model_service, overrides, snaps, stores, charms |
142 | |
143 | |
144 | @click.group( |
145 | @@ -79,6 +79,7 @@ def export(): |
146 | run.add_command(common.login) |
147 | run.add_command(common.register) |
148 | run.add_command(model_service.register_key) |
149 | +run.add_command(charms.get_bundle_yaml) |
150 | |
151 | # Commands under create |
152 | create.add_command(model_service.create_model) |
153 | @@ -109,3 +110,5 @@ remove.add_command(model_service.remove_serial_policy) |
154 | export.add_command(stores.export_store) |
155 | export.add_command(common.export_token) |
156 | export.add_command(snaps.export_snap) |
157 | +export.add_command(charms.export_charms) |
158 | +export.add_command(charms.export_bundle) |
159 | diff --git a/store_admin/logic/charms.py b/store_admin/logic/charms.py |
160 | index 8137f5d..5f65edb 100644 |
161 | --- a/store_admin/logic/charms.py |
162 | +++ b/store_admin/logic/charms.py |
163 | @@ -5,6 +5,7 @@ import pathlib |
164 | import shutil |
165 | import tarfile |
166 | import tempfile |
167 | +import zipfile |
168 | import yaml |
169 | from store_admin.types import ( |
170 | CharmChannel, |
171 | @@ -32,6 +33,7 @@ from store_admin.logic.snaps import ( |
172 | |
173 | DEFAULT_SERIES = "jammy" |
174 | DEFAULT_ARCHITECTURE = "amd64" |
175 | +DEFAULT_CHARMS_EXPORT_FILENAME_PREFIX = "charms-export" |
176 | |
177 | CHARM_INFO_FIELDS = {"type", "id", "name", "result", "channel-map"} |
178 | |
179 | @@ -41,7 +43,7 @@ REFRESH_REVISION_NOT_FOUND_CODE = "revision-not-found" |
180 | logger = logging.getLogger(__name__) |
181 | |
182 | |
183 | -def parse_charm_bundle_yaml(bundle_yaml_filepath, architecture): |
184 | +def parse_charm_bundle_yaml(bundle_yaml_filepath, series, architecture): |
185 | """Parse a bundle.yaml file into a list of CharmImport objects.""" |
186 | # Onus is on the user to conform to the yaml spec, i.e. no duplicate keys |
187 | # pyyaml will clobber the previous entries where keys are duplicated |
188 | @@ -49,16 +51,30 @@ def parse_charm_bundle_yaml(bundle_yaml_filepath, architecture): |
189 | bundle = yaml.safe_load(bundle_yaml) |
190 | |
191 | applications = bundle["applications"] |
192 | - preferred_series = bundle.get("series", DEFAULT_SERIES) |
193 | - charms = [ |
194 | - CharmImport( |
195 | - package_name=config["charm"], |
196 | - channel=config.get("channel"), |
197 | - series=config.get("series", preferred_series), |
198 | - architecture=architecture, |
199 | + |
200 | + # The series resolving logic here is rudimentary, but should be aligned with Juju. |
201 | + # It won't work in some cases, e.g. if the series and channel are not |
202 | + # user-specified, where we would fetch from the default track with a default |
203 | + # series of "jammy" (latest LTS). This can fail if the default track does not |
204 | + # support Jammy. We leave it to the user to provide feasible parameters. |
205 | + preferred_series = bundle.get("series", series) |
206 | + # We remap "kubernetes" to "focal", as we do in packagereview |
207 | + # See packagereview.types.CharmBase.from_series_str |
208 | + if preferred_series == "kubernetes": |
209 | + preferred_series = "focal" |
210 | + charms = [] |
211 | + for config in applications.values(): |
212 | + charm_series = config.get("series", preferred_series) |
213 | + if charm_series == "kubernetes": |
214 | + charm_series = "focal" |
215 | + charms.append( |
216 | + CharmImport( |
217 | + package_name=config["charm"], |
218 | + channel=config.get("channel"), |
219 | + series=charm_series, |
220 | + architecture=architecture, |
221 | + ) |
222 | ) |
223 | - for config in applications.values() |
224 | - ] |
225 | return charms |
226 | |
227 | |
228 | @@ -140,12 +156,12 @@ def make_charm_download_actions(charms_to_import, charms_info): |
229 | def _get_charm_channel_from_name_and_revision(channel_map, channel_name, revision): |
230 | """Helper function to search channel map entries for a given channel entry.""" |
231 | if "latest/" in channel_name: |
232 | - channel_name = channel_name.split("/")[1] |
233 | + channel_name_without_track = channel_name.split("/")[1] |
234 | for channel_dict in channel_map: |
235 | if ( |
236 | channel_dict["channel"]["name"] == channel_name |
237 | - and channel_dict["revision"]["revision"] == revision |
238 | - ): |
239 | + or channel_dict["channel"]["name"] == channel_name_without_track |
240 | + ) and channel_dict["revision"]["revision"] == revision: |
241 | return channel_dict |
242 | raise StoreRequestError("Inconsistent channel data") |
243 | |
244 | @@ -222,10 +238,12 @@ def charm_revisions_from_info(charms_info, charms_download_info): |
245 | binary_sha256=revision["download"]["hash-sha-256"], |
246 | ) |
247 | package_id = charm_info["id"] |
248 | + package_type = charm_info["type"] |
249 | charm_revision = CharmRevision( |
250 | channel=channel, |
251 | package_name=charm_name, |
252 | package_id=package_id, |
253 | + package_type=package_type, |
254 | revision=revision["revision"], |
255 | created_at=revision["created-at"], |
256 | architectures=revision["bases"], |
257 | @@ -248,10 +266,11 @@ def write_channel_map(charm_info_data, target_dir_path): |
258 | |
259 | |
260 | def download_charm_files(charm_revision, target_dir_path, file_read_cb): |
261 | - """Download charm binaries and its resource binaries from the online Charmhub. |
262 | + """Download a bundle binary, or a charm and its resource binaries from Charmhub. |
263 | |
264 | :param CharmRevision charm_revision: A CharmRevision object that contains download |
265 | - URLs for the charm and the resource(s) that it was released with. |
266 | + URLs for the either the bundle, or the charm and the resource(s) that it was |
267 | + released with. |
268 | :param pathlib.Path target_dir_path: target directory path |
269 | :param callable file_read_cb: Optional callback used for reading in charm and |
270 | resource revision blobs. |
271 | @@ -267,7 +286,11 @@ def download_charm_files(charm_revision, target_dir_path, file_read_cb): |
272 | shutil.copyfileobj(raw_response, blob_file) |
273 | logger.debug(f"Downloaded blob to {charm_revision_blob_path}") |
274 | |
275 | - if charm_revision.channel.resource_revisions: |
276 | + # Short circuit bundles here as their resources will always be None |
277 | + if ( |
278 | + charm_revision.package_type == "charm" |
279 | + and charm_revision.channel.resource_revisions |
280 | + ): |
281 | resource_dir = target_dir_path / "resources" |
282 | resource_dir.mkdir(parents=True, exist_ok=True) |
283 | for resource in charm_revision.channel.resource_revisions: |
284 | @@ -285,17 +308,132 @@ def download_charm_files(charm_revision, target_dir_path, file_read_cb): |
285 | logger.debug(f"Downloaded blob to {resource_revision_blob_path}") |
286 | |
287 | |
288 | +def _fetch_store_info_for_packages(packages_to_import): |
289 | + """Helper function for hitting /info and /refresh and parsing the results. |
290 | + |
291 | + :param list[CharmImport] packages_to_import: list of charms to fetch |
292 | + """ |
293 | + devicegw_root = defaults.get_devicegw_url() |
294 | + packages_info = { |
295 | + pkg.package_name: get_charm_info(devicegw_root, pkg) |
296 | + for pkg in packages_to_import |
297 | + } |
298 | + packages_download_actions = make_charm_download_actions( |
299 | + packages_to_import, packages_info |
300 | + ) |
301 | + packages_download_info = { |
302 | + res["name"]: res |
303 | + for res in get_charms_download_info(devicegw_root, packages_download_actions) |
304 | + } |
305 | + packages_revisions = charm_revisions_from_info( |
306 | + packages_info, packages_download_info |
307 | + ) |
308 | + |
309 | + return (packages_info, packages_revisions) |
310 | + |
311 | + |
312 | def export_charms( |
313 | - bundle_yaml_filepath, export_dir_path, architecture=DEFAULT_ARCHITECTURE |
314 | + bundle_yaml_filepath, |
315 | + export_dir_path, |
316 | + series=DEFAULT_SERIES, |
317 | + architecture=DEFAULT_ARCHITECTURE, |
318 | ): |
319 | """Orchestrating function for exporting a set of charms given a bundle.yaml. |
320 | |
321 | + Output structure: |
322 | + |
323 | + <export_dir_path>/ |
324 | + ├─ charms-export-<timestamp>.tar.gz/ |
325 | + │ ├─ bundle.yaml |
326 | + | ├─ <charm-name>.tar.gz/ |
327 | + │ │ ├─ <charm-name>_<charm-revision>.charm |
328 | + │ │ ├─ channel-map.json |
329 | + │ │ ├─ resources/ |
330 | + │ │ │ ├─ <charm-name>.<resource-name>_<resource-revision> |
331 | + |
332 | + |
333 | :param pathlib.Path bundle_yaml_filepath: path to the bundle.yaml to be exported |
334 | :param pathlib.Path export_dir_path: export directory path |
335 | - :param str architecture: architecture of the charms to fetch, defaults to amd64 |
336 | + :param str series: series of the charms to fetch, default "jammy" |
337 | + :param str architecture: architecture of the charms to fetch, default "amd64" |
338 | """ |
339 | - devicegw_root = defaults.get_devicegw_url() |
340 | + if not export_dir_path: |
341 | + snap_user_common_dir = os.getenv("SNAP_USER_COMMON") |
342 | + if snap_user_common_dir: |
343 | + export_dir_path = pathlib.Path(snap_user_common_dir) / "export" |
344 | + export_dir_path.mkdir(exist_ok=True) |
345 | + else: |
346 | + export_dir_path = pathlib.Path(os.getcwd()) |
347 | |
348 | + timestamp = ( |
349 | + datetime.datetime.utcnow().replace(microsecond=0).strftime("%Y%m%dT%H%M%S") |
350 | + ) |
351 | + charms_to_import = parse_charm_bundle_yaml( |
352 | + bundle_yaml_filepath, series, architecture |
353 | + ) |
354 | + charms_info, charms_revisions = _fetch_store_info_for_packages(charms_to_import) |
355 | + |
356 | + with tempfile.TemporaryDirectory() as bundle_dir_path: |
357 | + bundle_dir_path = pathlib.Path(bundle_dir_path) |
358 | + |
359 | + for charm_name, charm_revision in charms_revisions.items(): |
360 | + with tempfile.TemporaryDirectory() as charm_dir_path: |
361 | + charm_dir_path = pathlib.Path(charm_dir_path) |
362 | + |
363 | + write_channel_map(charms_info[charm_name], charm_dir_path) |
364 | + download_charm_files( |
365 | + charm_revision, |
366 | + charm_dir_path, |
367 | + progress_copy_to_file, |
368 | + ) |
369 | + # Pack each charm, its metadata, and resources into a tarball |
370 | + charm_tar_path = bundle_dir_path / f"{charm_name}.tar.gz" |
371 | + with tarfile.open(charm_tar_path, "w:gz", encoding="utf-8") as tar: |
372 | + for f in charm_dir_path.iterdir(): |
373 | + tar.add(str(f), f.name) |
374 | + # Pack originally supplied bundle.yaml into the final tarball |
375 | + shutil.copyfile(bundle_yaml_filepath, bundle_dir_path / "bundle.yaml") |
376 | + bundle_tar_path = ( |
377 | + export_dir_path |
378 | + / f"{DEFAULT_CHARMS_EXPORT_FILENAME_PREFIX}-{timestamp}.tar.gz" |
379 | + ) |
380 | + # Pack charm tarballs into a root tarball |
381 | + with tarfile.open(bundle_tar_path, "w:gz", encoding="utf-8") as tar: |
382 | + for f in bundle_dir_path.iterdir(): |
383 | + tar.add(str(f), f.name) |
384 | + |
385 | + return bundle_tar_path |
386 | + |
387 | + |
388 | +def export_charm_bundle( |
389 | + bundle_name, |
390 | + export_dir_path, |
391 | + channel, |
392 | + charm_series=DEFAULT_SERIES, |
393 | + charm_architecture=DEFAULT_ARCHITECTURE, |
394 | +): |
395 | + """Orchestrating function for exporting a charm bundle and its component charms. |
396 | + |
397 | + Output structure: |
398 | + |
399 | + <export_dir_path>/ |
400 | + ├─ <bundle-name>-<timestamp>.tar.gz/ |
401 | + │ ├─ bundle.yaml |
402 | + │ ├─ <bundle-name>_<bundle-revision>.bundle |
403 | + │ ├─ channel-map.json |
404 | + │ ├─ charms/ |
405 | + │ │ ├─ <charm-name>.tar.gz/ |
406 | + │ │ │ ├─ <charm-name>_<charm-revision>.charm |
407 | + │ │ │ ├─ channel-map.json |
408 | + │ │ │ ├─ resources/ |
409 | + │ │ │ │ ├─ <charm-name>.<resource-name>_<resource-revision> |
410 | + |
411 | + :param str bundle_name: Store package name for the bundle to be exported |
412 | + :param pathlib.Path export_dir_path: export directory path |
413 | + :param str channel: channel of the bundle itself to fetch, default "latest/stable" |
414 | + :param str charm_series: series of the charms to fetch, default "jammy" |
415 | + :param str charm_architecture: architecture of the charms to fetch, default "amd64" |
416 | + """ |
417 | if not export_dir_path: |
418 | snap_user_common_dir = os.getenv("SNAP_USER_COMMON") |
419 | if snap_user_common_dir: |
420 | @@ -307,36 +445,85 @@ def export_charms( |
421 | timestamp = ( |
422 | datetime.datetime.utcnow().replace(microsecond=0).strftime("%Y%m%dT%H%M%S") |
423 | ) |
424 | + # Functions calling /info and /refresh are shared with charms, but unlike charms |
425 | + # we would only export 1 bundle at a time. |
426 | + bundle_to_import = CharmImport( |
427 | + package_name=bundle_name, channel=channel, package_type="bundle" |
428 | + ) |
429 | + bundle_info, bundle_revision = _fetch_store_info_for_packages([bundle_to_import]) |
430 | + # Unpack bundle_revision into the format expected by `download_charm_files` |
431 | + bundle_revision = bundle_revision[bundle_name] |
432 | + |
433 | + with tempfile.TemporaryDirectory() as bundle_dir_path: |
434 | + bundle_dir_path = pathlib.Path(bundle_dir_path) |
435 | + |
436 | + # TODO bowenfan To format outputs per import requirements, |
437 | + # as opposed to just dumping the /info response |
438 | + write_channel_map(bundle_info[bundle_name], bundle_dir_path) |
439 | + download_charm_files(bundle_revision, bundle_dir_path, progress_copy_to_file) |
440 | + |
441 | + # Automatically export all component charms in the bundle |
442 | + # If a user needs to modify the bundle before exporting it, they can use |
443 | + # the get-bundle-yaml command to output bundle.yaml only, |
444 | + # without auto fetching charms |
445 | + with zipfile.ZipFile( |
446 | + bundle_dir_path |
447 | + / f"{bundle_revision.package_name}_{bundle_revision.revision}.bundle" |
448 | + ) as bundle_zip: |
449 | + bundle_zip.extract(member="bundle.yaml", path=bundle_dir_path) |
450 | + |
451 | + charms_dir_path = bundle_dir_path / "charms" |
452 | + charms_dir_path.mkdir(parents=True, exist_ok=False) |
453 | + charms_tar_path = export_charms( |
454 | + bundle_dir_path / "bundle.yaml", |
455 | + charms_dir_path, |
456 | + charm_series, |
457 | + charm_architecture, |
458 | + ) |
459 | |
460 | - charms_to_import = parse_charm_bundle_yaml(bundle_yaml_filepath, architecture) |
461 | - charms_info = { |
462 | - charm.package_name: get_charm_info(devicegw_root, charm) |
463 | - for charm in charms_to_import |
464 | - } |
465 | - charms_download_actions = make_charm_download_actions(charms_to_import, charms_info) |
466 | - charms_download_info = { |
467 | - res["name"]: res |
468 | - for res in get_charms_download_info(devicegw_root, charms_download_actions) |
469 | - } |
470 | - charms_revisions = charm_revisions_from_info(charms_info, charms_download_info) |
471 | - |
472 | - tarball_paths = [] |
473 | - for charm_name, charm_revision in charms_revisions.items(): |
474 | - with tempfile.TemporaryDirectory() as charm_dir_path: |
475 | - charm_dir_path = pathlib.Path(charm_dir_path) |
476 | - |
477 | - write_channel_map(charms_info[charm_name], charm_dir_path) |
478 | - download_charm_files( |
479 | - charm_revision, |
480 | - charm_dir_path, |
481 | - progress_copy_to_file, |
482 | - ) |
483 | + # Extract charms |
484 | + with tarfile.open(charms_tar_path, "r:gz") as charms_tar: |
485 | + charms_tar.extractall(charms_dir_path) |
486 | + # Remove extra bundle.yaml from the export_charms function |
487 | + os.remove(charms_dir_path / "bundle.yaml") |
488 | + # Remove the extracted exported charms tarball |
489 | + os.remove(charms_tar_path) |
490 | + |
491 | + # Pack what we have into a root tar named after the bundle |
492 | + # (see dir structure in docstring) |
493 | + tarball_path = export_dir_path / f"{bundle_name}-{timestamp}.tar.gz" |
494 | + with tarfile.open(tarball_path, "w:gz", encoding="utf-8") as tar: |
495 | + for f in bundle_dir_path.iterdir(): |
496 | + tar.add(str(object=f), f.name) |
497 | + |
498 | + return (bundle_name, tarball_path) |
499 | |
500 | - tarball_path = export_dir_path / f"{charm_name}-{timestamp}.tar.gz" |
501 | - with tarfile.open(tarball_path, "w:gz", encoding="utf-8") as tar: |
502 | - for f in charm_dir_path.iterdir(): |
503 | - tar.add(str(f), f.name) |
504 | |
505 | - tarball_paths.append((charm_name, tarball_path)) |
506 | +def get_bundle_yaml(bundle_name, channel): |
507 | + """Get the bundle.yaml for a given bundle name and channel. |
508 | |
509 | - return tarball_paths |
510 | + :param str bundle_name: Store package name for the bundle to be exported |
511 | + :param str channel: channel of the bundle itself to fetch, default "latest/stable" |
512 | + """ |
513 | + bundle_to_import = CharmImport( |
514 | + package_name=bundle_name, channel=channel, package_type="bundle" |
515 | + ) |
516 | + _, bundle_revision = _fetch_store_info_for_packages([bundle_to_import]) |
517 | + bundle_revision = bundle_revision[bundle_name] |
518 | + |
519 | + with tempfile.TemporaryDirectory() as bundle_dir_path: |
520 | + bundle_dir_path = pathlib.Path(bundle_dir_path) |
521 | + |
522 | + download_charm_files(bundle_revision, bundle_dir_path, progress_copy_to_file) |
523 | + |
524 | + with tempfile.TemporaryDirectory(dir=bundle_dir_path) as zip_dir_path: |
525 | + zip_dir_path = pathlib.Path(zip_dir_path) |
526 | + with zipfile.ZipFile( |
527 | + bundle_dir_path |
528 | + / f"{bundle_revision.package_name}_{bundle_revision.revision}.bundle" |
529 | + ) as bundle_zip: |
530 | + bundle_zip.extractall(path=zip_dir_path) |
531 | + with open(zip_dir_path / "bundle.yaml") as bundle_yaml_file: |
532 | + logger.info("\n-----BEGIN BUNDLE.YAML-----") |
533 | + logger.info(bundle_yaml_file.read()) |
534 | + logger.info("-----END BUNDLE.YAML-----") |
535 | diff --git a/store_admin/logic/tests/test_charms.py b/store_admin/logic/tests/test_charms.py |
536 | index f94a0b1..1d1c349 100644 |
537 | --- a/store_admin/logic/tests/test_charms.py |
538 | +++ b/store_admin/logic/tests/test_charms.py |
539 | @@ -1,13 +1,15 @@ |
540 | import binascii |
541 | import datetime |
542 | +import os |
543 | import pathlib |
544 | import tarfile |
545 | from textwrap import dedent |
546 | from unittest import mock |
547 | +import zipfile |
548 | |
549 | import pytest |
550 | import responses |
551 | - |
552 | +from io import BytesIO |
553 | from store_admin import exceptions |
554 | from store_admin.logic import charms |
555 | from store_admin.types import ( |
556 | @@ -21,6 +23,7 @@ from store_admin.types import ( |
557 | |
558 | PATCH_PREFIX = "store_admin.logic.charms." |
559 | TEST_CHARM_NAME = "charm1" |
560 | +TEST_BUNDLE_NAME = "bundle1" |
561 | |
562 | |
563 | @pytest.fixture |
564 | @@ -104,10 +107,142 @@ def setup_mock_charm_info(): |
565 | |
566 | |
567 | @pytest.fixture |
568 | +def setup_mock_bundle_info(): |
569 | + def setup(devicegw_root="", requests_mock=None, bundle_name=TEST_BUNDLE_NAME): |
570 | + info_data = { |
571 | + "type": "bundle", |
572 | + "id": f"{bundle_name}Id", |
573 | + "name": bundle_name, |
574 | + "channel-map": [ |
575 | + { |
576 | + "channel": { |
577 | + "base": None, |
578 | + "name": "stable", |
579 | + "released-at": "2024-01-03T14:53:38.952872+00:00", |
580 | + "risk": "stable", |
581 | + "track": "latest", |
582 | + }, |
583 | + "revision": { |
584 | + "attributes": {}, |
585 | + "bases": [None], |
586 | + "created-at": "2023-12-07T23:36:15.652705+00:00", |
587 | + "download": { |
588 | + "hash-sha-256": binascii.hexlify(b"5555").decode(), |
589 | + "size": 5555, |
590 | + "url": f"https://download/{bundle_name}_5.bundle", |
591 | + }, |
592 | + "revision": 5, |
593 | + "version": "5", |
594 | + }, |
595 | + }, |
596 | + ], |
597 | + "result": { |
598 | + "categories": [], |
599 | + "deployable-on": ["kubernetes"], |
600 | + "description": "A charming charm bundle", |
601 | + "license": "", |
602 | + "links": None, |
603 | + "media": [], |
604 | + "publisher": {"display-name": "Publisher of charming charms"}, |
605 | + "store-url": f"https://charmhub.io/{bundle_name}", |
606 | + "summary": "A charming charm bundle", |
607 | + "title": "A charming charm bundle", |
608 | + "unlisted": False, |
609 | + }, |
610 | + } |
611 | + if requests_mock: |
612 | + requests_mock.add( |
613 | + "GET", |
614 | + devicegw_root + f"/v2/charms/info/{bundle_name}", |
615 | + match=[ |
616 | + responses.matchers.query_param_matcher( |
617 | + {"fields": ",".join(charms.CHARM_INFO_FIELDS)} |
618 | + ), |
619 | + ], |
620 | + json=info_data, |
621 | + ) |
622 | + return bundle_name, info_data |
623 | + |
624 | + return setup |
625 | + |
626 | + |
627 | +def _make_charm_refresh_results(charm_names): |
628 | + return [ |
629 | + { |
630 | + "charm": { |
631 | + "created-at": "2023-12-07T23:36:15.652705+00:00", |
632 | + "download": { |
633 | + "hash-sha-256": binascii.hexlify(b"5555").decode(), |
634 | + "size": 5555, |
635 | + "url": f"https://download/{charm_name}_5.charm", |
636 | + }, |
637 | + "id": f"{charm_name}Id", |
638 | + "name": charm_name, |
639 | + "publisher": { |
640 | + "display-name": "Publisher of charming charms", |
641 | + "id": "charmpublisherId", |
642 | + "username": "charm-publisher", |
643 | + "validation": "unproven", |
644 | + }, |
645 | + "resources": [ |
646 | + { |
647 | + "created-at": "2023-11-28T23:19:59.819552", |
648 | + "description": "Charm OCI Resource", |
649 | + "download": { |
650 | + "hash-sha-256": binascii.hexlify(b"2222").decode(), |
651 | + "hash-sha-384": binascii.hexlify(b"222").decode(), |
652 | + "hash-sha-512": binascii.hexlify(b"22").decode(), |
653 | + "hash-sha3-384": binascii.hexlify(b"2").decode(), |
654 | + "size": 222, |
655 | + "url": f"https://download/{charm_name}-image_2", |
656 | + }, |
657 | + "filename": "", |
658 | + "name": f"{charm_name}-image", |
659 | + "revision": 2, |
660 | + "type": "oci-image", |
661 | + } |
662 | + ], |
663 | + "revision": 5, |
664 | + "summary": "A charming charm", |
665 | + "type": "charm", |
666 | + "version": "5", |
667 | + }, |
668 | + "effective-channel": "stable", |
669 | + "id": f"{charm_name}Id", |
670 | + "instance-key": "random-instance-key", |
671 | + "name": charm_name, |
672 | + "released-at": "2024-01-03T14:53:38.952872+00:00", |
673 | + "result": "download", |
674 | + } |
675 | + for charm_name in charm_names |
676 | + ] |
677 | + |
678 | + |
679 | +@pytest.fixture |
680 | def setup_mock_charm_refresh(): |
681 | def setup(devicegw_root="", requests_mock=None, charm_names=None): |
682 | charm_names = charm_names or [TEST_CHARM_NAME] |
683 | + refresh_data = { |
684 | + "error-list": [], |
685 | + "results": _make_charm_refresh_results(charm_names), |
686 | + } |
687 | + if requests_mock: |
688 | + requests_mock.add( |
689 | + "POST", devicegw_root + "/v2/charms/refresh", json=refresh_data |
690 | + ) |
691 | + return charm_names, refresh_data |
692 | + |
693 | + return setup |
694 | + |
695 | |
696 | +@pytest.fixture |
697 | +def setup_mock_bundle_refresh(): |
698 | + def setup( |
699 | + devicegw_root="", |
700 | + requests_mock=None, |
701 | + bundle_name=TEST_BUNDLE_NAME, |
702 | + charm_names=None, |
703 | + ): |
704 | refresh_data = { |
705 | "error-list": [], |
706 | "results": [ |
707 | @@ -117,54 +252,38 @@ def setup_mock_charm_refresh(): |
708 | "download": { |
709 | "hash-sha-256": binascii.hexlify(b"5555").decode(), |
710 | "size": 5555, |
711 | - "url": f"https://download/{charm_name}_5.charm", |
712 | + "url": f"https://download/{bundle_name}_5.bundle", |
713 | }, |
714 | - "id": f"{charm_name}Id", |
715 | - "name": charm_name, |
716 | + "id": f"{bundle_name}Id", |
717 | + "name": bundle_name, |
718 | "publisher": { |
719 | "display-name": "Publisher of charming charms", |
720 | "id": "charmpublisherId", |
721 | "username": "charm-publisher", |
722 | "validation": "unproven", |
723 | }, |
724 | - "resources": [ |
725 | - { |
726 | - "created-at": "2023-11-28T23:19:59.819552", |
727 | - "description": "Charm OCI Resource", |
728 | - "download": { |
729 | - "hash-sha-256": binascii.hexlify(b"2222").decode(), |
730 | - "hash-sha-384": binascii.hexlify(b"222").decode(), |
731 | - "hash-sha-512": binascii.hexlify(b"22").decode(), |
732 | - "hash-sha3-384": binascii.hexlify(b"2").decode(), |
733 | - "size": 222, |
734 | - "url": f"https://download/{charm_name}-image_2", |
735 | - }, |
736 | - "filename": "", |
737 | - "name": f"{charm_name}-image", |
738 | - "revision": 2, |
739 | - "type": "oci-image", |
740 | - } |
741 | - ], |
742 | + "resources": [], |
743 | "revision": 5, |
744 | - "summary": "A charming charm", |
745 | - "type": "charm", |
746 | + "summary": "A charming charm bundle", |
747 | + "type": "bundle", |
748 | "version": "5", |
749 | }, |
750 | "effective-channel": "stable", |
751 | - "id": f"{charm_name}Id", |
752 | + "id": f"{bundle_name}Id", |
753 | "instance-key": "random-instance-key", |
754 | - "name": charm_name, |
755 | + "name": bundle_name, |
756 | "released-at": "2024-01-03T14:53:38.952872+00:00", |
757 | "result": "download", |
758 | } |
759 | - for charm_name in charm_names |
760 | ], |
761 | } |
762 | + if charm_names: |
763 | + refresh_data["results"].extend(_make_charm_refresh_results(charm_names)) |
764 | if requests_mock: |
765 | requests_mock.add( |
766 | "POST", devicegw_root + "/v2/charms/refresh", json=refresh_data |
767 | ) |
768 | - return charm_names, refresh_data |
769 | + return [bundle_name], refresh_data |
770 | |
771 | return setup |
772 | |
773 | @@ -199,6 +318,7 @@ def mock_charm_revision_dataclass(): |
774 | ), |
775 | package_name=charm_name, |
776 | package_id=f"{charm_name}Id", |
777 | + package_type="charm", |
778 | revision=5, |
779 | created_at="2023-12-07T23:36:15.652705+00:00", |
780 | architectures=[{"architecture": "amd64", "channel": "22.04", "name": "ubuntu"}], |
781 | @@ -218,25 +338,72 @@ def mock_charm_revision_dataclass(): |
782 | ) |
783 | |
784 | |
785 | -@pytest.fixture |
786 | -def mock_bundle_yaml(): |
787 | - yaml_content = dedent( |
788 | - """\ |
789 | - bundle: kubernetes |
790 | - applications: |
791 | - charm1: |
792 | - charm: charm1 |
793 | - channel: stable |
794 | - series: jammy |
795 | - charm2: |
796 | - charm: charm2 |
797 | - """ |
798 | +def _yaml_content(charm_names): |
799 | + return dedent( |
800 | + f"""\ |
801 | + bundle: kubernetes |
802 | + applications: |
803 | + app1: |
804 | + charm: {charm_names[0]} |
805 | + channel: stable |
806 | + series: jammy |
807 | + app2: |
808 | + charm: {charm_names[1]} |
809 | + """ |
810 | ) |
811 | + |
812 | + |
813 | +@pytest.fixture |
814 | +def setup_mock_bundle_yaml(): |
815 | + yaml_content = _yaml_content(["charm1", "charm2"]) |
816 | with mock.patch("builtins.open", mock.mock_open(read_data=yaml_content)): |
817 | yield yaml_content |
818 | |
819 | |
820 | @pytest.fixture |
821 | +def setup_charm_and_resource_download_mocks(): |
822 | + def setup(charm_name, requests_mock): |
823 | + requests_mock.add( |
824 | + "GET", |
825 | + f"https://download/{charm_name}_5.charm", |
826 | + body="charm-blob".encode(), |
827 | + auto_calculate_content_length=True, |
828 | + content_type="application/octet-stream", |
829 | + ) |
830 | + requests_mock.add( |
831 | + "GET", |
832 | + f"https://download/{charm_name}-image_2", |
833 | + body="resource-blob".encode(), |
834 | + auto_calculate_content_length=True, |
835 | + content_type="application/octet-stream", |
836 | + ) |
837 | + |
838 | + return setup |
839 | + |
840 | + |
841 | +@pytest.fixture |
842 | +def setup_bundle_download_mock(): |
843 | + def setup(requests_mock, bundle_name=TEST_BUNDLE_NAME): |
844 | + yaml_content = _yaml_content(["charm1", "charm2"]) |
845 | + # Use an in-memory zip file as the .bundle blob |
846 | + bundle_buf = BytesIO(yaml_content.encode()) |
847 | + zip_buf = BytesIO() |
848 | + with zipfile.ZipFile(zip_buf, mode="a") as bundle_file: |
849 | + bundle_file.writestr("bundle.yaml", bundle_buf.getvalue()) |
850 | + |
851 | + requests_mock.add( |
852 | + "GET", |
853 | + f"https://download/{bundle_name}_5.bundle", |
854 | + body=zip_buf.getvalue(), |
855 | + auto_calculate_content_length=True, |
856 | + content_type="application/octet-stream", |
857 | + ) |
858 | + return yaml_content |
859 | + |
860 | + return setup |
861 | + |
862 | + |
863 | +@pytest.fixture |
864 | def mock_charm_import(): |
865 | charm_import = CharmImport( |
866 | package_name=TEST_CHARM_NAME, |
867 | @@ -247,7 +414,7 @@ def mock_charm_import(): |
868 | return charm_import |
869 | |
870 | |
871 | -def test_parse_charm_bundle_yaml(mock_bundle_yaml): |
872 | +def test_parse_charm_bundle_yaml(setup_mock_bundle_yaml): |
873 | expected = [ |
874 | CharmImport( |
875 | package_name="charm1", |
876 | @@ -262,7 +429,7 @@ def test_parse_charm_bundle_yaml(mock_bundle_yaml): |
877 | architecture="amd64", |
878 | ), |
879 | ] |
880 | - parsed = charms.parse_charm_bundle_yaml("test_bundle.yaml", "amd64") |
881 | + parsed = charms.parse_charm_bundle_yaml("test_bundle.yaml", "jammy", "amd64") |
882 | assert parsed == expected |
883 | |
884 | |
885 | @@ -434,6 +601,24 @@ def test_download_charm_files_success( |
886 | assert _file_read_cb.call_count == 2 |
887 | |
888 | |
889 | +def _assert_charm_tar_contents(charm_tar_paths, tmpdir): |
890 | + for path in charm_tar_paths: |
891 | + charm_name = os.path.basename(path).replace(".tar.gz", "") |
892 | + with tarfile.open(path, "r:gz") as tar: |
893 | + assert set(ti.name for ti in tar.getmembers()) == { |
894 | + "channel-map.json", |
895 | + f"{charm_name}_5.charm", |
896 | + "resources", |
897 | + f"resources/{charm_name}.{charm_name}-image_2", |
898 | + } |
899 | + path = pathlib.Path(tmpdir.mkdir(f"{charm_name}-extract")) |
900 | + tar.extractall(path) |
901 | + assert (path / f"{charm_name}_5.charm").read_text() == "charm-blob" |
902 | + assert ( |
903 | + path / f"resources/{charm_name}.{charm_name}-image_2" |
904 | + ).read_text() == "resource-blob" |
905 | + |
906 | + |
907 | def test_export_charms_success( |
908 | tmpdir, |
909 | devicegw_root, |
910 | @@ -441,33 +626,25 @@ def test_export_charms_success( |
911 | utcnow, |
912 | setup_mock_charm_info, |
913 | setup_mock_charm_refresh, |
914 | + setup_charm_and_resource_download_mocks, |
915 | ): |
916 | charm_names = ["charm1", "charm2"] |
917 | for charm_name in charm_names: |
918 | - requests_mock.add( |
919 | - "GET", |
920 | - f"https://download/{charm_name}_5.charm", |
921 | - body="charm-blob".encode(), |
922 | - auto_calculate_content_length=True, |
923 | - content_type="application/octet-stream", |
924 | - ) |
925 | - requests_mock.add( |
926 | - "GET", |
927 | - f"https://download/{charm_name}-image_2", |
928 | - body="resource-blob".encode(), |
929 | - auto_calculate_content_length=True, |
930 | - content_type="application/octet-stream", |
931 | - ) |
932 | - _ = setup_mock_charm_info(devicegw_root, requests_mock, charm_name) |
933 | - _ = setup_mock_charm_refresh(devicegw_root, requests_mock, charm_names) |
934 | + setup_charm_and_resource_download_mocks(charm_name, requests_mock) |
935 | + setup_mock_charm_info(devicegw_root, requests_mock, charm_name) |
936 | + setup_mock_charm_refresh(devicegw_root, requests_mock, charm_names) |
937 | |
938 | timestamp = utcnow.replace(microsecond=0).strftime("%Y%m%dT%H%M%S") |
939 | |
940 | - def _side_effect(raw_response, target_path, length): |
941 | + def _progress_copy_to_file_side_effect(raw_response, target_path, length): |
942 | data = raw_response.read() |
943 | assert len(data) == length |
944 | target_path.write_bytes(data) |
945 | |
946 | + def _copyfile_side_effect(_, destination): |
947 | + with open(destination, "w") as f: |
948 | + f.write("mocked_bundle.yaml-content-does-not-matter") |
949 | + |
950 | _m_parsed_charm_import = [ |
951 | CharmImport( |
952 | package_name="charm1", |
953 | @@ -484,30 +661,122 @@ def test_export_charms_success( |
954 | ] |
955 | |
956 | with mock.patch( |
957 | - PATCH_PREFIX + "progress_copy_to_file", side_effect=_side_effect |
958 | + PATCH_PREFIX + "progress_copy_to_file", |
959 | + side_effect=_progress_copy_to_file_side_effect, |
960 | ), mock.patch( |
961 | PATCH_PREFIX + "parse_charm_bundle_yaml", return_value=_m_parsed_charm_import |
962 | + ), mock.patch( |
963 | + PATCH_PREFIX + "shutil.copyfile", _copyfile_side_effect |
964 | ): |
965 | export_dir_path = pathlib.Path(tmpdir.mkdir("export-dir")) |
966 | - tarball_paths = charms.export_charms("test_bundle.yaml", export_dir_path) |
967 | + exported_tar_path = charms.export_charms("test_bundle.yaml", export_dir_path) |
968 | |
969 | - assert tarball_paths == [ |
970 | - ("charm1", export_dir_path / f"charm1-{timestamp}.tar.gz"), |
971 | - ("charm2", export_dir_path / f"charm2-{timestamp}.tar.gz"), |
972 | - ] |
973 | - assert set(export_dir_path.iterdir()) == {t[1] for t in tarball_paths} |
974 | + assert ( |
975 | + exported_tar_path |
976 | + == export_dir_path |
977 | + / f"{charms.DEFAULT_CHARMS_EXPORT_FILENAME_PREFIX}-{timestamp}.tar.gz" |
978 | + ) |
979 | |
980 | - for charm_name, tarball_path in tarball_paths: |
981 | - with tarfile.open(tarball_path, "r:gz") as tar: |
982 | - assert set(ti.name for ti in tar.getmembers()) == { |
983 | - "channel-map.json", |
984 | - f"{charm_name}_5.charm", |
985 | - "resources", |
986 | - f"resources/{charm_name}.{charm_name}-image_2", |
987 | - } |
988 | - path = pathlib.Path(tmpdir.mkdir(f"{charm_name}-extract")) |
989 | - tar.extractall(path) |
990 | - assert (path / f"{charm_name}_5.charm").read_text() == "charm-blob" |
991 | - assert ( |
992 | - path / f"resources/{charm_name}.{charm_name}-image_2" |
993 | - ).read_text() == "resource-blob" |
994 | + charms_dir_path = export_dir_path / "charms" |
995 | + with tarfile.open(exported_tar_path, "r:gz") as parent_tar: |
996 | + parent_tar.extractall(charms_dir_path) |
997 | + |
998 | + root_dir_contents = { |
999 | + pathlib.Path(os.path.join(charms_dir_path, p)) |
1000 | + for p in os.listdir(charms_dir_path) |
1001 | + } |
1002 | + assert root_dir_contents == { |
1003 | + charms_dir_path / "charm1.tar.gz", |
1004 | + charms_dir_path / "charm2.tar.gz", |
1005 | + charms_dir_path / "bundle.yaml", |
1006 | + } |
1007 | + |
1008 | + _assert_charm_tar_contents( |
1009 | + [charms_dir_path / "charm1.tar.gz", charms_dir_path / "charm2.tar.gz"], tmpdir |
1010 | + ) |
1011 | + |
1012 | + |
1013 | +def test_export_bundle_success( |
1014 | + tmpdir, |
1015 | + devicegw_root, |
1016 | + requests_mock, |
1017 | + utcnow, |
1018 | + setup_mock_bundle_info, |
1019 | + setup_mock_bundle_refresh, |
1020 | + setup_mock_charm_info, |
1021 | + setup_bundle_download_mock, |
1022 | + setup_charm_and_resource_download_mocks, |
1023 | +): |
1024 | + bundle_yaml_contents = setup_bundle_download_mock(requests_mock) |
1025 | + charm_names = ["charm1", "charm2"] |
1026 | + for charm_name in charm_names: |
1027 | + setup_charm_and_resource_download_mocks(charm_name, requests_mock) |
1028 | + setup_mock_charm_info(devicegw_root, requests_mock, charm_name) |
1029 | + |
1030 | + setup_mock_bundle_info(devicegw_root, requests_mock, TEST_BUNDLE_NAME) |
1031 | + setup_mock_bundle_refresh( |
1032 | + devicegw_root, requests_mock, TEST_BUNDLE_NAME, charm_names |
1033 | + ) |
1034 | + |
1035 | + timestamp = utcnow.replace(microsecond=0).strftime("%Y%m%dT%H%M%S") |
1036 | + |
1037 | + def _side_effect(raw_response, target_path, length): |
1038 | + data = raw_response.read() |
1039 | + assert len(data) == length |
1040 | + target_path.write_bytes(data) |
1041 | + |
1042 | + with mock.patch(PATCH_PREFIX + "progress_copy_to_file", side_effect=_side_effect): |
1043 | + export_dir_path = pathlib.Path(tmpdir.mkdir("export-dir")) |
1044 | + bundle_name, tarball_path = charms.export_charm_bundle( |
1045 | + TEST_BUNDLE_NAME, export_dir_path, "stable" |
1046 | + ) |
1047 | + |
1048 | + assert bundle_name == TEST_BUNDLE_NAME |
1049 | + assert tarball_path == export_dir_path / f"{TEST_BUNDLE_NAME}-{timestamp}.tar.gz" |
1050 | + |
1051 | + with tarfile.open(tarball_path, "r:gz") as tar: |
1052 | + assert set(ti.name for ti in tar.getmembers()) == { |
1053 | + "bundle.yaml", |
1054 | + "channel-map.json", |
1055 | + "bundle1_5.bundle", |
1056 | + "charms", |
1057 | + "charms/charm1.tar.gz", |
1058 | + "charms/charm2.tar.gz", |
1059 | + } |
1060 | + bundle_path = pathlib.Path(tmpdir.mkdir(f"{TEST_BUNDLE_NAME}-extract")) |
1061 | + tar.extractall(bundle_path) |
1062 | + assert (bundle_path / "bundle.yaml").read_text() == bundle_yaml_contents |
1063 | + |
1064 | + _assert_charm_tar_contents( |
1065 | + [ |
1066 | + bundle_path / "charms" / "charm1.tar.gz", |
1067 | + bundle_path / "charms" / "charm2.tar.gz", |
1068 | + ], |
1069 | + tmpdir, |
1070 | + ) |
1071 | + |
1072 | + |
1073 | +def test_get_bundle_yaml_success( |
1074 | + devicegw_root, |
1075 | + requests_mock, |
1076 | + caplog, |
1077 | + setup_mock_bundle_info, |
1078 | + setup_mock_bundle_refresh, |
1079 | + setup_bundle_download_mock, |
1080 | +): |
1081 | + bundle_yaml_contents = setup_bundle_download_mock(requests_mock) |
1082 | + charm_names = ["charm1", "charm2"] |
1083 | + _ = setup_mock_bundle_info(devicegw_root, requests_mock, TEST_BUNDLE_NAME) |
1084 | + _ = setup_mock_bundle_refresh( |
1085 | + devicegw_root, requests_mock, TEST_BUNDLE_NAME, charm_names |
1086 | + ) |
1087 | + |
1088 | + def _side_effect(raw_response, target_path, length): |
1089 | + data = raw_response.read() |
1090 | + assert len(data) == length |
1091 | + target_path.write_bytes(data) |
1092 | + |
1093 | + with mock.patch(PATCH_PREFIX + "progress_copy_to_file", side_effect=_side_effect): |
1094 | + charms.get_bundle_yaml(TEST_BUNDLE_NAME, "latest/stable") |
1095 | + |
1096 | + assert bundle_yaml_contents in caplog.messages |
1097 | diff --git a/store_admin/types.py b/store_admin/types.py |
1098 | index b3fe34f..e8cc1bd 100644 |
1099 | --- a/store_admin/types.py |
1100 | +++ b/store_admin/types.py |
1101 | @@ -44,9 +44,10 @@ DEFAULT_OS = "ubuntu" |
1102 | @dataclass |
1103 | class CharmImport: |
1104 | package_name: str |
1105 | - series: str |
1106 | - architecture: str |
1107 | + series: Optional[str] = None |
1108 | + architecture: Optional[str] = None |
1109 | channel: Optional[str] = None |
1110 | + package_type: str = "charm" |
1111 | |
1112 | def __post_init__(self): |
1113 | if self.channel: |
1114 | @@ -58,22 +59,16 @@ class CharmImport: |
1115 | def as_dict(self): |
1116 | return {k: v for k, v in asdict(obj=self).items() if v is not None} |
1117 | |
1118 | - def get_charm_base_dict(self): |
1119 | - return { |
1120 | - "name": DEFAULT_OS, |
1121 | - "channel": self.series, |
1122 | - "architecture": self.architecture, |
1123 | - } |
1124 | - |
1125 | |
1126 | @dataclass |
1127 | class CharmDownloadAction: |
1128 | package_name: str |
1129 | package_id: str |
1130 | - series: str |
1131 | - architecture: str |
1132 | + series: Optional[str] = None |
1133 | + architecture: Optional[str] = None |
1134 | channel: Optional[str] = None |
1135 | action: str = "download" |
1136 | + package_type: str = "charm" |
1137 | |
1138 | @property |
1139 | def instance_key(self): |
1140 | @@ -84,7 +79,7 @@ class CharmDownloadAction: |
1141 | """ |
1142 | return base64.b64encode(str(uuid4()).encode()).decode() |
1143 | |
1144 | - def get_charm_base_dict(self): |
1145 | + def _get_charm_base_dict(self): |
1146 | try: |
1147 | return { |
1148 | "name": DEFAULT_OS, |
1149 | @@ -97,7 +92,10 @@ class CharmDownloadAction: |
1150 | def to_request_dict(self): |
1151 | request_dict = { |
1152 | "action": self.action, |
1153 | - "base": self.get_charm_base_dict(), |
1154 | + # We still need to send {"base": null} for bundles to conform to the API |
1155 | + "base": ( |
1156 | + self._get_charm_base_dict() if self.package_type == "charm" else None |
1157 | + ), |
1158 | "instance-key": self.instance_key, |
1159 | "id": self.package_id, |
1160 | } |
1161 | @@ -157,6 +155,7 @@ class CharmRevision: |
1162 | channel: CharmChannel |
1163 | package_name: str |
1164 | package_id: str |
1165 | + package_type: str |
1166 | revision: int |
1167 | created_at: str |
1168 | architectures: List[dict] |
1169 | @@ -171,7 +170,7 @@ class CharmRevision: |
1170 | |
1171 | def blob_file_name(self): |
1172 | """Name of the downloaded revision blob file.""" |
1173 | - return f"{self.package_name}_{self.revision}.charm" |
1174 | + return f"{self.package_name}_{self.revision}.{self.package_type}" |
1175 | |
1176 | def resource_blob_file_name(self, resource): |
1177 | """Name of the downloaded resource revision blob file.""" |