Merge ~bowenfan/snapstore-client:SN2487b into snapstore-client:main

Proposed by Bowen Fan
Status: Merged
Merged at revision: 9dc36fbdcaa1baa35cfd15e3c8360589453d24c2
Proposed branch: ~bowenfan/snapstore-client:SN2487b
Merge into: snapstore-client:main
Diff against target: 1177 lines (+727/-144)
5 files modified
store_admin/cli/charms.py (+125/-0)
store_admin/cli/runner.py (+4/-1)
store_admin/logic/charms.py (+235/-48)
store_admin/logic/tests/test_charms.py (+350/-81)
store_admin/types.py (+13/-14)
Reviewer Review Type Date Requested Status
Ubuntu One hackers Pending
Review via email: mp+460621@code.launchpad.net
To post a comment you must log in.

Updating diff...

An updated diff will be available in a few minutes. Reload to see the changes.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/store_admin/cli/charms.py b/store_admin/cli/charms.py
2new file mode 100644
3index 0000000..f832896
4--- /dev/null
5+++ b/store_admin/cli/charms.py
6@@ -0,0 +1,125 @@
7+import logging
8+import pathlib
9+
10+import click
11+
12+from store_admin.logic import charms
13+
14+logger = logging.getLogger(__name__)
15+
16+
17+@click.command(name="charms")
18+@click.argument(
19+ "bundle_filepath",
20+ required=True,
21+ type=click.Path(
22+ exists=True,
23+ file_okay=True,
24+ dir_okay=False,
25+ path_type=pathlib.Path,
26+ ),
27+)
28+@click.option(
29+ "--export-dir",
30+ multiple=False,
31+ help="Directory to export charms to",
32+ type=click.Path(
33+ exists=True,
34+ file_okay=False,
35+ dir_okay=True,
36+ writable=True,
37+ path_type=pathlib.Path,
38+ ),
39+)
40+@click.option(
41+ "--series",
42+ multiple=False,
43+ default=charms.DEFAULT_SERIES,
44+ help="Series of the exported charms (jammy by default)",
45+)
46+@click.option(
47+ "--arch",
48+ "architecture",
49+ multiple=False,
50+ default=charms.DEFAULT_ARCHITECTURE,
51+ help="Architecture of the exported charms (amd64 by default)",
52+)
53+def export_charms(bundle_filepath, export_dir, series, architecture):
54+ """Download charms, resources, and metadata for importing to an offline store."""
55+ export_tar_path = charms.export_charms(
56+ bundle_yaml_filepath=bundle_filepath,
57+ export_dir_path=export_dir,
58+ series=series,
59+ architecture=architecture,
60+ )
61+ logger.info(f"Successfully exported charms to: {export_tar_path}")
62+
63+
64+@click.command(name="bundle")
65+@click.argument(
66+ "bundle_name",
67+ required=True,
68+)
69+@click.option(
70+ "--export-dir",
71+ multiple=False,
72+ help="Directory to export the bundle to",
73+ type=click.Path(
74+ exists=True,
75+ file_okay=False,
76+ dir_okay=True,
77+ writable=True,
78+ path_type=pathlib.Path,
79+ ),
80+)
81+@click.option(
82+ "--channel",
83+ multiple=False,
84+ default="latest/stable",
85+ help="Channel of the charm bundle to export (latest/stable by default)",
86+)
87+@click.option(
88+ "--series",
89+ multiple=False,
90+ default=charms.DEFAULT_SERIES,
91+ help="Series for the bundle's charms (jammy by default)",
92+)
93+@click.option(
94+ "--arch",
95+ "architecture",
96+ multiple=False,
97+ default=charms.DEFAULT_ARCHITECTURE,
98+ help="Architecture for the bundle's charms (amd64 by default)",
99+)
100+def export_bundle(bundle_name, export_dir, channel, series, architecture):
101+ """Download a charm bundle and its components for offline store import.
102+
103+ This command is for charm-bundle packages in the Store. It will automatically fetch
104+ all component charms and their component resources.
105+ """
106+ export_bundle_name, export_tar_path = charms.export_charm_bundle(
107+ bundle_name=bundle_name,
108+ export_dir_path=export_dir,
109+ channel=channel,
110+ charm_series=series,
111+ charm_architecture=architecture,
112+ )
113+ logger.info(
114+ f"Successfully exported charm bundle {export_bundle_name}: {export_tar_path}"
115+ )
116+
117+
118+@click.command(name="get-bundle-yaml") # TODO decide on a better command name
119+@click.argument(
120+ "bundle_name",
121+ required=True,
122+)
123+@click.option(
124+ "--channel",
125+ multiple=False,
126+ default="latest/stable",
127+ help="Channel of the charm bundle to export (latest/stable by default)",
128+)
129+def get_bundle_yaml(bundle_name, channel):
130+ """Fetch a charm bundle and print its bundle.yaml to stdout."""
131+ charms.get_bundle_yaml(bundle_name, channel)
132diff --git a/store_admin/cli/runner.py b/store_admin/cli/runner.py
133index 14d88a4..e4090e8 100644
134--- a/store_admin/cli/runner.py
135+++ b/store_admin/cli/runner.py
136@@ -3,7 +3,7 @@ import logging
137 import click
138
139 from store_admin import defaults, log, version
140-from store_admin.cli import common, model_service, overrides, snaps, stores
141+from store_admin.cli import common, model_service, overrides, snaps, stores, charms
142
143
144 @click.group(
145@@ -79,6 +79,7 @@ def export():
146 run.add_command(common.login)
147 run.add_command(common.register)
148 run.add_command(model_service.register_key)
149+run.add_command(charms.get_bundle_yaml)
150
151 # Commands under create
152 create.add_command(model_service.create_model)
153@@ -109,3 +110,5 @@ remove.add_command(model_service.remove_serial_policy)
154 export.add_command(stores.export_store)
155 export.add_command(common.export_token)
156 export.add_command(snaps.export_snap)
157+export.add_command(charms.export_charms)
158+export.add_command(charms.export_bundle)
159diff --git a/store_admin/logic/charms.py b/store_admin/logic/charms.py
160index 8137f5d..5f65edb 100644
161--- a/store_admin/logic/charms.py
162+++ b/store_admin/logic/charms.py
163@@ -5,6 +5,7 @@ import pathlib
164 import shutil
165 import tarfile
166 import tempfile
167+import zipfile
168 import yaml
169 from store_admin.types import (
170 CharmChannel,
171@@ -32,6 +33,7 @@ from store_admin.logic.snaps import (
172
173 DEFAULT_SERIES = "jammy"
174 DEFAULT_ARCHITECTURE = "amd64"
175+DEFAULT_CHARMS_EXPORT_FILENAME_PREFIX = "charms-export"
176
177 CHARM_INFO_FIELDS = {"type", "id", "name", "result", "channel-map"}
178
179@@ -41,7 +43,7 @@ REFRESH_REVISION_NOT_FOUND_CODE = "revision-not-found"
180 logger = logging.getLogger(__name__)
181
182
183-def parse_charm_bundle_yaml(bundle_yaml_filepath, architecture):
184+def parse_charm_bundle_yaml(bundle_yaml_filepath, series, architecture):
185 """Parse a bundle.yaml file into a list of CharmImport objects."""
186 # Onus is on the user to conform to the yaml spec, i.e. no duplicate keys
187 # pyyaml will clobber the previous entries where keys are duplicated
188@@ -49,16 +51,30 @@ def parse_charm_bundle_yaml(bundle_yaml_filepath, architecture):
189 bundle = yaml.safe_load(bundle_yaml)
190
191 applications = bundle["applications"]
192- preferred_series = bundle.get("series", DEFAULT_SERIES)
193- charms = [
194- CharmImport(
195- package_name=config["charm"],
196- channel=config.get("channel"),
197- series=config.get("series", preferred_series),
198- architecture=architecture,
199+
200+ # The series resolving logic here is rudimentary, but should be aligned with Juju.
201+ # It won't work in some cases, e.g. if the series and channel are not
202+ # user-specified, where we would fetch from the default track with a default
203+ # series of "jammy" (latest LTS). This can fail if the default track does not
204+ # support Jammy. We leave it to the user to provide feasible parameters.
205+ preferred_series = bundle.get("series", series)
206+ # We remap "kubernetes" to "focal", as we do in packagereview
207+ # See packagereview.types.CharmBase.from_series_str
208+ if preferred_series == "kubernetes":
209+ preferred_series = "focal"
210+ charms = []
211+ for config in applications.values():
212+ charm_series = config.get("series", preferred_series)
213+ if charm_series == "kubernetes":
214+ charm_series = "focal"
215+ charms.append(
216+ CharmImport(
217+ package_name=config["charm"],
218+ channel=config.get("channel"),
219+ series=charm_series,
220+ architecture=architecture,
221+ )
222 )
223- for config in applications.values()
224- ]
225 return charms
226
227
228@@ -140,12 +156,12 @@ def make_charm_download_actions(charms_to_import, charms_info):
229 def _get_charm_channel_from_name_and_revision(channel_map, channel_name, revision):
230 """Helper function to search channel map entries for a given channel entry."""
231 if "latest/" in channel_name:
232- channel_name = channel_name.split("/")[1]
233+ channel_name_without_track = channel_name.split("/")[1]
234 for channel_dict in channel_map:
235 if (
236 channel_dict["channel"]["name"] == channel_name
237- and channel_dict["revision"]["revision"] == revision
238- ):
239+ or channel_dict["channel"]["name"] == channel_name_without_track
240+ ) and channel_dict["revision"]["revision"] == revision:
241 return channel_dict
242 raise StoreRequestError("Inconsistent channel data")
243
244@@ -222,10 +238,12 @@ def charm_revisions_from_info(charms_info, charms_download_info):
245 binary_sha256=revision["download"]["hash-sha-256"],
246 )
247 package_id = charm_info["id"]
248+ package_type = charm_info["type"]
249 charm_revision = CharmRevision(
250 channel=channel,
251 package_name=charm_name,
252 package_id=package_id,
253+ package_type=package_type,
254 revision=revision["revision"],
255 created_at=revision["created-at"],
256 architectures=revision["bases"],
257@@ -248,10 +266,11 @@ def write_channel_map(charm_info_data, target_dir_path):
258
259
260 def download_charm_files(charm_revision, target_dir_path, file_read_cb):
261- """Download charm binaries and its resource binaries from the online Charmhub.
262+ """Download a bundle binary, or a charm and its resource binaries from Charmhub.
263
264 :param CharmRevision charm_revision: A CharmRevision object that contains download
265- URLs for the charm and the resource(s) that it was released with.
266+ URLs for the either the bundle, or the charm and the resource(s) that it was
267+ released with.
268 :param pathlib.Path target_dir_path: target directory path
269 :param callable file_read_cb: Optional callback used for reading in charm and
270 resource revision blobs.
271@@ -267,7 +286,11 @@ def download_charm_files(charm_revision, target_dir_path, file_read_cb):
272 shutil.copyfileobj(raw_response, blob_file)
273 logger.debug(f"Downloaded blob to {charm_revision_blob_path}")
274
275- if charm_revision.channel.resource_revisions:
276+ # Short circuit bundles here as their resources will always be None
277+ if (
278+ charm_revision.package_type == "charm"
279+ and charm_revision.channel.resource_revisions
280+ ):
281 resource_dir = target_dir_path / "resources"
282 resource_dir.mkdir(parents=True, exist_ok=True)
283 for resource in charm_revision.channel.resource_revisions:
284@@ -285,17 +308,132 @@ def download_charm_files(charm_revision, target_dir_path, file_read_cb):
285 logger.debug(f"Downloaded blob to {resource_revision_blob_path}")
286
287
288+def _fetch_store_info_for_packages(packages_to_import):
289+ """Helper function for hitting /info and /refresh and parsing the results.
290+
291+ :param list[CharmImport] packages_to_import: list of charms to fetch
292+ """
293+ devicegw_root = defaults.get_devicegw_url()
294+ packages_info = {
295+ pkg.package_name: get_charm_info(devicegw_root, pkg)
296+ for pkg in packages_to_import
297+ }
298+ packages_download_actions = make_charm_download_actions(
299+ packages_to_import, packages_info
300+ )
301+ packages_download_info = {
302+ res["name"]: res
303+ for res in get_charms_download_info(devicegw_root, packages_download_actions)
304+ }
305+ packages_revisions = charm_revisions_from_info(
306+ packages_info, packages_download_info
307+ )
308+
309+ return (packages_info, packages_revisions)
310+
311+
312 def export_charms(
313- bundle_yaml_filepath, export_dir_path, architecture=DEFAULT_ARCHITECTURE
314+ bundle_yaml_filepath,
315+ export_dir_path,
316+ series=DEFAULT_SERIES,
317+ architecture=DEFAULT_ARCHITECTURE,
318 ):
319 """Orchestrating function for exporting a set of charms given a bundle.yaml.
320
321+ Output structure:
322+
323+ <export_dir_path>/
324+ ├─ charms-export-<timestamp>.tar.gz/
325+ │ ├─ bundle.yaml
326+ | ├─ <charm-name>.tar.gz/
327+ │ │ ├─ <charm-name>_<charm-revision>.charm
328+ │ │ ├─ channel-map.json
329+ │ │ ├─ resources/
330+ │ │ │ ├─ <charm-name>.<resource-name>_<resource-revision>
331+
332+
333 :param pathlib.Path bundle_yaml_filepath: path to the bundle.yaml to be exported
334 :param pathlib.Path export_dir_path: export directory path
335- :param str architecture: architecture of the charms to fetch, defaults to amd64
336+ :param str series: series of the charms to fetch, default "jammy"
337+ :param str architecture: architecture of the charms to fetch, default "amd64"
338 """
339- devicegw_root = defaults.get_devicegw_url()
340+ if not export_dir_path:
341+ snap_user_common_dir = os.getenv("SNAP_USER_COMMON")
342+ if snap_user_common_dir:
343+ export_dir_path = pathlib.Path(snap_user_common_dir) / "export"
344+ export_dir_path.mkdir(exist_ok=True)
345+ else:
346+ export_dir_path = pathlib.Path(os.getcwd())
347
348+ timestamp = (
349+ datetime.datetime.utcnow().replace(microsecond=0).strftime("%Y%m%dT%H%M%S")
350+ )
351+ charms_to_import = parse_charm_bundle_yaml(
352+ bundle_yaml_filepath, series, architecture
353+ )
354+ charms_info, charms_revisions = _fetch_store_info_for_packages(charms_to_import)
355+
356+ with tempfile.TemporaryDirectory() as bundle_dir_path:
357+ bundle_dir_path = pathlib.Path(bundle_dir_path)
358+
359+ for charm_name, charm_revision in charms_revisions.items():
360+ with tempfile.TemporaryDirectory() as charm_dir_path:
361+ charm_dir_path = pathlib.Path(charm_dir_path)
362+
363+ write_channel_map(charms_info[charm_name], charm_dir_path)
364+ download_charm_files(
365+ charm_revision,
366+ charm_dir_path,
367+ progress_copy_to_file,
368+ )
369+ # Pack each charm, its metadata, and resources into a tarball
370+ charm_tar_path = bundle_dir_path / f"{charm_name}.tar.gz"
371+ with tarfile.open(charm_tar_path, "w:gz", encoding="utf-8") as tar:
372+ for f in charm_dir_path.iterdir():
373+ tar.add(str(f), f.name)
374+ # Pack originally supplied bundle.yaml into the final tarball
375+ shutil.copyfile(bundle_yaml_filepath, bundle_dir_path / "bundle.yaml")
376+ bundle_tar_path = (
377+ export_dir_path
378+ / f"{DEFAULT_CHARMS_EXPORT_FILENAME_PREFIX}-{timestamp}.tar.gz"
379+ )
380+ # Pack charm tarballs into a root tarball
381+ with tarfile.open(bundle_tar_path, "w:gz", encoding="utf-8") as tar:
382+ for f in bundle_dir_path.iterdir():
383+ tar.add(str(f), f.name)
384+
385+ return bundle_tar_path
386+
387+
388+def export_charm_bundle(
389+ bundle_name,
390+ export_dir_path,
391+ channel,
392+ charm_series=DEFAULT_SERIES,
393+ charm_architecture=DEFAULT_ARCHITECTURE,
394+):
395+ """Orchestrating function for exporting a charm bundle and its component charms.
396+
397+ Output structure:
398+
399+ <export_dir_path>/
400+ ├─ <bundle-name>-<timestamp>.tar.gz/
401+ │ ├─ bundle.yaml
402+ │ ├─ <bundle-name>_<bundle-revision>.bundle
403+ │ ├─ channel-map.json
404+ │ ├─ charms/
405+ │ │ ├─ <charm-name>.tar.gz/
406+ │ │ │ ├─ <charm-name>_<charm-revision>.charm
407+ │ │ │ ├─ channel-map.json
408+ │ │ │ ├─ resources/
409+ │ │ │ │ ├─ <charm-name>.<resource-name>_<resource-revision>
410+
411+ :param str bundle_name: Store package name for the bundle to be exported
412+ :param pathlib.Path export_dir_path: export directory path
413+ :param str channel: channel of the bundle itself to fetch, default "latest/stable"
414+ :param str charm_series: series of the charms to fetch, default "jammy"
415+ :param str charm_architecture: architecture of the charms to fetch, default "amd64"
416+ """
417 if not export_dir_path:
418 snap_user_common_dir = os.getenv("SNAP_USER_COMMON")
419 if snap_user_common_dir:
420@@ -307,36 +445,85 @@ def export_charms(
421 timestamp = (
422 datetime.datetime.utcnow().replace(microsecond=0).strftime("%Y%m%dT%H%M%S")
423 )
424+ # Functions calling /info and /refresh are shared with charms, but unlike charms
425+ # we would only export 1 bundle at a time.
426+ bundle_to_import = CharmImport(
427+ package_name=bundle_name, channel=channel, package_type="bundle"
428+ )
429+ bundle_info, bundle_revision = _fetch_store_info_for_packages([bundle_to_import])
430+ # Unpack bundle_revision into the format expected by `download_charm_files`
431+ bundle_revision = bundle_revision[bundle_name]
432+
433+ with tempfile.TemporaryDirectory() as bundle_dir_path:
434+ bundle_dir_path = pathlib.Path(bundle_dir_path)
435+
436+ # TODO bowenfan To format outputs per import requirements,
437+ # as opposed to just dumping the /info response
438+ write_channel_map(bundle_info[bundle_name], bundle_dir_path)
439+ download_charm_files(bundle_revision, bundle_dir_path, progress_copy_to_file)
440+
441+ # Automatically export all component charms in the bundle
442+ # If a user needs to modify the bundle before exporting it, they can use
443+ # the get-bundle-yaml command to output bundle.yaml only,
444+ # without auto fetching charms
445+ with zipfile.ZipFile(
446+ bundle_dir_path
447+ / f"{bundle_revision.package_name}_{bundle_revision.revision}.bundle"
448+ ) as bundle_zip:
449+ bundle_zip.extract(member="bundle.yaml", path=bundle_dir_path)
450+
451+ charms_dir_path = bundle_dir_path / "charms"
452+ charms_dir_path.mkdir(parents=True, exist_ok=False)
453+ charms_tar_path = export_charms(
454+ bundle_dir_path / "bundle.yaml",
455+ charms_dir_path,
456+ charm_series,
457+ charm_architecture,
458+ )
459
460- charms_to_import = parse_charm_bundle_yaml(bundle_yaml_filepath, architecture)
461- charms_info = {
462- charm.package_name: get_charm_info(devicegw_root, charm)
463- for charm in charms_to_import
464- }
465- charms_download_actions = make_charm_download_actions(charms_to_import, charms_info)
466- charms_download_info = {
467- res["name"]: res
468- for res in get_charms_download_info(devicegw_root, charms_download_actions)
469- }
470- charms_revisions = charm_revisions_from_info(charms_info, charms_download_info)
471-
472- tarball_paths = []
473- for charm_name, charm_revision in charms_revisions.items():
474- with tempfile.TemporaryDirectory() as charm_dir_path:
475- charm_dir_path = pathlib.Path(charm_dir_path)
476-
477- write_channel_map(charms_info[charm_name], charm_dir_path)
478- download_charm_files(
479- charm_revision,
480- charm_dir_path,
481- progress_copy_to_file,
482- )
483+ # Extract charms
484+ with tarfile.open(charms_tar_path, "r:gz") as charms_tar:
485+ charms_tar.extractall(charms_dir_path)
486+ # Remove extra bundle.yaml from the export_charms function
487+ os.remove(charms_dir_path / "bundle.yaml")
488+ # Remove the extracted exported charms tarball
489+ os.remove(charms_tar_path)
490+
491+ # Pack what we have into a root tar named after the bundle
492+ # (see dir structure in docstring)
493+ tarball_path = export_dir_path / f"{bundle_name}-{timestamp}.tar.gz"
494+ with tarfile.open(tarball_path, "w:gz", encoding="utf-8") as tar:
495+ for f in bundle_dir_path.iterdir():
496+ tar.add(str(object=f), f.name)
497+
498+ return (bundle_name, tarball_path)
499
500- tarball_path = export_dir_path / f"{charm_name}-{timestamp}.tar.gz"
501- with tarfile.open(tarball_path, "w:gz", encoding="utf-8") as tar:
502- for f in charm_dir_path.iterdir():
503- tar.add(str(f), f.name)
504
505- tarball_paths.append((charm_name, tarball_path))
506+def get_bundle_yaml(bundle_name, channel):
507+ """Get the bundle.yaml for a given bundle name and channel.
508
509- return tarball_paths
510+ :param str bundle_name: Store package name for the bundle to be exported
511+ :param str channel: channel of the bundle itself to fetch, default "latest/stable"
512+ """
513+ bundle_to_import = CharmImport(
514+ package_name=bundle_name, channel=channel, package_type="bundle"
515+ )
516+ _, bundle_revision = _fetch_store_info_for_packages([bundle_to_import])
517+ bundle_revision = bundle_revision[bundle_name]
518+
519+ with tempfile.TemporaryDirectory() as bundle_dir_path:
520+ bundle_dir_path = pathlib.Path(bundle_dir_path)
521+
522+ download_charm_files(bundle_revision, bundle_dir_path, progress_copy_to_file)
523+
524+ with tempfile.TemporaryDirectory(dir=bundle_dir_path) as zip_dir_path:
525+ zip_dir_path = pathlib.Path(zip_dir_path)
526+ with zipfile.ZipFile(
527+ bundle_dir_path
528+ / f"{bundle_revision.package_name}_{bundle_revision.revision}.bundle"
529+ ) as bundle_zip:
530+ bundle_zip.extractall(path=zip_dir_path)
531+ with open(zip_dir_path / "bundle.yaml") as bundle_yaml_file:
532+ logger.info("\n-----BEGIN BUNDLE.YAML-----")
533+ logger.info(bundle_yaml_file.read())
534+ logger.info("-----END BUNDLE.YAML-----")
535diff --git a/store_admin/logic/tests/test_charms.py b/store_admin/logic/tests/test_charms.py
536index f94a0b1..1d1c349 100644
537--- a/store_admin/logic/tests/test_charms.py
538+++ b/store_admin/logic/tests/test_charms.py
539@@ -1,13 +1,15 @@
540 import binascii
541 import datetime
542+import os
543 import pathlib
544 import tarfile
545 from textwrap import dedent
546 from unittest import mock
547+import zipfile
548
549 import pytest
550 import responses
551-
552+from io import BytesIO
553 from store_admin import exceptions
554 from store_admin.logic import charms
555 from store_admin.types import (
556@@ -21,6 +23,7 @@ from store_admin.types import (
557
558 PATCH_PREFIX = "store_admin.logic.charms."
559 TEST_CHARM_NAME = "charm1"
560+TEST_BUNDLE_NAME = "bundle1"
561
562
563 @pytest.fixture
564@@ -104,10 +107,142 @@ def setup_mock_charm_info():
565
566
567 @pytest.fixture
568+def setup_mock_bundle_info():
569+ def setup(devicegw_root="", requests_mock=None, bundle_name=TEST_BUNDLE_NAME):
570+ info_data = {
571+ "type": "bundle",
572+ "id": f"{bundle_name}Id",
573+ "name": bundle_name,
574+ "channel-map": [
575+ {
576+ "channel": {
577+ "base": None,
578+ "name": "stable",
579+ "released-at": "2024-01-03T14:53:38.952872+00:00",
580+ "risk": "stable",
581+ "track": "latest",
582+ },
583+ "revision": {
584+ "attributes": {},
585+ "bases": [None],
586+ "created-at": "2023-12-07T23:36:15.652705+00:00",
587+ "download": {
588+ "hash-sha-256": binascii.hexlify(b"5555").decode(),
589+ "size": 5555,
590+ "url": f"https://download/{bundle_name}_5.bundle",
591+ },
592+ "revision": 5,
593+ "version": "5",
594+ },
595+ },
596+ ],
597+ "result": {
598+ "categories": [],
599+ "deployable-on": ["kubernetes"],
600+ "description": "A charming charm bundle",
601+ "license": "",
602+ "links": None,
603+ "media": [],
604+ "publisher": {"display-name": "Publisher of charming charms"},
605+ "store-url": f"https://charmhub.io/{bundle_name}",
606+ "summary": "A charming charm bundle",
607+ "title": "A charming charm bundle",
608+ "unlisted": False,
609+ },
610+ }
611+ if requests_mock:
612+ requests_mock.add(
613+ "GET",
614+ devicegw_root + f"/v2/charms/info/{bundle_name}",
615+ match=[
616+ responses.matchers.query_param_matcher(
617+ {"fields": ",".join(charms.CHARM_INFO_FIELDS)}
618+ ),
619+ ],
620+ json=info_data,
621+ )
622+ return bundle_name, info_data
623+
624+ return setup
625+
626+
627+def _make_charm_refresh_results(charm_names):
628+ return [
629+ {
630+ "charm": {
631+ "created-at": "2023-12-07T23:36:15.652705+00:00",
632+ "download": {
633+ "hash-sha-256": binascii.hexlify(b"5555").decode(),
634+ "size": 5555,
635+ "url": f"https://download/{charm_name}_5.charm",
636+ },
637+ "id": f"{charm_name}Id",
638+ "name": charm_name,
639+ "publisher": {
640+ "display-name": "Publisher of charming charms",
641+ "id": "charmpublisherId",
642+ "username": "charm-publisher",
643+ "validation": "unproven",
644+ },
645+ "resources": [
646+ {
647+ "created-at": "2023-11-28T23:19:59.819552",
648+ "description": "Charm OCI Resource",
649+ "download": {
650+ "hash-sha-256": binascii.hexlify(b"2222").decode(),
651+ "hash-sha-384": binascii.hexlify(b"222").decode(),
652+ "hash-sha-512": binascii.hexlify(b"22").decode(),
653+ "hash-sha3-384": binascii.hexlify(b"2").decode(),
654+ "size": 222,
655+ "url": f"https://download/{charm_name}-image_2",
656+ },
657+ "filename": "",
658+ "name": f"{charm_name}-image",
659+ "revision": 2,
660+ "type": "oci-image",
661+ }
662+ ],
663+ "revision": 5,
664+ "summary": "A charming charm",
665+ "type": "charm",
666+ "version": "5",
667+ },
668+ "effective-channel": "stable",
669+ "id": f"{charm_name}Id",
670+ "instance-key": "random-instance-key",
671+ "name": charm_name,
672+ "released-at": "2024-01-03T14:53:38.952872+00:00",
673+ "result": "download",
674+ }
675+ for charm_name in charm_names
676+ ]
677+
678+
679+@pytest.fixture
680 def setup_mock_charm_refresh():
681 def setup(devicegw_root="", requests_mock=None, charm_names=None):
682 charm_names = charm_names or [TEST_CHARM_NAME]
683+ refresh_data = {
684+ "error-list": [],
685+ "results": _make_charm_refresh_results(charm_names),
686+ }
687+ if requests_mock:
688+ requests_mock.add(
689+ "POST", devicegw_root + "/v2/charms/refresh", json=refresh_data
690+ )
691+ return charm_names, refresh_data
692+
693+ return setup
694+
695
696+@pytest.fixture
697+def setup_mock_bundle_refresh():
698+ def setup(
699+ devicegw_root="",
700+ requests_mock=None,
701+ bundle_name=TEST_BUNDLE_NAME,
702+ charm_names=None,
703+ ):
704 refresh_data = {
705 "error-list": [],
706 "results": [
707@@ -117,54 +252,38 @@ def setup_mock_charm_refresh():
708 "download": {
709 "hash-sha-256": binascii.hexlify(b"5555").decode(),
710 "size": 5555,
711- "url": f"https://download/{charm_name}_5.charm",
712+ "url": f"https://download/{bundle_name}_5.bundle",
713 },
714- "id": f"{charm_name}Id",
715- "name": charm_name,
716+ "id": f"{bundle_name}Id",
717+ "name": bundle_name,
718 "publisher": {
719 "display-name": "Publisher of charming charms",
720 "id": "charmpublisherId",
721 "username": "charm-publisher",
722 "validation": "unproven",
723 },
724- "resources": [
725- {
726- "created-at": "2023-11-28T23:19:59.819552",
727- "description": "Charm OCI Resource",
728- "download": {
729- "hash-sha-256": binascii.hexlify(b"2222").decode(),
730- "hash-sha-384": binascii.hexlify(b"222").decode(),
731- "hash-sha-512": binascii.hexlify(b"22").decode(),
732- "hash-sha3-384": binascii.hexlify(b"2").decode(),
733- "size": 222,
734- "url": f"https://download/{charm_name}-image_2",
735- },
736- "filename": "",
737- "name": f"{charm_name}-image",
738- "revision": 2,
739- "type": "oci-image",
740- }
741- ],
742+ "resources": [],
743 "revision": 5,
744- "summary": "A charming charm",
745- "type": "charm",
746+ "summary": "A charming charm bundle",
747+ "type": "bundle",
748 "version": "5",
749 },
750 "effective-channel": "stable",
751- "id": f"{charm_name}Id",
752+ "id": f"{bundle_name}Id",
753 "instance-key": "random-instance-key",
754- "name": charm_name,
755+ "name": bundle_name,
756 "released-at": "2024-01-03T14:53:38.952872+00:00",
757 "result": "download",
758 }
759- for charm_name in charm_names
760 ],
761 }
762+ if charm_names:
763+ refresh_data["results"].extend(_make_charm_refresh_results(charm_names))
764 if requests_mock:
765 requests_mock.add(
766 "POST", devicegw_root + "/v2/charms/refresh", json=refresh_data
767 )
768- return charm_names, refresh_data
769+ return [bundle_name], refresh_data
770
771 return setup
772
773@@ -199,6 +318,7 @@ def mock_charm_revision_dataclass():
774 ),
775 package_name=charm_name,
776 package_id=f"{charm_name}Id",
777+ package_type="charm",
778 revision=5,
779 created_at="2023-12-07T23:36:15.652705+00:00",
780 architectures=[{"architecture": "amd64", "channel": "22.04", "name": "ubuntu"}],
781@@ -218,25 +338,72 @@ def mock_charm_revision_dataclass():
782 )
783
784
785-@pytest.fixture
786-def mock_bundle_yaml():
787- yaml_content = dedent(
788- """\
789- bundle: kubernetes
790- applications:
791- charm1:
792- charm: charm1
793- channel: stable
794- series: jammy
795- charm2:
796- charm: charm2
797- """
798+def _yaml_content(charm_names):
799+ return dedent(
800+ f"""\
801+ bundle: kubernetes
802+ applications:
803+ app1:
804+ charm: {charm_names[0]}
805+ channel: stable
806+ series: jammy
807+ app2:
808+ charm: {charm_names[1]}
809+ """
810 )
811+
812+
813+@pytest.fixture
814+def setup_mock_bundle_yaml():
815+ yaml_content = _yaml_content(["charm1", "charm2"])
816 with mock.patch("builtins.open", mock.mock_open(read_data=yaml_content)):
817 yield yaml_content
818
819
820 @pytest.fixture
821+def setup_charm_and_resource_download_mocks():
822+ def setup(charm_name, requests_mock):
823+ requests_mock.add(
824+ "GET",
825+ f"https://download/{charm_name}_5.charm",
826+ body="charm-blob".encode(),
827+ auto_calculate_content_length=True,
828+ content_type="application/octet-stream",
829+ )
830+ requests_mock.add(
831+ "GET",
832+ f"https://download/{charm_name}-image_2",
833+ body="resource-blob".encode(),
834+ auto_calculate_content_length=True,
835+ content_type="application/octet-stream",
836+ )
837+
838+ return setup
839+
840+
841+@pytest.fixture
842+def setup_bundle_download_mock():
843+ def setup(requests_mock, bundle_name=TEST_BUNDLE_NAME):
844+ yaml_content = _yaml_content(["charm1", "charm2"])
845+ # Use an in-memory zip file as the .bundle blob
846+ bundle_buf = BytesIO(yaml_content.encode())
847+ zip_buf = BytesIO()
848+ with zipfile.ZipFile(zip_buf, mode="a") as bundle_file:
849+ bundle_file.writestr("bundle.yaml", bundle_buf.getvalue())
850+
851+ requests_mock.add(
852+ "GET",
853+ f"https://download/{bundle_name}_5.bundle",
854+ body=zip_buf.getvalue(),
855+ auto_calculate_content_length=True,
856+ content_type="application/octet-stream",
857+ )
858+ return yaml_content
859+
860+ return setup
861+
862+
863+@pytest.fixture
864 def mock_charm_import():
865 charm_import = CharmImport(
866 package_name=TEST_CHARM_NAME,
867@@ -247,7 +414,7 @@ def mock_charm_import():
868 return charm_import
869
870
871-def test_parse_charm_bundle_yaml(mock_bundle_yaml):
872+def test_parse_charm_bundle_yaml(setup_mock_bundle_yaml):
873 expected = [
874 CharmImport(
875 package_name="charm1",
876@@ -262,7 +429,7 @@ def test_parse_charm_bundle_yaml(mock_bundle_yaml):
877 architecture="amd64",
878 ),
879 ]
880- parsed = charms.parse_charm_bundle_yaml("test_bundle.yaml", "amd64")
881+ parsed = charms.parse_charm_bundle_yaml("test_bundle.yaml", "jammy", "amd64")
882 assert parsed == expected
883
884
885@@ -434,6 +601,24 @@ def test_download_charm_files_success(
886 assert _file_read_cb.call_count == 2
887
888
889+def _assert_charm_tar_contents(charm_tar_paths, tmpdir):
890+ for path in charm_tar_paths:
891+ charm_name = os.path.basename(path).replace(".tar.gz", "")
892+ with tarfile.open(path, "r:gz") as tar:
893+ assert set(ti.name for ti in tar.getmembers()) == {
894+ "channel-map.json",
895+ f"{charm_name}_5.charm",
896+ "resources",
897+ f"resources/{charm_name}.{charm_name}-image_2",
898+ }
899+ path = pathlib.Path(tmpdir.mkdir(f"{charm_name}-extract"))
900+ tar.extractall(path)
901+ assert (path / f"{charm_name}_5.charm").read_text() == "charm-blob"
902+ assert (
903+ path / f"resources/{charm_name}.{charm_name}-image_2"
904+ ).read_text() == "resource-blob"
905+
906+
907 def test_export_charms_success(
908 tmpdir,
909 devicegw_root,
910@@ -441,33 +626,25 @@ def test_export_charms_success(
911 utcnow,
912 setup_mock_charm_info,
913 setup_mock_charm_refresh,
914+ setup_charm_and_resource_download_mocks,
915 ):
916 charm_names = ["charm1", "charm2"]
917 for charm_name in charm_names:
918- requests_mock.add(
919- "GET",
920- f"https://download/{charm_name}_5.charm",
921- body="charm-blob".encode(),
922- auto_calculate_content_length=True,
923- content_type="application/octet-stream",
924- )
925- requests_mock.add(
926- "GET",
927- f"https://download/{charm_name}-image_2",
928- body="resource-blob".encode(),
929- auto_calculate_content_length=True,
930- content_type="application/octet-stream",
931- )
932- _ = setup_mock_charm_info(devicegw_root, requests_mock, charm_name)
933- _ = setup_mock_charm_refresh(devicegw_root, requests_mock, charm_names)
934+ setup_charm_and_resource_download_mocks(charm_name, requests_mock)
935+ setup_mock_charm_info(devicegw_root, requests_mock, charm_name)
936+ setup_mock_charm_refresh(devicegw_root, requests_mock, charm_names)
937
938 timestamp = utcnow.replace(microsecond=0).strftime("%Y%m%dT%H%M%S")
939
940- def _side_effect(raw_response, target_path, length):
941+ def _progress_copy_to_file_side_effect(raw_response, target_path, length):
942 data = raw_response.read()
943 assert len(data) == length
944 target_path.write_bytes(data)
945
946+ def _copyfile_side_effect(_, destination):
947+ with open(destination, "w") as f:
948+ f.write("mocked_bundle.yaml-content-does-not-matter")
949+
950 _m_parsed_charm_import = [
951 CharmImport(
952 package_name="charm1",
953@@ -484,30 +661,122 @@ def test_export_charms_success(
954 ]
955
956 with mock.patch(
957- PATCH_PREFIX + "progress_copy_to_file", side_effect=_side_effect
958+ PATCH_PREFIX + "progress_copy_to_file",
959+ side_effect=_progress_copy_to_file_side_effect,
960 ), mock.patch(
961 PATCH_PREFIX + "parse_charm_bundle_yaml", return_value=_m_parsed_charm_import
962+ ), mock.patch(
963+ PATCH_PREFIX + "shutil.copyfile", _copyfile_side_effect
964 ):
965 export_dir_path = pathlib.Path(tmpdir.mkdir("export-dir"))
966- tarball_paths = charms.export_charms("test_bundle.yaml", export_dir_path)
967+ exported_tar_path = charms.export_charms("test_bundle.yaml", export_dir_path)
968
969- assert tarball_paths == [
970- ("charm1", export_dir_path / f"charm1-{timestamp}.tar.gz"),
971- ("charm2", export_dir_path / f"charm2-{timestamp}.tar.gz"),
972- ]
973- assert set(export_dir_path.iterdir()) == {t[1] for t in tarball_paths}
974+ assert (
975+ exported_tar_path
976+ == export_dir_path
977+ / f"{charms.DEFAULT_CHARMS_EXPORT_FILENAME_PREFIX}-{timestamp}.tar.gz"
978+ )
979
980- for charm_name, tarball_path in tarball_paths:
981- with tarfile.open(tarball_path, "r:gz") as tar:
982- assert set(ti.name for ti in tar.getmembers()) == {
983- "channel-map.json",
984- f"{charm_name}_5.charm",
985- "resources",
986- f"resources/{charm_name}.{charm_name}-image_2",
987- }
988- path = pathlib.Path(tmpdir.mkdir(f"{charm_name}-extract"))
989- tar.extractall(path)
990- assert (path / f"{charm_name}_5.charm").read_text() == "charm-blob"
991- assert (
992- path / f"resources/{charm_name}.{charm_name}-image_2"
993- ).read_text() == "resource-blob"
994+ charms_dir_path = export_dir_path / "charms"
995+ with tarfile.open(exported_tar_path, "r:gz") as parent_tar:
996+ parent_tar.extractall(charms_dir_path)
997+
998+ root_dir_contents = {
999+ pathlib.Path(os.path.join(charms_dir_path, p))
1000+ for p in os.listdir(charms_dir_path)
1001+ }
1002+ assert root_dir_contents == {
1003+ charms_dir_path / "charm1.tar.gz",
1004+ charms_dir_path / "charm2.tar.gz",
1005+ charms_dir_path / "bundle.yaml",
1006+ }
1007+
1008+ _assert_charm_tar_contents(
1009+ [charms_dir_path / "charm1.tar.gz", charms_dir_path / "charm2.tar.gz"], tmpdir
1010+ )
1011+
1012+
1013+def test_export_bundle_success(
1014+ tmpdir,
1015+ devicegw_root,
1016+ requests_mock,
1017+ utcnow,
1018+ setup_mock_bundle_info,
1019+ setup_mock_bundle_refresh,
1020+ setup_mock_charm_info,
1021+ setup_bundle_download_mock,
1022+ setup_charm_and_resource_download_mocks,
1023+):
1024+ bundle_yaml_contents = setup_bundle_download_mock(requests_mock)
1025+ charm_names = ["charm1", "charm2"]
1026+ for charm_name in charm_names:
1027+ setup_charm_and_resource_download_mocks(charm_name, requests_mock)
1028+ setup_mock_charm_info(devicegw_root, requests_mock, charm_name)
1029+
1030+ setup_mock_bundle_info(devicegw_root, requests_mock, TEST_BUNDLE_NAME)
1031+ setup_mock_bundle_refresh(
1032+ devicegw_root, requests_mock, TEST_BUNDLE_NAME, charm_names
1033+ )
1034+
1035+ timestamp = utcnow.replace(microsecond=0).strftime("%Y%m%dT%H%M%S")
1036+
1037+ def _side_effect(raw_response, target_path, length):
1038+ data = raw_response.read()
1039+ assert len(data) == length
1040+ target_path.write_bytes(data)
1041+
1042+ with mock.patch(PATCH_PREFIX + "progress_copy_to_file", side_effect=_side_effect):
1043+ export_dir_path = pathlib.Path(tmpdir.mkdir("export-dir"))
1044+ bundle_name, tarball_path = charms.export_charm_bundle(
1045+ TEST_BUNDLE_NAME, export_dir_path, "stable"
1046+ )
1047+
1048+ assert bundle_name == TEST_BUNDLE_NAME
1049+ assert tarball_path == export_dir_path / f"{TEST_BUNDLE_NAME}-{timestamp}.tar.gz"
1050+
1051+ with tarfile.open(tarball_path, "r:gz") as tar:
1052+ assert set(ti.name for ti in tar.getmembers()) == {
1053+ "bundle.yaml",
1054+ "channel-map.json",
1055+ "bundle1_5.bundle",
1056+ "charms",
1057+ "charms/charm1.tar.gz",
1058+ "charms/charm2.tar.gz",
1059+ }
1060+ bundle_path = pathlib.Path(tmpdir.mkdir(f"{TEST_BUNDLE_NAME}-extract"))
1061+ tar.extractall(bundle_path)
1062+ assert (bundle_path / "bundle.yaml").read_text() == bundle_yaml_contents
1063+
1064+ _assert_charm_tar_contents(
1065+ [
1066+ bundle_path / "charms" / "charm1.tar.gz",
1067+ bundle_path / "charms" / "charm2.tar.gz",
1068+ ],
1069+ tmpdir,
1070+ )
1071+
1072+
1073+def test_get_bundle_yaml_success(
1074+ devicegw_root,
1075+ requests_mock,
1076+ caplog,
1077+ setup_mock_bundle_info,
1078+ setup_mock_bundle_refresh,
1079+ setup_bundle_download_mock,
1080+):
1081+ bundle_yaml_contents = setup_bundle_download_mock(requests_mock)
1082+ charm_names = ["charm1", "charm2"]
1083+ _ = setup_mock_bundle_info(devicegw_root, requests_mock, TEST_BUNDLE_NAME)
1084+ _ = setup_mock_bundle_refresh(
1085+ devicegw_root, requests_mock, TEST_BUNDLE_NAME, charm_names
1086+ )
1087+
1088+ def _side_effect(raw_response, target_path, length):
1089+ data = raw_response.read()
1090+ assert len(data) == length
1091+ target_path.write_bytes(data)
1092+
1093+ with mock.patch(PATCH_PREFIX + "progress_copy_to_file", side_effect=_side_effect):
1094+ charms.get_bundle_yaml(TEST_BUNDLE_NAME, "latest/stable")
1095+
1096+ assert bundle_yaml_contents in caplog.messages
1097diff --git a/store_admin/types.py b/store_admin/types.py
1098index b3fe34f..e8cc1bd 100644
1099--- a/store_admin/types.py
1100+++ b/store_admin/types.py
1101@@ -44,9 +44,10 @@ DEFAULT_OS = "ubuntu"
1102 @dataclass
1103 class CharmImport:
1104 package_name: str
1105- series: str
1106- architecture: str
1107+ series: Optional[str] = None
1108+ architecture: Optional[str] = None
1109 channel: Optional[str] = None
1110+ package_type: str = "charm"
1111
1112 def __post_init__(self):
1113 if self.channel:
1114@@ -58,22 +59,16 @@ class CharmImport:
1115 def as_dict(self):
1116 return {k: v for k, v in asdict(obj=self).items() if v is not None}
1117
1118- def get_charm_base_dict(self):
1119- return {
1120- "name": DEFAULT_OS,
1121- "channel": self.series,
1122- "architecture": self.architecture,
1123- }
1124-
1125
1126 @dataclass
1127 class CharmDownloadAction:
1128 package_name: str
1129 package_id: str
1130- series: str
1131- architecture: str
1132+ series: Optional[str] = None
1133+ architecture: Optional[str] = None
1134 channel: Optional[str] = None
1135 action: str = "download"
1136+ package_type: str = "charm"
1137
1138 @property
1139 def instance_key(self):
1140@@ -84,7 +79,7 @@ class CharmDownloadAction:
1141 """
1142 return base64.b64encode(str(uuid4()).encode()).decode()
1143
1144- def get_charm_base_dict(self):
1145+ def _get_charm_base_dict(self):
1146 try:
1147 return {
1148 "name": DEFAULT_OS,
1149@@ -97,7 +92,10 @@ class CharmDownloadAction:
1150 def to_request_dict(self):
1151 request_dict = {
1152 "action": self.action,
1153- "base": self.get_charm_base_dict(),
1154+ # We still need to send {"base": null} for bundles to conform to the API
1155+ "base": (
1156+ self._get_charm_base_dict() if self.package_type == "charm" else None
1157+ ),
1158 "instance-key": self.instance_key,
1159 "id": self.package_id,
1160 }
1161@@ -157,6 +155,7 @@ class CharmRevision:
1162 channel: CharmChannel
1163 package_name: str
1164 package_id: str
1165+ package_type: str
1166 revision: int
1167 created_at: str
1168 architectures: List[dict]
1169@@ -171,7 +170,7 @@ class CharmRevision:
1170
1171 def blob_file_name(self):
1172 """Name of the downloaded revision blob file."""
1173- return f"{self.package_name}_{self.revision}.charm"
1174+ return f"{self.package_name}_{self.revision}.{self.package_type}"
1175
1176 def resource_blob_file_name(self, resource):
1177 """Name of the downloaded resource revision blob file."""

Subscribers

People subscribed via source and target branches

to all changes: