Merge lp:~jtv/maas/split-boot_resources into lp:~maas-committers/maas/trunk

Proposed by Jeroen T. Vermeulen
Status: Merged
Approved by: Jeroen T. Vermeulen
Approved revision: no longer in the source branch.
Merged at revision: 2245
Proposed branch: lp:~jtv/maas/split-boot_resources
Merge into: lp:~maas-committers/maas/trunk
Diff against target: 1823 lines (+953/-729)
11 files modified
src/provisioningserver/import_images/boot_image_mapping.py (+61/-0)
src/provisioningserver/import_images/boot_resources.py (+8/-299)
src/provisioningserver/import_images/download_descriptions.py (+187/-0)
src/provisioningserver/import_images/helpers.py (+73/-0)
src/provisioningserver/import_images/product_mapping.py (+65/-0)
src/provisioningserver/import_images/testing/factory.py (+56/-0)
src/provisioningserver/import_images/tests/test_boot_image_mapping.py (+98/-0)
src/provisioningserver/import_images/tests/test_boot_resources.py (+10/-430)
src/provisioningserver/import_images/tests/test_download_descriptions.py (+209/-0)
src/provisioningserver/import_images/tests/test_helpers.py (+61/-0)
src/provisioningserver/import_images/tests/test_product_mapping.py (+125/-0)
To merge this branch: bzr merge lp:~jtv/maas/split-boot_resources
Reviewer Review Type Date Requested Status
Graham Binns (community) Approve
Review via email: mp+214670@code.launchpad.net

Commit message

Break the boot_resources.py file up into smaller parts, for manageability.

Description of the change

This may look like a big branch, but it isn't, really. The code is moved, but nothing else is changed (except available_boot_resources: being unused and untested, it was removed).

Lots of things were going on in this file, including two Simplestreams synchronisations. Now that we have proper classes for some of the additional components, and a clearer idea of what RepoDumper does and how it fits into the big picture, we can split things up. It is particularly gratifying that RepoDumper itself is now entirely local to its module. A lot of cognitive load is relieved by simply not needing to know about this class for maintenance that doesn't involve its specific function.

Jeroen

To post a comment you must log in.
Revision history for this message
Graham Binns (gmb) wrote :

I love these branches that are smaller than they look. Gives me a sense of achievement for not much cost :).

review: Approve
Revision history for this message
MAAS Lander (maas-lander) wrote :

Attempt to merge into lp:maas failed due to conflicts:

text conflict in src/provisioningserver/import_images/tests/test_boot_resources.py

Revision history for this message
Jeroen T. Vermeulen (jtv) wrote :

I think that may just have been a silly little conflict between an import that I removed in this branch, but also ran through the formatter in a concurrent lint branch.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== added file 'src/provisioningserver/import_images/boot_image_mapping.py'
--- src/provisioningserver/import_images/boot_image_mapping.py 1970-01-01 00:00:00 +0000
+++ src/provisioningserver/import_images/boot_image_mapping.py 2014-04-08 10:22:55 +0000
@@ -0,0 +1,61 @@
1# Copyright 2014 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).
3
4"""The `BootImageMapping` class."""
5
6from __future__ import (
7 absolute_import,
8 print_function,
9 unicode_literals,
10 )
11
12str = None
13
14__metaclass__ = type
15__all__ = [
16 'BootImageMapping',
17 ]
18
19import json
20
21from provisioningserver.import_images.helpers import ImageSpec
22
23
24class BootImageMapping:
25 """Mapping of boot-image data.
26
27 Maps `ImageSpec` tuples to metadata for Simplestreams products.
28
29 This class is deliberately a bit more restrictive and less ad-hoc than a
30 dict. It helps keep a clear view of the data structures in this module.
31 """
32
33 def __init__(self):
34 self.mapping = {}
35
36 def items(self):
37 """Iterate over `ImageSpec` keys, and their stored values."""
38 for image_spec, item in sorted(self.mapping.items()):
39 yield image_spec, item
40
41 def setdefault(self, image_spec, item):
42 """Set metadata for `image_spec` to item, if not already set."""
43 assert isinstance(image_spec, ImageSpec)
44 self.mapping.setdefault(image_spec, item)
45
46 def dump_json(self):
47 """Produce JSON representing the mapped boot images.
48
49 Tries to keep the output deterministic, so that identical data is
50 likely to produce identical JSON.
51 """
52 # The meta files represent the mapping as a nested hierarchy of dicts.
53 # Keep that format.
54 data = {}
55 for image, resource in self.items():
56 arch, subarch, release, label = image
57 data.setdefault(arch, {})
58 data[arch].setdefault(subarch, {})
59 data[arch][subarch].setdefault(release, {})
60 data[arch][subarch][release][label] = resource
61 return json.dumps(data, sort_keys=True)
062
=== modified file 'src/provisioningserver/import_images/boot_resources.py'
--- src/provisioningserver/import_images/boot_resources.py 2014-04-08 05:25:24 +0000
+++ src/provisioningserver/import_images/boot_resources.py 2014-04-08 10:22:55 +0000
@@ -12,26 +12,27 @@
12__metaclass__ = type12__metaclass__ = type
13__all__ = [13__all__ = [
14 'main',14 'main',
15 'available_boot_resources',
16 'make_arg_parser',15 'make_arg_parser',
17 ]16 ]
1817
19from argparse import ArgumentParser18from argparse import ArgumentParser
20from collections import namedtuple
21from datetime import datetime19from datetime import datetime
22import errno20import errno
23import functools
24import glob
25from gzip import GzipFile21from gzip import GzipFile
26import json
27import logging
28from logging import getLogger
29import os22import os
30from textwrap import dedent23from textwrap import dedent
3124
32from provisioningserver.boot import BootMethodRegistry25from provisioningserver.boot import BootMethodRegistry
33from provisioningserver.boot.tftppath import list_boot_images26from provisioningserver.boot.tftppath import list_boot_images
34from provisioningserver.config import BootConfig27from provisioningserver.config import BootConfig
28from provisioningserver.import_images.download_descriptions import (
29 download_all_image_descriptions,
30 )
31from provisioningserver.import_images.helpers import (
32 get_signing_policy,
33 logger,
34 )
35from provisioningserver.import_images.product_mapping import ProductMapping
35from provisioningserver.utils import (36from provisioningserver.utils import (
36 atomic_write,37 atomic_write,
37 call_and_check,38 call_and_check,
@@ -47,200 +48,14 @@
47from simplestreams.util import (48from simplestreams.util import (
48 item_checksums,49 item_checksums,
49 path_from_mirror_url,50 path_from_mirror_url,
50 policy_read_signed,
51 products_exdata,51 products_exdata,
52 )52 )
5353
5454
55def init_logger(log_level=logging.INFO):
56 logger = getLogger(__name__)
57 formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
58 handler = logging.StreamHandler()
59 handler.setFormatter(formatter)
60 logger.addHandler(handler)
61 logger.setLevel(log_level)
62 return logger
63
64
65logger = init_logger()
66
67
68class NoConfigFile(Exception):55class NoConfigFile(Exception):
69 """Raised when the config file for the script doesn't exist."""56 """Raised when the config file for the script doesn't exist."""
7057
7158
72# A tuple of the items that together select a boot image.
73ImageSpec = namedtuple(b'ImageSpec', [
74 'arch',
75 'subarch',
76 'release',
77 'label',
78 ])
79
80
81class BootImageMapping:
82 """Mapping of boot-image data.
83
84 Maps `ImageSpec` tuples to metadata for Simplestreams products.
85
86 This class is deliberately a bit more restrictive and less ad-hoc than a
87 dict. It helps keep a clear view of the data structures in this module.
88 """
89
90 def __init__(self):
91 self.mapping = {}
92
93 def items(self):
94 """Iterate over `ImageSpec` keys, and their stored values."""
95 for image_spec, item in sorted(self.mapping.items()):
96 yield image_spec, item
97
98 def setdefault(self, image_spec, item):
99 """Set metadata for `image_spec` to item, if not already set."""
100 assert isinstance(image_spec, ImageSpec)
101 self.mapping.setdefault(image_spec, item)
102
103 def dump_json(self):
104 """Produce JSON representing the mapped boot images.
105
106 Tries to keep the output deterministic, so that identical data is
107 likely to produce identical JSON.
108 """
109 # The meta files represent the mapping as a nested hierarchy of dicts.
110 # Keep that format.
111 data = {}
112 for image, resource in self.items():
113 arch, subarch, release, label = image
114 data.setdefault(arch, {})
115 data[arch].setdefault(subarch, {})
116 data[arch][subarch].setdefault(release, {})
117 data[arch][subarch][release][label] = resource
118 return json.dumps(data, sort_keys=True)
119
120
121def value_passes_filter_list(filter_list, property_value):
122 """Does the given property of a boot image pass the given filter list?
123
124 The value passes if either it matches one of the entries in the list of
125 filter values, or one of the filter values is an asterisk (`*`).
126 """
127 return '*' in filter_list or property_value in filter_list
128
129
130def value_passes_filter(filter_value, property_value):
131 """Does the given property of a boot image pass the given filter?
132
133 The value passes the filter if either the filter value is an asterisk
134 (`*`) or the value is equal to the filter value.
135 """
136 return filter_value in ('*', property_value)
137
138
139def image_passes_filter(filters, arch, subarch, release, label):
140 """Filter a boot image against configured import filters.
141
142 :param filters: A list of dicts describing the filters, as in `boot_merge`.
143 If the list is empty, or `None`, any image matches. Any entry in a
144 filter may be a string containing just an asterisk (`*`) to denote that
145 the entry will match any value.
146 :param arch: The given boot image's architecture.
147 :param subarch: The given boot image's subarchitecture.
148 :param release: The given boot image's OS release.
149 :param label: The given boot image's label.
150 :return: Whether the image matches any of the dicts in `filters`.
151 """
152 if filters is None or len(filters) == 0:
153 return True
154 for filter_dict in filters:
155 item_matches = (
156 value_passes_filter(filter_dict['release'], release) and
157 value_passes_filter_list(filter_dict['arches'], arch) and
158 value_passes_filter_list(filter_dict['subarches'], subarch) and
159 value_passes_filter_list(filter_dict['labels'], label)
160 )
161 if item_matches:
162 return True
163 return False
164
165
166def boot_merge(destination, additions, filters=None):
167 """Complement one `BootImageMapping` with entries from another.
168
169 This adds entries from `additions` (that match `filters`, if given) to
170 `destination`, but only for those image specs for which `destination` does
171 not have entries yet.
172
173 :param destination: `BootImageMapping` to be updated. It will be extended
174 in-place.
175 :param additions: A second `BootImageMapping`, which will be used as a
176 source of additional entries.
177 :param filters: List of dicts, each of which contains 'arch', 'subarch',
178 and 'release' keys. If given, entries are only considered for copying
179 from `additions` to `destination` if they match at least one of the
180 filters. Entries in the filter may be the string `*` (or for entries
181 that are lists, may contain the string `*`) to make them match any
182 value.
183 """
184 for image, resource in additions.items():
185 arch, subarch, release, label = image
186 if image_passes_filter(filters, arch, subarch, release, label):
187 logger.debug(
188 "Merging boot resource for %s/%s/%s/%s.",
189 arch, subarch, release, label)
190 # Do not override an existing entry with the same
191 # arch/subarch/release/label: the first entry found takes
192 # precedence.
193 destination.setdefault(image, resource)
194
195
196class ProductMapping:
197 """Mapping of product data.
198
199 Maps a combination of boot resource metadata (`content_id`, `product_name`,
200 `version_name`) to a list of subarchitectures supported by that boot
201 resource.
202 """
203
204 def __init__(self):
205 self.mapping = {}
206
207 @staticmethod
208 def make_key(resource):
209 """Extract a key tuple from `resource`.
210
211 The key is used for indexing `mapping`.
212
213 :param resource: A dict describing a boot resource. It must contain
214 the keys `content_id`, `product_name`, and `version_name`.
215 :return: A tuple of the resource's content ID, product name, and
216 version name.
217 """
218 return (
219 resource['content_id'],
220 resource['product_name'],
221 resource['version_name'],
222 )
223
224 def add(self, resource, subarch):
225 """Add `subarch` to the list of subarches supported by a boot resource.
226
227 The `resource` is a dict as returned by `products_exdata`. The method
228 will use the values identified by keys `content_id`, `product_name`,
229 and `version_name`.
230 """
231 key = self.make_key(resource)
232 self.mapping.setdefault(key, [])
233 self.mapping[key].append(subarch)
234
235 def contains(self, resource):
236 """Does the dict contain a mapping for the given resource?"""
237 return self.make_key(resource) in self.mapping
238
239 def get(self, resource):
240 """Return the mapped subarchitectures for `resource`."""
241 return self.mapping[self.make_key(resource)]
242
243
244def boot_reverse(boot_images_dict):59def boot_reverse(boot_images_dict):
245 """Determine the subarches supported by each boot resource.60 """Determine the subarches supported by each boot resource.
24661
@@ -299,106 +114,6 @@
299 return entry114 return entry
300115
301116
302def get_signing_policy(path, keyring=None):
303 """Return Simplestreams signing policy for the given path.
304
305 :param path: Path to the Simplestreams index file.
306 :param keyring: Optional keyring file for verifying signatures.
307 :return: A "signing policy" callable. It accepts a file's content, path,
308 and optional keyring as arguments, and if the signature verifies
309 correctly, returns the content. The keyring defaults to the one you
310 pass.
311 """
312 if path.endswith('.json'):
313 # The configuration deliberately selected an un-signed index. A signed
314 # index would have a suffix of '.sjson'. Use a policy that doesn't
315 # check anything.
316 policy = lambda content, path, keyring: content
317 else:
318 # Otherwise: use default Simplestreams policy for verifying signatures.
319 policy = policy_read_signed
320
321 if keyring is not None:
322 # Pass keyring to the policy, to use if the caller inside Simplestreams
323 # does not provide one.
324 policy = functools.partial(policy, keyring=keyring)
325
326 return policy
327
328
329def clean_up_repo_item(item):
330 """Return a subset of dict `item` for storing in a boot images dict."""
331 keys_to_keep = ['content_id', 'product_name', 'version_name', 'path']
332 compact_item = {key: item[key] for key in keys_to_keep}
333 return compact_item
334
335
336class RepoDumper(BasicMirrorWriter):
337 """Gather metadata about boot images available in a Simplestreams repo.
338
339 Used inside `download_image_descriptions`. Stores basic metadata about
340 each image it finds upstream in a given `BootImageMapping`. Each stored
341 item is a dict containing the basic metadata for retrieving a boot image.
342
343 Simplestreams' `BasicMirrorWriter` in itself is stateless. It relies on
344 a subclass (such as this one) to store data.
345
346 :ivar boot_images_dict: A `BootImageMapping`. Image metadata will be
347 stored here as it is discovered. Simplestreams does not interact with
348 this variable.
349 """
350
351 def __init__(self, boot_images_dict):
352 super(RepoDumper, self).__init__()
353 self.boot_images_dict = boot_images_dict
354
355 def load_products(self, path=None, content_id=None):
356 """Overridable from `BasicMirrorWriter`."""
357 # It looks as if this method only makes sense for MirrorReaders, not
358 # for MirrorWriters. The default MirrorWriter implementation just
359 # raises NotImplementedError. Stop it from doing that.
360 return
361
362 def insert_item(self, data, src, target, pedigree, contentsource):
363 """Overridable from `BasicMirrorWriter`."""
364 item = products_exdata(src, pedigree)
365 arch, subarches = item['arch'], item['subarches']
366 release = item['release']
367 label = item['label']
368 base_image = ImageSpec(arch, None, release, label)
369 compact_item = clean_up_repo_item(item)
370 for subarch in subarches.split(','):
371 self.boot_images_dict.setdefault(
372 base_image._replace(subarch=subarch), compact_item)
373
374
375def download_image_descriptions(path, keyring=None):
376 """Download image metadata from upstream Simplestreams repo.
377
378 :param path: The path to a Simplestreams repo.
379 :param keyring: Optional keyring for verifying the repo's signatures.
380 :return: A nested dict of data, indexed by image arch, subarch, release,
381 and label.
382 """
383 mirror, rpath = path_from_mirror_url(path, None)
384 policy = get_signing_policy(rpath, keyring)
385 reader = UrlMirrorReader(mirror, policy=policy)
386 boot_images_dict = BootImageMapping()
387 dumper = RepoDumper(boot_images_dict)
388 dumper.sync(reader, rpath)
389 return boot_images_dict
390
391
392def download_all_image_descriptions(config):
393 """Download image metadata for all sources in `config`."""
394 boot = BootImageMapping()
395 for source in config['boot']['sources']:
396 repo_boot = download_image_descriptions(
397 source['path'], keyring=source['keyring'])
398 boot_merge(boot, repo_boot, source['selections'])
399 return boot
400
401
402class RepoWriter(BasicMirrorWriter):117class RepoWriter(BasicMirrorWriter):
403 """Download boot resources from an upstream Simplestreams repo.118 """Download boot resources from an upstream Simplestreams repo.
404119
@@ -474,12 +189,6 @@
474 os.link(src, link_path)189 os.link(src, link_path)
475190
476191
477def available_boot_resources(root):
478 for resource_path in glob.glob(os.path.join(root, '*/*/*/*')):
479 arch, subarch, release, label = resource_path.split('/')[-4:]
480 yield (arch, subarch, release, label)
481
482
483def install_boot_loaders(destination):192def install_boot_loaders(destination):
484 """Install the all the required file from each bootloader method.193 """Install the all the required file from each bootloader method.
485 :param destination: Directory where the loaders should be stored.194 :param destination: Directory where the loaders should be stored.
486195
=== added file 'src/provisioningserver/import_images/download_descriptions.py'
--- src/provisioningserver/import_images/download_descriptions.py 1970-01-01 00:00:00 +0000
+++ src/provisioningserver/import_images/download_descriptions.py 2014-04-08 10:22:55 +0000
@@ -0,0 +1,187 @@
1# Copyright 2014 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).
3
4"""Download boot resource descriptions from Simplestreams repo.
5
6This module is responsible only for syncing the repo's metadata, not the boot
7resources themselves. The two are handled in separate Simplestreams
8synchronisation stages.
9"""
10
11from __future__ import (
12 absolute_import,
13 print_function,
14 unicode_literals,
15 )
16
17str = None
18
19__metaclass__ = type
20__all__ = [
21 'download_all_image_descriptions',
22 ]
23
24
25from provisioningserver.import_images.boot_image_mapping import (
26 BootImageMapping,
27 )
28from provisioningserver.import_images.helpers import (
29 get_signing_policy,
30 ImageSpec,
31 logger,
32 )
33from simplestreams.mirrors import (
34 BasicMirrorWriter,
35 UrlMirrorReader,
36 )
37from simplestreams.util import (
38 path_from_mirror_url,
39 products_exdata,
40 )
41
42
43def clean_up_repo_item(item):
44 """Return a subset of dict `item` for storing in a boot images dict."""
45 keys_to_keep = ['content_id', 'product_name', 'version_name', 'path']
46 compact_item = {key: item[key] for key in keys_to_keep}
47 return compact_item
48
49
50class RepoDumper(BasicMirrorWriter):
51 """Gather metadata about boot images available in a Simplestreams repo.
52
53 Used inside `download_image_descriptions`. Stores basic metadata about
54 each image it finds upstream in a given `BootImageMapping`. Each stored
55 item is a dict containing the basic metadata for retrieving a boot image.
56
57 Simplestreams' `BasicMirrorWriter` in itself is stateless. It relies on
58 a subclass (such as this one) to store data.
59
60 :ivar boot_images_dict: A `BootImageMapping`. Image metadata will be
61 stored here as it is discovered. Simplestreams does not interact with
62 this variable.
63 """
64
65 def __init__(self, boot_images_dict):
66 super(RepoDumper, self).__init__()
67 self.boot_images_dict = boot_images_dict
68
69 def load_products(self, path=None, content_id=None):
70 """Overridable from `BasicMirrorWriter`."""
71 # It looks as if this method only makes sense for MirrorReaders, not
72 # for MirrorWriters. The default MirrorWriter implementation just
73 # raises NotImplementedError. Stop it from doing that.
74 return
75
76 def insert_item(self, data, src, target, pedigree, contentsource):
77 """Overridable from `BasicMirrorWriter`."""
78 item = products_exdata(src, pedigree)
79 arch, subarches = item['arch'], item['subarches']
80 release = item['release']
81 label = item['label']
82 base_image = ImageSpec(arch, None, release, label)
83 compact_item = clean_up_repo_item(item)
84 for subarch in subarches.split(','):
85 self.boot_images_dict.setdefault(
86 base_image._replace(subarch=subarch), compact_item)
87
88
89def value_passes_filter_list(filter_list, property_value):
90 """Does the given property of a boot image pass the given filter list?
91
92 The value passes if either it matches one of the entries in the list of
93 filter values, or one of the filter values is an asterisk (`*`).
94 """
95 return '*' in filter_list or property_value in filter_list
96
97
98def value_passes_filter(filter_value, property_value):
99 """Does the given property of a boot image pass the given filter?
100
101 The value passes the filter if either the filter value is an asterisk
102 (`*`) or the value is equal to the filter value.
103 """
104 return filter_value in ('*', property_value)
105
106
107def image_passes_filter(filters, arch, subarch, release, label):
108 """Filter a boot image against configured import filters.
109
110 :param filters: A list of dicts describing the filters, as in `boot_merge`.
111 If the list is empty, or `None`, any image matches. Any entry in a
112 filter may be a string containing just an asterisk (`*`) to denote that
113 the entry will match any value.
114 :param arch: The given boot image's architecture.
115 :param subarch: The given boot image's subarchitecture.
116 :param release: The given boot image's OS release.
117 :param label: The given boot image's label.
118 :return: Whether the image matches any of the dicts in `filters`.
119 """
120 if filters is None or len(filters) == 0:
121 return True
122 for filter_dict in filters:
123 item_matches = (
124 value_passes_filter(filter_dict['release'], release) and
125 value_passes_filter_list(filter_dict['arches'], arch) and
126 value_passes_filter_list(filter_dict['subarches'], subarch) and
127 value_passes_filter_list(filter_dict['labels'], label)
128 )
129 if item_matches:
130 return True
131 return False
132
133
134def boot_merge(destination, additions, filters=None):
135 """Complement one `BootImageMapping` with entries from another.
136
137 This adds entries from `additions` (that match `filters`, if given) to
138 `destination`, but only for those image specs for which `destination` does
139 not have entries yet.
140
141 :param destination: `BootImageMapping` to be updated. It will be extended
142 in-place.
143 :param additions: A second `BootImageMapping`, which will be used as a
144 source of additional entries.
145 :param filters: List of dicts, each of which contains 'arch', 'subarch',
146 and 'release' keys. If given, entries are only considered for copying
147 from `additions` to `destination` if they match at least one of the
148 filters. Entries in the filter may be the string `*` (or for entries
149 that are lists, may contain the string `*`) to make them match any
150 value.
151 """
152 for image, resource in additions.items():
153 arch, subarch, release, label = image
154 if image_passes_filter(filters, arch, subarch, release, label):
155 logger.debug(
156 "Merging boot resource for %s/%s/%s/%s.",
157 arch, subarch, release, label)
158 # Do not override an existing entry with the same
159 # arch/subarch/release/label: the first entry found takes
160 # precedence.
161 destination.setdefault(image, resource)
162
163
164def download_image_descriptions(path, keyring=None):
165 """Download image metadata from upstream Simplestreams repo.
166
167 :param path: The path to a Simplestreams repo.
168 :param keyring: Optional keyring for verifying the repo's signatures.
169 :return: A `BootImageMapping` describing available boot resources.
170 """
171 mirror, rpath = path_from_mirror_url(path, None)
172 policy = get_signing_policy(rpath, keyring)
173 reader = UrlMirrorReader(mirror, policy=policy)
174 boot_images_dict = BootImageMapping()
175 dumper = RepoDumper(boot_images_dict)
176 dumper.sync(reader, rpath)
177 return boot_images_dict
178
179
180def download_all_image_descriptions(config):
181 """Download image metadata for all sources in `config`."""
182 boot = BootImageMapping()
183 for source in config['boot']['sources']:
184 repo_boot = download_image_descriptions(
185 source['path'], keyring=source['keyring'])
186 boot_merge(boot, repo_boot, source['selections'])
187 return boot
0188
=== added file 'src/provisioningserver/import_images/helpers.py'
--- src/provisioningserver/import_images/helpers.py 1970-01-01 00:00:00 +0000
+++ src/provisioningserver/import_images/helpers.py 2014-04-08 10:22:55 +0000
@@ -0,0 +1,73 @@
1# Copyright 2014 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).
3
4"""Miscellaneous small definitions in support of boot-resource import."""
5
6from __future__ import (
7 absolute_import,
8 print_function,
9 unicode_literals,
10 )
11
12str = None
13
14__metaclass__ = type
15__all__ = [
16 'get_signing_policy',
17 'ImageSpec',
18 'logger',
19 ]
20
21from collections import namedtuple
22import functools
23import logging
24
25from simplestreams.util import policy_read_signed
26
27# A tuple of the items that together select a boot image.
28ImageSpec = namedtuple(b'ImageSpec', [
29 'arch',
30 'subarch',
31 'release',
32 'label',
33 ])
34
35
36def get_signing_policy(path, keyring=None):
37 """Return Simplestreams signing policy for the given path.
38
39 :param path: Path to the Simplestreams index file.
40 :param keyring: Optional keyring file for verifying signatures.
41 :return: A "signing policy" callable. It accepts a file's content, path,
42 and optional keyring as arguments, and if the signature verifies
43 correctly, returns the content. The keyring defaults to the one you
44 pass.
45 """
46 if path.endswith('.json'):
47 # The configuration deliberately selected an un-signed index. A signed
48 # index would have a suffix of '.sjson'. Use a policy that doesn't
49 # check anything.
50 policy = lambda content, path, keyring: content
51 else:
52 # Otherwise: use default Simplestreams policy for verifying signatures.
53 policy = policy_read_signed
54
55 if keyring is not None:
56 # Pass keyring to the policy, to use if the caller inside Simplestreams
57 # does not provide one.
58 policy = functools.partial(policy, keyring=keyring)
59
60 return policy
61
62
63def init_logger(log_level=logging.INFO):
64 logger = logging.getLogger(__name__)
65 formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
66 handler = logging.StreamHandler()
67 handler.setFormatter(formatter)
68 logger.addHandler(handler)
69 logger.setLevel(log_level)
70 return logger
71
72
73logger = init_logger()
074
=== added file 'src/provisioningserver/import_images/product_mapping.py'
--- src/provisioningserver/import_images/product_mapping.py 1970-01-01 00:00:00 +0000
+++ src/provisioningserver/import_images/product_mapping.py 2014-04-08 10:22:55 +0000
@@ -0,0 +1,65 @@
1# Copyright 2014 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).
3
4"""The `ProductMapping` class."""
5
6from __future__ import (
7 absolute_import,
8 print_function,
9 unicode_literals,
10 )
11
12str = None
13
14__metaclass__ = type
15__all__ = [
16 'ProductMapping',
17 ]
18
19
20class ProductMapping:
21 """Mapping of product data.
22
23 Maps a combination of boot resource metadata (`content_id`, `product_name`,
24 `version_name`) to a list of subarchitectures supported by that boot
25 resource.
26 """
27
28 def __init__(self):
29 self.mapping = {}
30
31 @staticmethod
32 def make_key(resource):
33 """Extract a key tuple from `resource`.
34
35 The key is used for indexing `mapping`.
36
37 :param resource: A dict describing a boot resource. It must contain
38 the keys `content_id`, `product_name`, and `version_name`.
39 :return: A tuple of the resource's content ID, product name, and
40 version name.
41 """
42 return (
43 resource['content_id'],
44 resource['product_name'],
45 resource['version_name'],
46 )
47
48 def add(self, resource, subarch):
49 """Add `subarch` to the list of subarches supported by a boot resource.
50
51 The `resource` is a dict as returned by `products_exdata`. The method
52 will use the values identified by keys `content_id`, `product_name`,
53 and `version_name`.
54 """
55 key = self.make_key(resource)
56 self.mapping.setdefault(key, [])
57 self.mapping[key].append(subarch)
58
59 def contains(self, resource):
60 """Does the dict contain a mapping for the given resource?"""
61 return self.make_key(resource) in self.mapping
62
63 def get(self, resource):
64 """Return the mapped subarchitectures for `resource`."""
65 return self.mapping[self.make_key(resource)]
066
=== added directory 'src/provisioningserver/import_images/testing'
=== added file 'src/provisioningserver/import_images/testing/__init__.py'
=== added file 'src/provisioningserver/import_images/testing/factory.py'
--- src/provisioningserver/import_images/testing/factory.py 1970-01-01 00:00:00 +0000
+++ src/provisioningserver/import_images/testing/factory.py 2014-04-08 10:22:55 +0000
@@ -0,0 +1,56 @@
1# Copyright 2014 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).
3
4"""Factory helpers for the `import_images` package."""
5
6from __future__ import (
7 absolute_import,
8 print_function,
9 unicode_literals,
10 )
11
12str = None
13
14__metaclass__ = type
15__all__ = [
16 'make_boot_resource',
17 'make_image_spec',
18 'set_resource',
19 ]
20
21from maastesting.factory import factory
22from provisioningserver.import_images.boot_image_mapping import (
23 BootImageMapping,
24 )
25from provisioningserver.import_images.helpers import ImageSpec
26
27
28def make_boot_resource():
29 """Create a fake resource dict."""
30 return {
31 'content_id': factory.make_name('content_id'),
32 'product_name': factory.make_name('product_name'),
33 'version_name': factory.make_name('version_name'),
34 }
35
36
37def make_image_spec():
38 """Return an `ImageSpec` with random values."""
39 return ImageSpec(
40 factory.make_name('arch'),
41 factory.make_name('subarch'),
42 factory.make_name('release'),
43 factory.make_name('label'),
44 )
45
46
47def set_resource(boot_dict=None, image_spec=None, resource=None):
48 """Add boot resource to a `BootImageMapping`, creating it if necessary."""
49 if boot_dict is None:
50 boot_dict = BootImageMapping()
51 if image_spec is None:
52 image_spec = make_image_spec()
53 if resource is None:
54 resource = factory.make_name('boot-resource')
55 boot_dict.mapping[image_spec] = resource
56 return boot_dict
057
=== added file 'src/provisioningserver/import_images/tests/test_boot_image_mapping.py'
--- src/provisioningserver/import_images/tests/test_boot_image_mapping.py 1970-01-01 00:00:00 +0000
+++ src/provisioningserver/import_images/tests/test_boot_image_mapping.py 2014-04-08 10:22:55 +0000
@@ -0,0 +1,98 @@
1# Copyright 2014 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).
3
4"""Tests for `BootImageMapping` and its module."""
5
6from __future__ import (
7 absolute_import,
8 print_function,
9 unicode_literals,
10 )
11
12str = None
13
14__metaclass__ = type
15__all__ = []
16
17import json
18
19from maastesting.factory import factory
20from maastesting.testcase import MAASTestCase
21from provisioningserver.import_images.boot_image_mapping import (
22 BootImageMapping,
23 )
24from provisioningserver.import_images.testing.factory import (
25 make_image_spec,
26 set_resource,
27 )
28
29
30class TestBootImageMapping(MAASTestCase):
31 """Tests for `BootImageMapping`."""
32
33 def test_initially_empty(self):
34 self.assertItemsEqual([], BootImageMapping().items())
35
36 def test_items_returns_items(self):
37 image = make_image_spec()
38 resource = factory.make_name('resource')
39 image_dict = set_resource(image_spec=image, resource=resource)
40 self.assertItemsEqual([(image, resource)], image_dict.items())
41
42 def test_setdefault_sets_unset_item(self):
43 image_dict = BootImageMapping()
44 image = make_image_spec()
45 resource = factory.make_name('resource')
46 image_dict.setdefault(image, resource)
47 self.assertItemsEqual([(image, resource)], image_dict.items())
48
49 def test_setdefault_leaves_set_item_unchanged(self):
50 image = make_image_spec()
51 old_resource = factory.make_name('resource')
52 image_dict = set_resource(image_spec=image, resource=old_resource)
53 image_dict.setdefault(image, factory.make_name('newresource'))
54 self.assertItemsEqual([(image, old_resource)], image_dict.items())
55
56 def test_dump_json_is_consistent(self):
57 image = make_image_spec()
58 resource = factory.make_name('resource')
59 image_dict_1 = set_resource(image_spec=image, resource=resource)
60 image_dict_2 = set_resource(image_spec=image, resource=resource)
61 self.assertEqual(image_dict_1.dump_json(), image_dict_2.dump_json())
62
63 def test_dump_json_represents_empty_dict_as_empty_object(self):
64 self.assertEqual('{}', BootImageMapping().dump_json())
65
66 def test_dump_json_represents_entry(self):
67 image = make_image_spec()
68 resource = factory.make_name('resource')
69 image_dict = set_resource(image_spec=image, resource=resource)
70 self.assertEqual(
71 {
72 image.arch: {
73 image.subarch: {
74 image.release: {image.label: resource},
75 },
76 },
77 },
78 json.loads(image_dict.dump_json()))
79
80 def test_dump_json_combines_similar_entries(self):
81 image = make_image_spec()
82 other_release = factory.make_name('other-release')
83 resource1 = factory.make_name('resource')
84 resource2 = factory.make_name('other-resource')
85 image_dict = BootImageMapping()
86 set_resource(image_dict, image, resource1)
87 set_resource(
88 image_dict, image._replace(release=other_release), resource2)
89 self.assertEqual(
90 {
91 image.arch: {
92 image.subarch: {
93 image.release: {image.label: resource1},
94 other_release: {image.label: resource2},
95 },
96 },
97 },
98 json.loads(image_dict.dump_json()))
099
=== modified file 'src/provisioningserver/import_images/tests/test_boot_resources.py'
--- src/provisioningserver/import_images/tests/test_boot_resources.py 2014-04-08 06:00:29 +0000
+++ src/provisioningserver/import_images/tests/test_boot_resources.py 2014-04-08 10:22:55 +0000
@@ -25,14 +25,20 @@
25 )25 )
2626
27from maastesting.factory import factory27from maastesting.factory import factory
28from maastesting.matchers import MockCalledOnceWith
29from maastesting.testcase import MAASTestCase28from maastesting.testcase import MAASTestCase
30import mock29import mock
31from provisioningserver.boot.uefi import UEFIBootMethod30from provisioningserver.boot.uefi import UEFIBootMethod
32from provisioningserver.config import BootConfig31from provisioningserver.config import BootConfig
33from provisioningserver.import_images import boot_resources32from provisioningserver.import_images import boot_resources
33from provisioningserver.import_images.boot_image_mapping import (
34 BootImageMapping,
35 )
36from provisioningserver.import_images.testing.factory import (
37 make_boot_resource,
38 make_image_spec,
39 set_resource,
40 )
34from provisioningserver.utils import write_text_file41from provisioningserver.utils import write_text_file
35from simplestreams.util import SignatureMissingException
36from testtools.content import Content42from testtools.content import Content
37from testtools.content_type import UTF8_TEXT43from testtools.content_type import UTF8_TEXT
38from testtools.matchers import (44from testtools.matchers import (
@@ -42,257 +48,11 @@
42import yaml48import yaml
4349
4450
45def make_image_spec():
46 """Return an `ImageSpec` with random values."""
47 return boot_resources.ImageSpec(
48 factory.make_name('arch'),
49 factory.make_name('subarch'),
50 factory.make_name('release'),
51 factory.make_name('label'),
52 )
53
54
55class TestBootImageMapping(MAASTestCase):
56 """Tests for `BootImageMapping`."""
57
58 def test_initially_empty(self):
59 self.assertItemsEqual([], boot_resources.BootImageMapping().items())
60
61 def test_items_returns_items(self):
62 image = make_image_spec()
63 resource = factory.make_name('resource')
64 image_dict = set_resource(image_spec=image, resource=resource)
65 self.assertItemsEqual([(image, resource)], image_dict.items())
66
67 def test_setdefault_sets_unset_item(self):
68 image_dict = boot_resources.BootImageMapping()
69 image = make_image_spec()
70 resource = factory.make_name('resource')
71 image_dict.setdefault(image, resource)
72 self.assertItemsEqual([(image, resource)], image_dict.items())
73
74 def test_setdefault_leaves_set_item_unchanged(self):
75 image = make_image_spec()
76 old_resource = factory.make_name('resource')
77 image_dict = set_resource(image_spec=image, resource=old_resource)
78 image_dict.setdefault(image, factory.make_name('newresource'))
79 self.assertItemsEqual([(image, old_resource)], image_dict.items())
80
81 def test_dump_json_is_consistent(self):
82 image = make_image_spec()
83 resource = factory.make_name('resource')
84 image_dict_1 = set_resource(image_spec=image, resource=resource)
85 image_dict_2 = set_resource(image_spec=image, resource=resource)
86 self.assertEqual(image_dict_1.dump_json(), image_dict_2.dump_json())
87
88 def test_dump_json_represents_empty_dict_as_empty_object(self):
89 self.assertEqual('{}', boot_resources.BootImageMapping().dump_json())
90
91 def test_dump_json_represents_entry(self):
92 image = make_image_spec()
93 resource = factory.make_name('resource')
94 image_dict = set_resource(image_spec=image, resource=resource)
95 self.assertEqual(
96 {
97 image.arch: {
98 image.subarch: {
99 image.release: {image.label: resource},
100 },
101 },
102 },
103 json.loads(image_dict.dump_json()))
104
105 def test_dump_json_combines_similar_entries(self):
106 image = make_image_spec()
107 other_release = factory.make_name('other-release')
108 resource1 = factory.make_name('resource')
109 resource2 = factory.make_name('other-resource')
110 image_dict = boot_resources.BootImageMapping()
111 set_resource(image_dict, image, resource1)
112 set_resource(
113 image_dict, image._replace(release=other_release), resource2)
114 self.assertEqual(
115 {
116 image.arch: {
117 image.subarch: {
118 image.release: {image.label: resource1},
119 other_release: {image.label: resource2},
120 },
121 },
122 },
123 json.loads(image_dict.dump_json()))
124
125
126class TestValuePassesFilterList(MAASTestCase):
127 """Tests for `value_passes_filter_list`."""
128
129 def test_nothing_passes_empty_list(self):
130 self.assertFalse(
131 boot_resources.value_passes_filter_list(
132 [], factory.make_name('value')))
133
134 def test_unmatched_value_does_not_pass(self):
135 self.assertFalse(
136 boot_resources.value_passes_filter_list(
137 [factory.make_name('filter')], factory.make_name('value')))
138
139 def test_matched_value_passes(self):
140 value = factory.make_name('value')
141 self.assertTrue(
142 boot_resources.value_passes_filter_list([value], value))
143
144 def test_value_passes_if_matched_anywhere_in_filter(self):
145 value = factory.make_name('value')
146 self.assertTrue(
147 boot_resources.value_passes_filter_list(
148 [
149 factory.make_name('filter'),
150 value,
151 factory.make_name('filter'),
152 ],
153 value))
154
155 def test_any_value_passes_asterisk(self):
156 self.assertTrue(
157 boot_resources.value_passes_filter_list(
158 ['*'], factory.make_name('value')))
159
160
161class TestValuePassesFilter(MAASTestCase):
162 """Tests for `value_passes_filter`."""
163
164 def test_unmatched_value_does_not_pass(self):
165 self.assertFalse(
166 boot_resources.value_passes_filter(
167 factory.make_name('filter'), factory.make_name('value')))
168
169 def test_matching_value_passes(self):
170 value = factory.make_name('value')
171 self.assertTrue(boot_resources.value_passes_filter(value, value))
172
173 def test_any_value_matches_asterisk(self):
174 self.assertTrue(
175 boot_resources.value_passes_filter(
176 '*', factory.make_name('value')))
177
178
179def make_boot_resource():
180 """Create a fake resource dict."""
181 return {
182 'content_id': factory.make_name('content_id'),
183 'product_name': factory.make_name('product_name'),
184 'version_name': factory.make_name('version_name'),
185 }
186
187
188class TestProductMapping(MAASTestCase):
189 """Tests for `ProductMapping`."""
190
191 def test_initially_empty(self):
192 self.assertEqual({}, boot_resources.ProductMapping().mapping)
193
194 def test_make_key_extracts_identifying_items(self):
195 resource = make_boot_resource()
196 content_id = resource['content_id']
197 product_name = resource['product_name']
198 version_name = resource['version_name']
199 self.assertEqual(
200 (content_id, product_name, version_name),
201 boot_resources.ProductMapping.make_key(resource))
202
203 def test_make_key_ignores_other_items(self):
204 resource = make_boot_resource()
205 resource['other_item'] = factory.make_name('other')
206 self.assertEqual(
207 (
208 resource['content_id'],
209 resource['product_name'],
210 resource['version_name'],
211 ),
212 boot_resources.ProductMapping.make_key(resource))
213
214 def test_make_key_fails_if_key_missing(self):
215 resource = make_boot_resource()
216 del resource['version_name']
217 self.assertRaises(
218 KeyError,
219 boot_resources.ProductMapping.make_key, resource)
220
221 def test_add_creates_subarches_list_if_needed(self):
222 product_dict = boot_resources.ProductMapping()
223 resource = make_boot_resource()
224 subarch = factory.make_name('subarch')
225 product_dict.add(resource, subarch)
226 self.assertEqual(
227 {product_dict.make_key(resource): [subarch]},
228 product_dict.mapping)
229
230 def test_add_appends_to_existing_list(self):
231 product_dict = boot_resources.ProductMapping()
232 resource = make_boot_resource()
233 subarches = [factory.make_name('subarch') for _ in range(2)]
234 for subarch in subarches:
235 product_dict.add(resource, subarch)
236 self.assertEqual(
237 {product_dict.make_key(resource): subarches},
238 product_dict.mapping)
239
240 def test_contains_returns_true_for_stored_item(self):
241 product_dict = boot_resources.ProductMapping()
242 resource = make_boot_resource()
243 subarch = factory.make_name('subarch')
244 product_dict.add(resource, subarch)
245 self.assertTrue(product_dict.contains(resource))
246
247 def test_contains_returns_false_for_unstored_item(self):
248 self.assertFalse(
249 boot_resources.ProductMapping().contains(make_boot_resource()))
250
251 def test_contains_ignores_similar_items(self):
252 product_dict = boot_resources.ProductMapping()
253 resource = make_boot_resource()
254 subarch = factory.make_name('subarch')
255 product_dict.add(resource.copy(), subarch)
256 resource['product_name'] = factory.make_name('other')
257 self.assertFalse(product_dict.contains(resource))
258
259 def test_contains_ignores_extraneous_keys(self):
260 product_dict = boot_resources.ProductMapping()
261 resource = make_boot_resource()
262 subarch = factory.make_name('subarch')
263 product_dict.add(resource.copy(), subarch)
264 resource['other_item'] = factory.make_name('other')
265 self.assertTrue(product_dict.contains(resource))
266
267 def test_get_returns_stored_item(self):
268 product_dict = boot_resources.ProductMapping()
269 resource = make_boot_resource()
270 subarch = factory.make_name('subarch')
271 product_dict.add(resource, subarch)
272 self.assertEqual([subarch], product_dict.get(resource))
273
274 def test_get_fails_for_unstored_item(self):
275 product_dict = boot_resources.ProductMapping()
276 resource = make_boot_resource()
277 subarch = factory.make_name('subarch')
278 product_dict.add(resource.copy(), subarch)
279 resource['content_id'] = factory.make_name('other')
280 self.assertRaises(KeyError, product_dict.get, resource)
281
282 def test_get_ignores_extraneous_keys(self):
283 product_dict = boot_resources.ProductMapping()
284 resource = make_boot_resource()
285 subarch = factory.make_name('subarch')
286 product_dict.add(resource, subarch)
287 resource['other_item'] = factory.make_name('other')
288 self.assertEqual([subarch], product_dict.get(resource))
289
290
291class TestBootReverse(MAASTestCase):51class TestBootReverse(MAASTestCase):
292 """Tests for `boot_reverse`."""52 """Tests for `boot_reverse`."""
29353
294 def test_maps_empty_dict_to_empty_dict(self):54 def test_maps_empty_dict_to_empty_dict(self):
295 empty_boot_image_dict = boot_resources.BootImageMapping()55 empty_boot_image_dict = BootImageMapping()
296 self.assertEqual(56 self.assertEqual(
297 {},57 {},
298 boot_resources.boot_reverse(empty_boot_image_dict).mapping)58 boot_resources.boot_reverse(empty_boot_image_dict).mapping)
@@ -315,7 +75,7 @@
315 image1 = make_image_spec()75 image1 = make_image_spec()
316 image2 = make_image_spec()76 image2 = make_image_spec()
317 resource = make_boot_resource()77 resource = make_boot_resource()
318 boot_dict = boot_resources.BootImageMapping()78 boot_dict = BootImageMapping()
319 # Create two images in boot_dict, both containing the same resource.79 # Create two images in boot_dict, both containing the same resource.
320 for image in [image1, image2]:80 for image in [image1, image2]:
321 set_resource(81 set_resource(
@@ -334,186 +94,6 @@
334 reverse_dict.get(resource))94 reverse_dict.get(resource))
33595
33696
337class TestImagePassesFilter(MAASTestCase):
338 """Tests for `image_passes_filter`."""
339
340 def make_filter_from_image(self, image_spec=None):
341 """Create a filter dict that matches the given `ImageSpec`.
342
343 If `image_spec` is not given, creates a random value.
344 """
345 if image_spec is None:
346 image_spec = make_image_spec()
347 return {
348 'arches': [image_spec.arch],
349 'subarches': [image_spec.subarch],
350 'release': image_spec.release,
351 'labels': [image_spec.label],
352 }
353
354 def test_any_image_passes_none_filter(self):
355 arch, subarch, release, label = make_image_spec()
356 self.assertTrue(
357 boot_resources.image_passes_filter(
358 None, arch, subarch, release, label))
359
360 def test_any_image_passes_empty_filter(self):
361 arch, subarch, release, label = make_image_spec()
362 self.assertTrue(
363 boot_resources.image_passes_filter(
364 [], arch, subarch, release, label))
365
366 def test_image_passes_matching_filter(self):
367 image = make_image_spec()
368 self.assertTrue(
369 boot_resources.image_passes_filter(
370 [self.make_filter_from_image(image)],
371 image.arch, image.subarch, image.release, image.label))
372
373 def test_image_does_not_pass_nonmatching_filter(self):
374 image = make_image_spec()
375 self.assertFalse(
376 boot_resources.image_passes_filter(
377 [self.make_filter_from_image()],
378 image.arch, image.subarch, image.release, image.label))
379
380 def test_image_passes_if_one_filter_matches(self):
381 image = make_image_spec()
382 self.assertTrue(
383 boot_resources.image_passes_filter(
384 [
385 self.make_filter_from_image(),
386 self.make_filter_from_image(image),
387 self.make_filter_from_image(),
388 ], image.arch, image.subarch, image.release, image.label))
389
390 def test_filter_checks_release(self):
391 image = make_image_spec()
392 self.assertFalse(
393 boot_resources.image_passes_filter(
394 [
395 self.make_filter_from_image(image._replace(
396 release=factory.make_name('other-release')))
397 ], image.arch, image.subarch, image.release, image.label))
398
399 def test_filter_checks_arches(self):
400 image = make_image_spec()
401 self.assertFalse(
402 boot_resources.image_passes_filter(
403 [
404 self.make_filter_from_image(image._replace(
405 arch=factory.make_name('other-arch')))
406 ], image.arch, image.subarch, image.release, image.label))
407
408 def test_filter_checks_subarches(self):
409 image = make_image_spec()
410 self.assertFalse(
411 boot_resources.image_passes_filter(
412 [
413 self.make_filter_from_image(image._replace(
414 subarch=factory.make_name('other-subarch')))
415 ], image.arch, image.subarch, image.release, image.label))
416
417 def test_filter_checks_labels(self):
418 image = make_image_spec()
419 self.assertFalse(
420 boot_resources.image_passes_filter(
421 [
422 self.make_filter_from_image(image._replace(
423 label=factory.make_name('other-label')))
424 ], image.arch, image.subarch, image.release, image.label))
425
426
427def set_resource(boot_dict=None, image_spec=None, resource=None):
428 """Add boot resource to a `BootImageMapping`, creating it if necessary."""
429 if boot_dict is None:
430 boot_dict = boot_resources.BootImageMapping()
431 if image_spec is None:
432 image_spec = make_image_spec()
433 if resource is None:
434 resource = factory.make_name('boot-resource')
435 boot_dict.mapping[image_spec] = resource
436 return boot_dict
437
438
439class TestBootMerge(MAASTestCase):
440 """Tests for `boot_merge`."""
441
442 def test_integrates(self):
443 # End-to-end scenario for boot_merge: start with an empty boot
444 # resources dict, and receive one resource from Simplestreams.
445 total_resources = boot_resources.BootImageMapping()
446 resources_from_repo = set_resource()
447 boot_resources.boot_merge(total_resources, resources_from_repo)
448 # Since we started with an empty dict, the result contains the same
449 # item that we got from Simplestreams, and nothing else.
450 self.assertEqual(resources_from_repo.mapping, total_resources.mapping)
451
452 def test_obeys_filters(self):
453 filters = [
454 {
455 'arches': [factory.make_name('other-arch')],
456 'subarches': [factory.make_name('other-subarch')],
457 'release': factory.make_name('other-release'),
458 'label': [factory.make_name('other-label')],
459 },
460 ]
461 total_resources = boot_resources.BootImageMapping()
462 resources_from_repo = set_resource()
463 boot_resources.boot_merge(
464 total_resources, resources_from_repo, filters=filters)
465 self.assertEqual({}, total_resources.mapping)
466
467 def test_does_not_overwrite_existing_entry(self):
468 image = make_image_spec()
469 total_resources = set_resource(
470 resource="Original resource", image_spec=image)
471 original_resources = total_resources.mapping.copy()
472 resources_from_repo = set_resource(
473 resource="New resource", image_spec=image)
474 boot_resources.boot_merge(total_resources, resources_from_repo)
475 self.assertEqual(original_resources, total_resources.mapping)
476
477
478class TestGetSigningPolicy(MAASTestCase):
479 """Tests for `get_signing_policy`."""
480
481 def test_picks_nonchecking_policy_for_json_index(self):
482 path = 'streams/v1/index.json'
483 policy = boot_resources.get_signing_policy(path)
484 content = factory.getRandomString()
485 self.assertEqual(
486 content,
487 policy(content, path, factory.make_name('keyring')))
488
489 def test_picks_checking_policy_for_sjson_index(self):
490 path = 'streams/v1/index.sjson'
491 content = factory.getRandomString()
492 policy = boot_resources.get_signing_policy(path)
493 self.assertRaises(
494 SignatureMissingException,
495 policy, content, path, factory.make_name('keyring'))
496
497 def test_picks_checking_policy_for_json_gpg_index(self):
498 path = 'streams/v1/index.json.gpg'
499 content = factory.getRandomString()
500 policy = boot_resources.get_signing_policy(path)
501 self.assertRaises(
502 SignatureMissingException,
503 policy, content, path, factory.make_name('keyring'))
504
505 def test_injects_default_keyring_if_passed(self):
506 path = 'streams/v1/index.json.gpg'
507 content = factory.getRandomString()
508 keyring = factory.make_name('keyring')
509 self.patch(boot_resources, 'policy_read_signed')
510 policy = boot_resources.get_signing_policy(path, keyring)
511 policy(content, path)
512 self.assertThat(
513 boot_resources.policy_read_signed,
514 MockCalledOnceWith(mock.ANY, mock.ANY, keyring=keyring))
515
516
517class TestTgtEntry(MAASTestCase):97class TestTgtEntry(MAASTestCase):
518 """Tests for `tgt_entry`."""98 """Tests for `tgt_entry`."""
51999
520100
=== added file 'src/provisioningserver/import_images/tests/test_download_descriptions.py'
--- src/provisioningserver/import_images/tests/test_download_descriptions.py 1970-01-01 00:00:00 +0000
+++ src/provisioningserver/import_images/tests/test_download_descriptions.py 2014-04-08 10:22:55 +0000
@@ -0,0 +1,209 @@
1# Copyright 2014 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).
3
4"""Tests for the `download_descriptions` module."""
5
6from __future__ import (
7 absolute_import,
8 print_function,
9 unicode_literals,
10 )
11
12str = None
13
14__metaclass__ = type
15__all__ = []
16
17from maastesting.factory import factory
18from maastesting.testcase import MAASTestCase
19from provisioningserver.import_images import download_descriptions
20from provisioningserver.import_images.boot_image_mapping import (
21 BootImageMapping,
22 )
23from provisioningserver.import_images.testing.factory import (
24 make_image_spec,
25 set_resource,
26 )
27
28
29class TestValuePassesFilterList(MAASTestCase):
30 """Tests for `value_passes_filter_list`."""
31
32 def test_nothing_passes_empty_list(self):
33 self.assertFalse(
34 download_descriptions.value_passes_filter_list(
35 [], factory.make_name('value')))
36
37 def test_unmatched_value_does_not_pass(self):
38 self.assertFalse(
39 download_descriptions.value_passes_filter_list(
40 [factory.make_name('filter')], factory.make_name('value')))
41
42 def test_matched_value_passes(self):
43 value = factory.make_name('value')
44 self.assertTrue(
45 download_descriptions.value_passes_filter_list([value], value))
46
47 def test_value_passes_if_matched_anywhere_in_filter(self):
48 value = factory.make_name('value')
49 self.assertTrue(
50 download_descriptions.value_passes_filter_list(
51 [
52 factory.make_name('filter'),
53 value,
54 factory.make_name('filter'),
55 ],
56 value))
57
58 def test_any_value_passes_asterisk(self):
59 self.assertTrue(
60 download_descriptions.value_passes_filter_list(
61 ['*'], factory.make_name('value')))
62
63
64class TestValuePassesFilter(MAASTestCase):
65 """Tests for `value_passes_filter`."""
66
67 def test_unmatched_value_does_not_pass(self):
68 self.assertFalse(
69 download_descriptions.value_passes_filter(
70 factory.make_name('filter'), factory.make_name('value')))
71
72 def test_matching_value_passes(self):
73 value = factory.make_name('value')
74 self.assertTrue(
75 download_descriptions.value_passes_filter(value, value))
76
77 def test_any_value_matches_asterisk(self):
78 self.assertTrue(
79 download_descriptions.value_passes_filter(
80 '*', factory.make_name('value')))
81
82
83class TestImagePassesFilter(MAASTestCase):
84 """Tests for `image_passes_filter`."""
85
86 def make_filter_from_image(self, image_spec=None):
87 """Create a filter dict that matches the given `ImageSpec`.
88
89 If `image_spec` is not given, creates a random value.
90 """
91 if image_spec is None:
92 image_spec = make_image_spec()
93 return {
94 'arches': [image_spec.arch],
95 'subarches': [image_spec.subarch],
96 'release': image_spec.release,
97 'labels': [image_spec.label],
98 }
99
100 def test_any_image_passes_none_filter(self):
101 arch, subarch, release, label = make_image_spec()
102 self.assertTrue(
103 download_descriptions.image_passes_filter(
104 None, arch, subarch, release, label))
105
106 def test_any_image_passes_empty_filter(self):
107 arch, subarch, release, label = make_image_spec()
108 self.assertTrue(
109 download_descriptions.image_passes_filter(
110 [], arch, subarch, release, label))
111
112 def test_image_passes_matching_filter(self):
113 image = make_image_spec()
114 self.assertTrue(
115 download_descriptions.image_passes_filter(
116 [self.make_filter_from_image(image)],
117 image.arch, image.subarch, image.release, image.label))
118
119 def test_image_does_not_pass_nonmatching_filter(self):
120 image = make_image_spec()
121 self.assertFalse(
122 download_descriptions.image_passes_filter(
123 [self.make_filter_from_image()],
124 image.arch, image.subarch, image.release, image.label))
125
126 def test_image_passes_if_one_filter_matches(self):
127 image = make_image_spec()
128 self.assertTrue(
129 download_descriptions.image_passes_filter(
130 [
131 self.make_filter_from_image(),
132 self.make_filter_from_image(image),
133 self.make_filter_from_image(),
134 ], image.arch, image.subarch, image.release, image.label))
135
136 def test_filter_checks_release(self):
137 image = make_image_spec()
138 self.assertFalse(
139 download_descriptions.image_passes_filter(
140 [
141 self.make_filter_from_image(image._replace(
142 release=factory.make_name('other-release')))
143 ], image.arch, image.subarch, image.release, image.label))
144
145 def test_filter_checks_arches(self):
146 image = make_image_spec()
147 self.assertFalse(
148 download_descriptions.image_passes_filter(
149 [
150 self.make_filter_from_image(image._replace(
151 arch=factory.make_name('other-arch')))
152 ], image.arch, image.subarch, image.release, image.label))
153
154 def test_filter_checks_subarches(self):
155 image = make_image_spec()
156 self.assertFalse(
157 download_descriptions.image_passes_filter(
158 [
159 self.make_filter_from_image(image._replace(
160 subarch=factory.make_name('other-subarch')))
161 ], image.arch, image.subarch, image.release, image.label))
162
163 def test_filter_checks_labels(self):
164 image = make_image_spec()
165 self.assertFalse(
166 download_descriptions.image_passes_filter(
167 [
168 self.make_filter_from_image(image._replace(
169 label=factory.make_name('other-label')))
170 ], image.arch, image.subarch, image.release, image.label))
171
172
173class TestBootMerge(MAASTestCase):
174 """Tests for `boot_merge`."""
175
176 def test_integrates(self):
177 # End-to-end scenario for boot_merge: start with an empty boot
178 # resources dict, and receive one resource from Simplestreams.
179 total_resources = BootImageMapping()
180 resources_from_repo = set_resource()
181 download_descriptions.boot_merge(total_resources, resources_from_repo)
182 # Since we started with an empty dict, the result contains the same
183 # item that we got from Simplestreams, and nothing else.
184 self.assertEqual(resources_from_repo.mapping, total_resources.mapping)
185
186 def test_obeys_filters(self):
187 filters = [
188 {
189 'arches': [factory.make_name('other-arch')],
190 'subarches': [factory.make_name('other-subarch')],
191 'release': factory.make_name('other-release'),
192 'label': [factory.make_name('other-label')],
193 },
194 ]
195 total_resources = BootImageMapping()
196 resources_from_repo = set_resource()
197 download_descriptions.boot_merge(
198 total_resources, resources_from_repo, filters=filters)
199 self.assertEqual({}, total_resources.mapping)
200
201 def test_does_not_overwrite_existing_entry(self):
202 image = make_image_spec()
203 total_resources = set_resource(
204 resource="Original resource", image_spec=image)
205 original_resources = total_resources.mapping.copy()
206 resources_from_repo = set_resource(
207 resource="New resource", image_spec=image)
208 download_descriptions.boot_merge(total_resources, resources_from_repo)
209 self.assertEqual(original_resources, total_resources.mapping)
0210
=== added file 'src/provisioningserver/import_images/tests/test_helpers.py'
--- src/provisioningserver/import_images/tests/test_helpers.py 1970-01-01 00:00:00 +0000
+++ src/provisioningserver/import_images/tests/test_helpers.py 2014-04-08 10:22:55 +0000
@@ -0,0 +1,61 @@
1# Copyright 2014 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).
3
4"""Tests for the `helpers` module."""
5
6from __future__ import (
7 absolute_import,
8 print_function,
9 unicode_literals,
10 )
11
12str = None
13
14__metaclass__ = type
15__all__ = []
16
17from maastesting.factory import factory
18from maastesting.matchers import MockCalledOnceWith
19from maastesting.testcase import MAASTestCase
20import mock
21from provisioningserver.import_images import helpers
22from simplestreams.util import SignatureMissingException
23
24
25class TestGetSigningPolicy(MAASTestCase):
26 """Tests for `get_signing_policy`."""
27
28 def test_picks_nonchecking_policy_for_json_index(self):
29 path = 'streams/v1/index.json'
30 policy = helpers.get_signing_policy(path)
31 content = factory.getRandomString()
32 self.assertEqual(
33 content,
34 policy(content, path, factory.make_name('keyring')))
35
36 def test_picks_checking_policy_for_sjson_index(self):
37 path = 'streams/v1/index.sjson'
38 content = factory.getRandomString()
39 policy = helpers.get_signing_policy(path)
40 self.assertRaises(
41 SignatureMissingException,
42 policy, content, path, factory.make_name('keyring'))
43
44 def test_picks_checking_policy_for_json_gpg_index(self):
45 path = 'streams/v1/index.json.gpg'
46 content = factory.getRandomString()
47 policy = helpers.get_signing_policy(path)
48 self.assertRaises(
49 SignatureMissingException,
50 policy, content, path, factory.make_name('keyring'))
51
52 def test_injects_default_keyring_if_passed(self):
53 path = 'streams/v1/index.json.gpg'
54 content = factory.getRandomString()
55 keyring = factory.make_name('keyring')
56 self.patch(helpers, 'policy_read_signed')
57 policy = helpers.get_signing_policy(path, keyring)
58 policy(content, path)
59 self.assertThat(
60 helpers.policy_read_signed,
61 MockCalledOnceWith(mock.ANY, mock.ANY, keyring=keyring))
062
=== added file 'src/provisioningserver/import_images/tests/test_product_mapping.py'
--- src/provisioningserver/import_images/tests/test_product_mapping.py 1970-01-01 00:00:00 +0000
+++ src/provisioningserver/import_images/tests/test_product_mapping.py 2014-04-08 10:22:55 +0000
@@ -0,0 +1,125 @@
1# Copyright 2014 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).
3
4"""Tests for the `ProductMapping` class."""
5
6from __future__ import (
7 absolute_import,
8 print_function,
9 unicode_literals,
10 )
11
12str = None
13
14__metaclass__ = type
15__all__ = []
16
17from maastesting.factory import factory
18from maastesting.testcase import MAASTestCase
19from provisioningserver.import_images.product_mapping import ProductMapping
20from provisioningserver.import_images.testing.factory import (
21 make_boot_resource,
22 )
23
24
25class TestProductMapping(MAASTestCase):
26 """Tests for `ProductMapping`."""
27
28 def test_initially_empty(self):
29 self.assertEqual({}, ProductMapping().mapping)
30
31 def test_make_key_extracts_identifying_items(self):
32 resource = make_boot_resource()
33 content_id = resource['content_id']
34 product_name = resource['product_name']
35 version_name = resource['version_name']
36 self.assertEqual(
37 (content_id, product_name, version_name),
38 ProductMapping.make_key(resource))
39
40 def test_make_key_ignores_other_items(self):
41 resource = make_boot_resource()
42 resource['other_item'] = factory.make_name('other')
43 self.assertEqual(
44 (
45 resource['content_id'],
46 resource['product_name'],
47 resource['version_name'],
48 ),
49 ProductMapping.make_key(resource))
50
51 def test_make_key_fails_if_key_missing(self):
52 resource = make_boot_resource()
53 del resource['version_name']
54 self.assertRaises(
55 KeyError,
56 ProductMapping.make_key, resource)
57
58 def test_add_creates_subarches_list_if_needed(self):
59 product_dict = ProductMapping()
60 resource = make_boot_resource()
61 subarch = factory.make_name('subarch')
62 product_dict.add(resource, subarch)
63 self.assertEqual(
64 {product_dict.make_key(resource): [subarch]},
65 product_dict.mapping)
66
67 def test_add_appends_to_existing_list(self):
68 product_dict = ProductMapping()
69 resource = make_boot_resource()
70 subarches = [factory.make_name('subarch') for _ in range(2)]
71 for subarch in subarches:
72 product_dict.add(resource, subarch)
73 self.assertEqual(
74 {product_dict.make_key(resource): subarches},
75 product_dict.mapping)
76
77 def test_contains_returns_true_for_stored_item(self):
78 product_dict = ProductMapping()
79 resource = make_boot_resource()
80 subarch = factory.make_name('subarch')
81 product_dict.add(resource, subarch)
82 self.assertTrue(product_dict.contains(resource))
83
84 def test_contains_returns_false_for_unstored_item(self):
85 self.assertFalse(
86 ProductMapping().contains(make_boot_resource()))
87
88 def test_contains_ignores_similar_items(self):
89 product_dict = ProductMapping()
90 resource = make_boot_resource()
91 subarch = factory.make_name('subarch')
92 product_dict.add(resource.copy(), subarch)
93 resource['product_name'] = factory.make_name('other')
94 self.assertFalse(product_dict.contains(resource))
95
96 def test_contains_ignores_extraneous_keys(self):
97 product_dict = ProductMapping()
98 resource = make_boot_resource()
99 subarch = factory.make_name('subarch')
100 product_dict.add(resource.copy(), subarch)
101 resource['other_item'] = factory.make_name('other')
102 self.assertTrue(product_dict.contains(resource))
103
104 def test_get_returns_stored_item(self):
105 product_dict = ProductMapping()
106 resource = make_boot_resource()
107 subarch = factory.make_name('subarch')
108 product_dict.add(resource, subarch)
109 self.assertEqual([subarch], product_dict.get(resource))
110
111 def test_get_fails_for_unstored_item(self):
112 product_dict = ProductMapping()
113 resource = make_boot_resource()
114 subarch = factory.make_name('subarch')
115 product_dict.add(resource.copy(), subarch)
116 resource['content_id'] = factory.make_name('other')
117 self.assertRaises(KeyError, product_dict.get, resource)
118
119 def test_get_ignores_extraneous_keys(self):
120 product_dict = ProductMapping()
121 resource = make_boot_resource()
122 subarch = factory.make_name('subarch')
123 product_dict.add(resource, subarch)
124 resource['other_item'] = factory.make_name('other')
125 self.assertEqual([subarch], product_dict.get(resource))