Merge lp:~jtv/maas/split-boot_resources into lp:~maas-committers/maas/trunk

[Jeroen T. Vermeulen]

Break the boot_resources.py file up into smaller parts, for manageability.

This may look like a big branch, but it isn't, really. The code is moved, but nothing else is changed (except available_boot_resources: being unused and untested, it was removed).

Lots of things were going on in this file, including two Simplestreams synchronisations. Now that we have proper classes for some of the additional components, and a clearer idea of what RepoDumper does and how it fits into the big picture, we can split things up. It is particularly gratifying that RepoDumper itself is now entirely local to its module. A lot of cognitive load is relieved by simply not needing to know about this class for maintenance that doesn't involve its specific function.

Jeroen

[Graham Binns (gmb)]

I love these branches that are smaller than they look. Gives me a sense of achievement for not much cost :).

review: Approve

[MAAS Lander (maas-lander)]

Attempt to merge into lp:maas failed due to conflicts:

text conflict in src/provisioningserver/import_images/tests/test_boot_resources.py

[Jeroen T. Vermeulen (jtv)]

I think that may just have been a silly little conflict between an import that I removed in this branch, but also ran through the formatter in a concurrent lint branch.

=== added file 'src/provisioningserver/import_images/boot_image_mapping.py'
--- src/provisioningserver/import_images/boot_image_mapping.py	1970-01-01 00:00:00 +0000
+++ src/provisioningserver/import_images/boot_image_mapping.py	2014-04-08 10:22:55 +0000
@@ -0,0 +1,61 @@
+# Copyright 2014 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""The `BootImageMapping` class."""
+
+from __future__ import (
+    absolute_import,
+    print_function,
+    unicode_literals,
+    )
+
+str = None
+
+__metaclass__ = type
+__all__ = [
+    'BootImageMapping',
+    ]
+
+import json
+
+from provisioningserver.import_images.helpers import ImageSpec
+
+
+class BootImageMapping:
+    """Mapping of boot-image data.
+
+    Maps `ImageSpec` tuples to metadata for Simplestreams products.
+
+    This class is deliberately a bit more restrictive and less ad-hoc than a
+    dict.  It helps keep a clear view of the data structures in this module.
+    """
+
+    def __init__(self):
+        self.mapping = {}
+
+    def items(self):
+        """Iterate over `ImageSpec` keys, and their stored values."""
+        for image_spec, item in sorted(self.mapping.items()):
+            yield image_spec, item
+
+    def setdefault(self, image_spec, item):
+        """Set metadata for `image_spec` to item, if not already set."""
+        assert isinstance(image_spec, ImageSpec)
+        self.mapping.setdefault(image_spec, item)
+
+    def dump_json(self):
+        """Produce JSON representing the mapped boot images.
+
+        Tries to keep the output deterministic, so that identical data is
+        likely to produce identical JSON.
+        """
+        # The meta files represent the mapping as a nested hierarchy of dicts.
+        # Keep that format.
+        data = {}
+        for image, resource in self.items():
+            arch, subarch, release, label = image
+            data.setdefault(arch, {})
+            data[arch].setdefault(subarch, {})
+            data[arch][subarch].setdefault(release, {})
+            data[arch][subarch][release][label] = resource
+        return json.dumps(data, sort_keys=True)

=== modified file 'src/provisioningserver/import_images/boot_resources.py'
--- src/provisioningserver/import_images/boot_resources.py	2014-04-08 05:25:24 +0000
+++ src/provisioningserver/import_images/boot_resources.py	2014-04-08 10:22:55 +0000
@@ -12,26 +12,27 @@
 __metaclass__ = type
 __all__ = [
     'main',
-    'available_boot_resources',
     'make_arg_parser',
     ]
 
 from argparse import ArgumentParser
-from collections import namedtuple
 from datetime import datetime
 import errno
-import functools
-import glob
 from gzip import GzipFile
-import json
-import logging
-from logging import getLogger
 import os
 from textwrap import dedent
 
 from provisioningserver.boot import BootMethodRegistry
 from provisioningserver.boot.tftppath import list_boot_images
 from provisioningserver.config import BootConfig
+from provisioningserver.import_images.download_descriptions import (
+    download_all_image_descriptions,
+    )
+from provisioningserver.import_images.helpers import (
+    get_signing_policy,
+    logger,
+    )
+from provisioningserver.import_images.product_mapping import ProductMapping
 from provisioningserver.utils import (
     atomic_write,
     call_and_check,
@@ -47,200 +48,14 @@
 from simplestreams.util import (
     item_checksums,
     path_from_mirror_url,
-    policy_read_signed,
     products_exdata,
     )
 
 
-def init_logger(log_level=logging.INFO):
-    logger = getLogger(__name__)
-    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
-    handler = logging.StreamHandler()
-    handler.setFormatter(formatter)
-    logger.addHandler(handler)
-    logger.setLevel(log_level)
-    return logger
-
-
-logger = init_logger()
-
-
 class NoConfigFile(Exception):
     """Raised when the config file for the script doesn't exist."""
 
 
-# A tuple of the items that together select a boot image.
-ImageSpec = namedtuple(b'ImageSpec', [
-    'arch',
-    'subarch',
-    'release',
-    'label',
-    ])
-
-
-class BootImageMapping:
-    """Mapping of boot-image data.
-
-    Maps `ImageSpec` tuples to metadata for Simplestreams products.
-
-    This class is deliberately a bit more restrictive and less ad-hoc than a
-    dict.  It helps keep a clear view of the data structures in this module.
-    """
-
-    def __init__(self):
-        self.mapping = {}
-
-    def items(self):
-        """Iterate over `ImageSpec` keys, and their stored values."""
-        for image_spec, item in sorted(self.mapping.items()):
-            yield image_spec, item
-
-    def setdefault(self, image_spec, item):
-        """Set metadata for `image_spec` to item, if not already set."""
-        assert isinstance(image_spec, ImageSpec)
-        self.mapping.setdefault(image_spec, item)
-
-    def dump_json(self):
-        """Produce JSON representing the mapped boot images.
-
-        Tries to keep the output deterministic, so that identical data is
-        likely to produce identical JSON.
-        """
-        # The meta files represent the mapping as a nested hierarchy of dicts.
-        # Keep that format.
-        data = {}
-        for image, resource in self.items():
-            arch, subarch, release, label = image
-            data.setdefault(arch, {})
-            data[arch].setdefault(subarch, {})
-            data[arch][subarch].setdefault(release, {})
-            data[arch][subarch][release][label] = resource
-        return json.dumps(data, sort_keys=True)
-
-
-def value_passes_filter_list(filter_list, property_value):
-    """Does the given property of a boot image pass the given filter list?
-
-    The value passes if either it matches one of the entries in the list of
-    filter values, or one of the filter values is an asterisk (`*`).
-    """
-    return '*' in filter_list or property_value in filter_list
-
-
-def value_passes_filter(filter_value, property_value):
-    """Does the given property of a boot image pass the given filter?
-
-    The value passes the filter if either the filter value is an asterisk
-    (`*`) or the value is equal to the filter value.
-    """
-    return filter_value in ('*', property_value)
-
-
-def image_passes_filter(filters, arch, subarch, release, label):
-    """Filter a boot image against configured import filters.
-
-    :param filters: A list of dicts describing the filters, as in `boot_merge`.
-        If the list is empty, or `None`, any image matches.  Any entry in a
-        filter may be a string containing just an asterisk (`*`) to denote that
-        the entry will match any value.
-    :param arch: The given boot image's architecture.
-    :param subarch: The given boot image's subarchitecture.
-    :param release: The given boot image's OS release.
-    :param label: The given boot image's label.
-    :return: Whether the image matches any of the dicts in `filters`.
-    """
-    if filters is None or len(filters) == 0:
-        return True
-    for filter_dict in filters:
-        item_matches = (
-            value_passes_filter(filter_dict['release'], release) and
-            value_passes_filter_list(filter_dict['arches'], arch) and
-            value_passes_filter_list(filter_dict['subarches'], subarch) and
-            value_passes_filter_list(filter_dict['labels'], label)
-        )
-        if item_matches:
-            return True
-    return False
-
-
-def boot_merge(destination, additions, filters=None):
-    """Complement one `BootImageMapping` with entries from another.
-
-    This adds entries from `additions` (that match `filters`, if given) to
-    `destination`, but only for those image specs for which `destination` does
-    not have entries yet.
-
-    :param destination: `BootImageMapping` to be updated.  It will be extended
-        in-place.
-    :param additions: A second `BootImageMapping`, which will be used as a
-        source of additional entries.
-    :param filters: List of dicts, each of which contains 'arch', 'subarch',
-        and 'release' keys.  If given, entries are only considered for copying
-        from `additions` to `destination` if they match at least one of the
-        filters.  Entries in the filter may be the string `*` (or for entries
-        that are lists, may contain the string `*`) to make them match any
-        value.
-    """
-    for image, resource in additions.items():
-        arch, subarch, release, label = image
-        if image_passes_filter(filters, arch, subarch, release, label):
-            logger.debug(
-                "Merging boot resource for %s/%s/%s/%s.",
-                arch, subarch, release, label)
-            # Do not override an existing entry with the same
-            # arch/subarch/release/label: the first entry found takes
-            # precedence.
-            destination.setdefault(image, resource)
-
-
-class ProductMapping:
-    """Mapping of product data.
-
-    Maps a combination of boot resource metadata (`content_id`, `product_name`,
-    `version_name`) to a list of subarchitectures supported by that boot
-    resource.
-    """
-
-    def __init__(self):
-        self.mapping = {}
-
-    @staticmethod
-    def make_key(resource):
-        """Extract a key tuple from `resource`.
-
-        The key is used for indexing `mapping`.
-
-        :param resource: A dict describing a boot resource.  It must contain
-            the keys `content_id`, `product_name`, and `version_name`.
-        :return: A tuple of the resource's content ID, product name, and
-            version name.
-        """
-        return (
-            resource['content_id'],
-            resource['product_name'],
-            resource['version_name'],
-            )
-
-    def add(self, resource, subarch):
-        """Add `subarch` to the list of subarches supported by a boot resource.
-
-        The `resource` is a dict as returned by `products_exdata`.  The method
-        will use the values identified by keys `content_id`, `product_name`,
-        and `version_name`.
-        """
-        key = self.make_key(resource)
-        self.mapping.setdefault(key, [])
-        self.mapping[key].append(subarch)
-
-    def contains(self, resource):
-        """Does the dict contain a mapping for the given resource?"""
-        return self.make_key(resource) in self.mapping
-
-    def get(self, resource):
-        """Return the mapped subarchitectures for `resource`."""
-        return self.mapping[self.make_key(resource)]
-
-
 def boot_reverse(boot_images_dict):
     """Determine the subarches supported by each boot resource.
 
@@ -299,106 +114,6 @@
     return entry
 
 
-def get_signing_policy(path, keyring=None):
-    """Return Simplestreams signing policy for the given path.
-
-    :param path: Path to the Simplestreams index file.
-    :param keyring: Optional keyring file for verifying signatures.
-    :return: A "signing policy" callable.  It accepts a file's content, path,
-        and optional keyring as arguments, and if the signature verifies
-        correctly, returns the content.  The keyring defaults to the one you
-        pass.
-    """
-    if path.endswith('.json'):
-        # The configuration deliberately selected an un-signed index.  A signed
-        # index would have a suffix of '.sjson'.  Use a policy that doesn't
-        # check anything.
-        policy = lambda content, path, keyring: content
-    else:
-        # Otherwise: use default Simplestreams policy for verifying signatures.
-        policy = policy_read_signed
-
-    if keyring is not None:
-        # Pass keyring to the policy, to use if the caller inside Simplestreams
-        # does not provide one.
-        policy = functools.partial(policy, keyring=keyring)
-
-    return policy
-
-
-def clean_up_repo_item(item):
-    """Return a subset of dict `item` for storing in a boot images dict."""
-    keys_to_keep = ['content_id', 'product_name', 'version_name', 'path']
-    compact_item = {key: item[key] for key in keys_to_keep}
-    return compact_item
-
-
-class RepoDumper(BasicMirrorWriter):
-    """Gather metadata about boot images available in a Simplestreams repo.
-
-    Used inside `download_image_descriptions`.  Stores basic metadata about
-    each image it finds upstream in a given `BootImageMapping`.  Each stored
-    item is a dict containing the basic metadata for retrieving a boot image.
-
-    Simplestreams' `BasicMirrorWriter` in itself is stateless.  It relies on
-    a subclass (such as this one) to store data.
-
-    :ivar boot_images_dict: A `BootImageMapping`.  Image metadata will be
-        stored here as it is discovered.  Simplestreams does not interact with
-        this variable.
-    """
-
-    def __init__(self, boot_images_dict):
-        super(RepoDumper, self).__init__()
-        self.boot_images_dict = boot_images_dict
-
-    def load_products(self, path=None, content_id=None):
-        """Overridable from `BasicMirrorWriter`."""
-        # It looks as if this method only makes sense for MirrorReaders, not
-        # for MirrorWriters.  The default MirrorWriter implementation just
-        # raises NotImplementedError.  Stop it from doing that.
-        return
-
-    def insert_item(self, data, src, target, pedigree, contentsource):
-        """Overridable from `BasicMirrorWriter`."""
-        item = products_exdata(src, pedigree)
-        arch, subarches = item['arch'], item['subarches']
-        release = item['release']
-        label = item['label']
-        base_image = ImageSpec(arch, None, release, label)
-        compact_item = clean_up_repo_item(item)
-        for subarch in subarches.split(','):
-            self.boot_images_dict.setdefault(
-                base_image._replace(subarch=subarch), compact_item)
-
-
-def download_image_descriptions(path, keyring=None):
-    """Download image metadata from upstream Simplestreams repo.
-
-    :param path: The path to a Simplestreams repo.
-    :param keyring: Optional keyring for verifying the repo's signatures.
-    :return: A nested dict of data, indexed by image arch, subarch, release,
-        and label.
-    """
-    mirror, rpath = path_from_mirror_url(path, None)
-    policy = get_signing_policy(rpath, keyring)
-    reader = UrlMirrorReader(mirror, policy=policy)
-    boot_images_dict = BootImageMapping()
-    dumper = RepoDumper(boot_images_dict)
-    dumper.sync(reader, rpath)
-    return boot_images_dict
-
-
-def download_all_image_descriptions(config):
-    """Download image metadata for all sources in `config`."""
-    boot = BootImageMapping()
-    for source in config['boot']['sources']:
-        repo_boot = download_image_descriptions(
-            source['path'], keyring=source['keyring'])
-        boot_merge(boot, repo_boot, source['selections'])
-    return boot
-
-
 class RepoWriter(BasicMirrorWriter):
     """Download boot resources from an upstream Simplestreams repo.
 
@@ -474,12 +189,6 @@
                 os.link(src, link_path)
 
 
-def available_boot_resources(root):
-    for resource_path in glob.glob(os.path.join(root, '*/*/*/*')):
-        arch, subarch, release, label = resource_path.split('/')[-4:]
-        yield (arch, subarch, release, label)
-
-
 def install_boot_loaders(destination):
     """Install the all the required file from each bootloader method.
     :param destination: Directory where the loaders should be stored.

=== added file 'src/provisioningserver/import_images/download_descriptions.py'
--- src/provisioningserver/import_images/download_descriptions.py	1970-01-01 00:00:00 +0000
+++ src/provisioningserver/import_images/download_descriptions.py	2014-04-08 10:22:55 +0000
@@ -0,0 +1,187 @@
+# Copyright 2014 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Download boot resource descriptions from Simplestreams repo.
+
+This module is responsible only for syncing the repo's metadata, not the boot
+resources themselves.  The two are handled in separate Simplestreams
+synchronisation stages.
+"""
+
+from __future__ import (
+    absolute_import,
+    print_function,
+    unicode_literals,
+    )
+
+str = None
+
+__metaclass__ = type
+__all__ = [
+    'download_all_image_descriptions',
+    ]
+
+
+from provisioningserver.import_images.boot_image_mapping import (
+    BootImageMapping,
+    )
+from provisioningserver.import_images.helpers import (
+    get_signing_policy,
+    ImageSpec,
+    logger,
+    )
+from simplestreams.mirrors import (
+    BasicMirrorWriter,
+    UrlMirrorReader,
+    )
+from simplestreams.util import (
+    path_from_mirror_url,
+    products_exdata,
+    )
+
+
+def clean_up_repo_item(item):
+    """Return a subset of dict `item` for storing in a boot images dict."""
+    keys_to_keep = ['content_id', 'product_name', 'version_name', 'path']
+    compact_item = {key: item[key] for key in keys_to_keep}
+    return compact_item
+
+
+class RepoDumper(BasicMirrorWriter):
+    """Gather metadata about boot images available in a Simplestreams repo.
+
+    Used inside `download_image_descriptions`.  Stores basic metadata about
+    each image it finds upstream in a given `BootImageMapping`.  Each stored
+    item is a dict containing the basic metadata for retrieving a boot image.
+
+    Simplestreams' `BasicMirrorWriter` in itself is stateless.  It relies on
+    a subclass (such as this one) to store data.
+
+    :ivar boot_images_dict: A `BootImageMapping`.  Image metadata will be
+        stored here as it is discovered.  Simplestreams does not interact with
+        this variable.
+    """
+
+    def __init__(self, boot_images_dict):
+        super(RepoDumper, self).__init__()
+        self.boot_images_dict = boot_images_dict
+
+    def load_products(self, path=None, content_id=None):
+        """Overridable from `BasicMirrorWriter`."""
+        # It looks as if this method only makes sense for MirrorReaders, not
+        # for MirrorWriters.  The default MirrorWriter implementation just
+        # raises NotImplementedError.  Stop it from doing that.
+        return
+
+    def insert_item(self, data, src, target, pedigree, contentsource):
+        """Overridable from `BasicMirrorWriter`."""
+        item = products_exdata(src, pedigree)
+        arch, subarches = item['arch'], item['subarches']
+        release = item['release']
+        label = item['label']
+        base_image = ImageSpec(arch, None, release, label)
+        compact_item = clean_up_repo_item(item)
+        for subarch in subarches.split(','):
+            self.boot_images_dict.setdefault(
+                base_image._replace(subarch=subarch), compact_item)
+
+
+def value_passes_filter_list(filter_list, property_value):
+    """Does the given property of a boot image pass the given filter list?
+
+    The value passes if either it matches one of the entries in the list of
+    filter values, or one of the filter values is an asterisk (`*`).
+    """
+    return '*' in filter_list or property_value in filter_list
+
+
+def value_passes_filter(filter_value, property_value):
+    """Does the given property of a boot image pass the given filter?
+
+    The value passes the filter if either the filter value is an asterisk
+    (`*`) or the value is equal to the filter value.
+    """
+    return filter_value in ('*', property_value)
+
+
+def image_passes_filter(filters, arch, subarch, release, label):
+    """Filter a boot image against configured import filters.
+
+    :param filters: A list of dicts describing the filters, as in `boot_merge`.
+        If the list is empty, or `None`, any image matches.  Any entry in a
+        filter may be a string containing just an asterisk (`*`) to denote that
+        the entry will match any value.
+    :param arch: The given boot image's architecture.
+    :param subarch: The given boot image's subarchitecture.
+    :param release: The given boot image's OS release.
+    :param label: The given boot image's label.
+    :return: Whether the image matches any of the dicts in `filters`.
+    """
+    if filters is None or len(filters) == 0:
+        return True
+    for filter_dict in filters:
+        item_matches = (
+            value_passes_filter(filter_dict['release'], release) and
+            value_passes_filter_list(filter_dict['arches'], arch) and
+            value_passes_filter_list(filter_dict['subarches'], subarch) and
+            value_passes_filter_list(filter_dict['labels'], label)
+        )
+        if item_matches:
+            return True
+    return False
+
+
+def boot_merge(destination, additions, filters=None):
+    """Complement one `BootImageMapping` with entries from another.
+
+    This adds entries from `additions` (that match `filters`, if given) to
+    `destination`, but only for those image specs for which `destination` does
+    not have entries yet.
+
+    :param destination: `BootImageMapping` to be updated.  It will be extended
+        in-place.
+    :param additions: A second `BootImageMapping`, which will be used as a
+        source of additional entries.
+    :param filters: List of dicts, each of which contains 'arch', 'subarch',
+        and 'release' keys.  If given, entries are only considered for copying
+        from `additions` to `destination` if they match at least one of the
+        filters.  Entries in the filter may be the string `*` (or for entries
+        that are lists, may contain the string `*`) to make them match any
+        value.
+    """
+    for image, resource in additions.items():
+        arch, subarch, release, label = image
+        if image_passes_filter(filters, arch, subarch, release, label):
+            logger.debug(
+                "Merging boot resource for %s/%s/%s/%s.",
+                arch, subarch, release, label)
+            # Do not override an existing entry with the same
+            # arch/subarch/release/label: the first entry found takes
+            # precedence.
+            destination.setdefault(image, resource)
+
+
+def download_image_descriptions(path, keyring=None):
+    """Download image metadata from upstream Simplestreams repo.
+
+    :param path: The path to a Simplestreams repo.
+    :param keyring: Optional keyring for verifying the repo's signatures.
+    :return: A `BootImageMapping` describing available boot resources.
+    """
+    mirror, rpath = path_from_mirror_url(path, None)
+    policy = get_signing_policy(rpath, keyring)
+    reader = UrlMirrorReader(mirror, policy=policy)
+    boot_images_dict = BootImageMapping()
+    dumper = RepoDumper(boot_images_dict)
+    dumper.sync(reader, rpath)
+    return boot_images_dict
+
+
+def download_all_image_descriptions(config):
+    """Download image metadata for all sources in `config`."""
+    boot = BootImageMapping()
+    for source in config['boot']['sources']:
+        repo_boot = download_image_descriptions(
+            source['path'], keyring=source['keyring'])
+        boot_merge(boot, repo_boot, source['selections'])
+    return boot

=== added file 'src/provisioningserver/import_images/helpers.py'
--- src/provisioningserver/import_images/helpers.py	1970-01-01 00:00:00 +0000
+++ src/provisioningserver/import_images/helpers.py	2014-04-08 10:22:55 +0000
@@ -0,0 +1,73 @@
+# Copyright 2014 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Miscellaneous small definitions in support of boot-resource import."""
+
+from __future__ import (
+    absolute_import,
+    print_function,
+    unicode_literals,
+    )
+
+str = None
+
+__metaclass__ = type
+__all__ = [
+    'get_signing_policy',
+    'ImageSpec',
+    'logger',
+    ]
+
+from collections import namedtuple
+import functools
+import logging
+
+from simplestreams.util import policy_read_signed
+
+# A tuple of the items that together select a boot image.
+ImageSpec = namedtuple(b'ImageSpec', [
+    'arch',
+    'subarch',
+    'release',
+    'label',
+    ])
+
+
+def get_signing_policy(path, keyring=None):
+    """Return Simplestreams signing policy for the given path.
+
+    :param path: Path to the Simplestreams index file.
+    :param keyring: Optional keyring file for verifying signatures.
+    :return: A "signing policy" callable.  It accepts a file's content, path,
+        and optional keyring as arguments, and if the signature verifies
+        correctly, returns the content.  The keyring defaults to the one you
+        pass.
+    """
+    if path.endswith('.json'):
+        # The configuration deliberately selected an un-signed index.  A signed
+        # index would have a suffix of '.sjson'.  Use a policy that doesn't
+        # check anything.
+        policy = lambda content, path, keyring: content
+    else:
+        # Otherwise: use default Simplestreams policy for verifying signatures.
+        policy = policy_read_signed
+
+    if keyring is not None:
+        # Pass keyring to the policy, to use if the caller inside Simplestreams
+        # does not provide one.
+        policy = functools.partial(policy, keyring=keyring)
+
+    return policy
+
+
+def init_logger(log_level=logging.INFO):
+    logger = logging.getLogger(__name__)
+    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+    logger.setLevel(log_level)
+    return logger
+
+
+logger = init_logger()

=== added file 'src/provisioningserver/import_images/product_mapping.py'
--- src/provisioningserver/import_images/product_mapping.py	1970-01-01 00:00:00 +0000
+++ src/provisioningserver/import_images/product_mapping.py	2014-04-08 10:22:55 +0000
@@ -0,0 +1,65 @@
+# Copyright 2014 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""The `ProductMapping` class."""
+
+from __future__ import (
+    absolute_import,
+    print_function,
+    unicode_literals,
+    )
+
+str = None
+
+__metaclass__ = type
+__all__ = [
+    'ProductMapping',
+    ]
+
+
+class ProductMapping:
+    """Mapping of product data.
+
+    Maps a combination of boot resource metadata (`content_id`, `product_name`,
+    `version_name`) to a list of subarchitectures supported by that boot
+    resource.
+    """
+
+    def __init__(self):
+        self.mapping = {}
+
+    @staticmethod
+    def make_key(resource):
+        """Extract a key tuple from `resource`.
+
+        The key is used for indexing `mapping`.
+
+        :param resource: A dict describing a boot resource.  It must contain
+            the keys `content_id`, `product_name`, and `version_name`.
+        :return: A tuple of the resource's content ID, product name, and
+            version name.
+        """
+        return (
+            resource['content_id'],
+            resource['product_name'],
+            resource['version_name'],
+            )
+
+    def add(self, resource, subarch):
+        """Add `subarch` to the list of subarches supported by a boot resource.
+
+        The `resource` is a dict as returned by `products_exdata`.  The method
+        will use the values identified by keys `content_id`, `product_name`,
+        and `version_name`.
+        """
+        key = self.make_key(resource)
+        self.mapping.setdefault(key, [])
+        self.mapping[key].append(subarch)
+
+    def contains(self, resource):
+        """Does the dict contain a mapping for the given resource?"""
+        return self.make_key(resource) in self.mapping
+
+    def get(self, resource):
+        """Return the mapped subarchitectures for `resource`."""
+        return self.mapping[self.make_key(resource)]

=== added directory 'src/provisioningserver/import_images/testing'
=== added file 'src/provisioningserver/import_images/testing/__init__.py'
=== added file 'src/provisioningserver/import_images/testing/factory.py'
--- src/provisioningserver/import_images/testing/factory.py	1970-01-01 00:00:00 +0000
+++ src/provisioningserver/import_images/testing/factory.py	2014-04-08 10:22:55 +0000
@@ -0,0 +1,56 @@
+# Copyright 2014 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Factory helpers for the `import_images` package."""
+
+from __future__ import (
+    absolute_import,
+    print_function,
+    unicode_literals,
+    )
+
+str = None
+
+__metaclass__ = type
+__all__ = [
+    'make_boot_resource',
+    'make_image_spec',
+    'set_resource',
+    ]
+
+from maastesting.factory import factory
+from provisioningserver.import_images.boot_image_mapping import (
+    BootImageMapping,
+    )
+from provisioningserver.import_images.helpers import ImageSpec
+
+
+def make_boot_resource():
+    """Create a fake resource dict."""
+    return {
+        'content_id': factory.make_name('content_id'),
+        'product_name': factory.make_name('product_name'),
+        'version_name': factory.make_name('version_name'),
+        }
+
+
+def make_image_spec():
+    """Return an `ImageSpec` with random values."""
+    return ImageSpec(
+        factory.make_name('arch'),
+        factory.make_name('subarch'),
+        factory.make_name('release'),
+        factory.make_name('label'),
+        )
+
+
+def set_resource(boot_dict=None, image_spec=None, resource=None):
+    """Add boot resource to a `BootImageMapping`, creating it if necessary."""
+    if boot_dict is None:
+        boot_dict = BootImageMapping()
+    if image_spec is None:
+        image_spec = make_image_spec()
+    if resource is None:
+        resource = factory.make_name('boot-resource')
+    boot_dict.mapping[image_spec] = resource
+    return boot_dict

=== added file 'src/provisioningserver/import_images/tests/test_boot_image_mapping.py'
--- src/provisioningserver/import_images/tests/test_boot_image_mapping.py	1970-01-01 00:00:00 +0000
+++ src/provisioningserver/import_images/tests/test_boot_image_mapping.py	2014-04-08 10:22:55 +0000
@@ -0,0 +1,98 @@
+# Copyright 2014 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Tests for `BootImageMapping` and its module."""
+
+from __future__ import (
+    absolute_import,
+    print_function,
+    unicode_literals,
+    )
+
+str = None
+
+__metaclass__ = type
+__all__ = []
+
+import json
+
+from maastesting.factory import factory
+from maastesting.testcase import MAASTestCase
+from provisioningserver.import_images.boot_image_mapping import (
+    BootImageMapping,
+    )
+from provisioningserver.import_images.testing.factory import (
+    make_image_spec,
+    set_resource,
+    )
+
+
+class TestBootImageMapping(MAASTestCase):
+    """Tests for `BootImageMapping`."""
+
+    def test_initially_empty(self):
+        self.assertItemsEqual([], BootImageMapping().items())
+
+    def test_items_returns_items(self):
+        image = make_image_spec()
+        resource = factory.make_name('resource')
+        image_dict = set_resource(image_spec=image, resource=resource)
+        self.assertItemsEqual([(image, resource)], image_dict.items())
+
+    def test_setdefault_sets_unset_item(self):
+        image_dict = BootImageMapping()
+        image = make_image_spec()
+        resource = factory.make_name('resource')
+        image_dict.setdefault(image, resource)
+        self.assertItemsEqual([(image, resource)], image_dict.items())
+
+    def test_setdefault_leaves_set_item_unchanged(self):
+        image = make_image_spec()
+        old_resource = factory.make_name('resource')
+        image_dict = set_resource(image_spec=image, resource=old_resource)
+        image_dict.setdefault(image, factory.make_name('newresource'))
+        self.assertItemsEqual([(image, old_resource)], image_dict.items())
+
+    def test_dump_json_is_consistent(self):
+        image = make_image_spec()
+        resource = factory.make_name('resource')
+        image_dict_1 = set_resource(image_spec=image, resource=resource)
+        image_dict_2 = set_resource(image_spec=image, resource=resource)
+        self.assertEqual(image_dict_1.dump_json(), image_dict_2.dump_json())
+
+    def test_dump_json_represents_empty_dict_as_empty_object(self):
+        self.assertEqual('{}', BootImageMapping().dump_json())
+
+    def test_dump_json_represents_entry(self):
+        image = make_image_spec()
+        resource = factory.make_name('resource')
+        image_dict = set_resource(image_spec=image, resource=resource)
+        self.assertEqual(
+            {
+                image.arch: {
+                    image.subarch: {
+                        image.release: {image.label: resource},
+                    },
+                },
+            },
+            json.loads(image_dict.dump_json()))
+
+    def test_dump_json_combines_similar_entries(self):
+        image = make_image_spec()
+        other_release = factory.make_name('other-release')
+        resource1 = factory.make_name('resource')
+        resource2 = factory.make_name('other-resource')
+        image_dict = BootImageMapping()
+        set_resource(image_dict, image, resource1)
+        set_resource(
+            image_dict, image._replace(release=other_release), resource2)
+        self.assertEqual(
+            {
+                image.arch: {
+                    image.subarch: {
+                        image.release: {image.label: resource1},
+                        other_release: {image.label: resource2},
+                    },
+                },
+            },
+            json.loads(image_dict.dump_json()))

=== modified file 'src/provisioningserver/import_images/tests/test_boot_resources.py'
--- src/provisioningserver/import_images/tests/test_boot_resources.py	2014-04-08 06:00:29 +0000
+++ src/provisioningserver/import_images/tests/test_boot_resources.py	2014-04-08 10:22:55 +0000
@@ -25,14 +25,20 @@
     )
 
 from maastesting.factory import factory
-from maastesting.matchers import MockCalledOnceWith
 from maastesting.testcase import MAASTestCase
 import mock
 from provisioningserver.boot.uefi import UEFIBootMethod
 from provisioningserver.config import BootConfig
 from provisioningserver.import_images import boot_resources
+from provisioningserver.import_images.boot_image_mapping import (
+    BootImageMapping,
+    )
+from provisioningserver.import_images.testing.factory import (
+    make_boot_resource,
+    make_image_spec,
+    set_resource,
+    )
 from provisioningserver.utils import write_text_file
-from simplestreams.util import SignatureMissingException
 from testtools.content import Content
 from testtools.content_type import UTF8_TEXT
 from testtools.matchers import (
@@ -42,257 +48,11 @@
 import yaml
 
 
-def make_image_spec():
-    """Return an `ImageSpec` with random values."""
-    return boot_resources.ImageSpec(
-        factory.make_name('arch'),
-        factory.make_name('subarch'),
-        factory.make_name('release'),
-        factory.make_name('label'),
-        )
-
-
-class TestBootImageMapping(MAASTestCase):
-    """Tests for `BootImageMapping`."""
-
-    def test_initially_empty(self):
-        self.assertItemsEqual([], boot_resources.BootImageMapping().items())
-
-    def test_items_returns_items(self):
-        image = make_image_spec()
-        resource = factory.make_name('resource')
-        image_dict = set_resource(image_spec=image, resource=resource)
-        self.assertItemsEqual([(image, resource)], image_dict.items())
-
-    def test_setdefault_sets_unset_item(self):
-        image_dict = boot_resources.BootImageMapping()
-        image = make_image_spec()
-        resource = factory.make_name('resource')
-        image_dict.setdefault(image, resource)
-        self.assertItemsEqual([(image, resource)], image_dict.items())
-
-    def test_setdefault_leaves_set_item_unchanged(self):
-        image = make_image_spec()
-        old_resource = factory.make_name('resource')
-        image_dict = set_resource(image_spec=image, resource=old_resource)
-        image_dict.setdefault(image, factory.make_name('newresource'))
-        self.assertItemsEqual([(image, old_resource)], image_dict.items())
-
-    def test_dump_json_is_consistent(self):
-        image = make_image_spec()
-        resource = factory.make_name('resource')
-        image_dict_1 = set_resource(image_spec=image, resource=resource)
-        image_dict_2 = set_resource(image_spec=image, resource=resource)
-        self.assertEqual(image_dict_1.dump_json(), image_dict_2.dump_json())
-
-    def test_dump_json_represents_empty_dict_as_empty_object(self):
-        self.assertEqual('{}', boot_resources.BootImageMapping().dump_json())
-
-    def test_dump_json_represents_entry(self):
-        image = make_image_spec()
-        resource = factory.make_name('resource')
-        image_dict = set_resource(image_spec=image, resource=resource)
-        self.assertEqual(
-            {
-                image.arch: {
-                    image.subarch: {
-                        image.release: {image.label: resource},
-                    },
-                },
-            },
-            json.loads(image_dict.dump_json()))
-
-    def test_dump_json_combines_similar_entries(self):
-        image = make_image_spec()
-        other_release = factory.make_name('other-release')
-        resource1 = factory.make_name('resource')
-        resource2 = factory.make_name('other-resource')
-        image_dict = boot_resources.BootImageMapping()
-        set_resource(image_dict, image, resource1)
-        set_resource(
-            image_dict, image._replace(release=other_release), resource2)
-        self.assertEqual(
-            {
-                image.arch: {
-                    image.subarch: {
-                        image.release: {image.label: resource1},
-                        other_release: {image.label: resource2},
-                    },
-                },
-            },
-            json.loads(image_dict.dump_json()))
-
-
-class TestValuePassesFilterList(MAASTestCase):
-    """Tests for `value_passes_filter_list`."""
-
-    def test_nothing_passes_empty_list(self):
-        self.assertFalse(
-            boot_resources.value_passes_filter_list(
-                [], factory.make_name('value')))
-
-    def test_unmatched_value_does_not_pass(self):
-        self.assertFalse(
-            boot_resources.value_passes_filter_list(
-                [factory.make_name('filter')], factory.make_name('value')))
-
-    def test_matched_value_passes(self):
-        value = factory.make_name('value')
-        self.assertTrue(
-            boot_resources.value_passes_filter_list([value], value))
-
-    def test_value_passes_if_matched_anywhere_in_filter(self):
-        value = factory.make_name('value')
-        self.assertTrue(
-            boot_resources.value_passes_filter_list(
-                [
-                    factory.make_name('filter'),
-                    value,
-                    factory.make_name('filter'),
-                ],
-                value))
-
-    def test_any_value_passes_asterisk(self):
-        self.assertTrue(
-            boot_resources.value_passes_filter_list(
-                ['*'], factory.make_name('value')))
-
-
-class TestValuePassesFilter(MAASTestCase):
-    """Tests for `value_passes_filter`."""
-
-    def test_unmatched_value_does_not_pass(self):
-        self.assertFalse(
-            boot_resources.value_passes_filter(
-                factory.make_name('filter'), factory.make_name('value')))
-
-    def test_matching_value_passes(self):
-        value = factory.make_name('value')
-        self.assertTrue(boot_resources.value_passes_filter(value, value))
-
-    def test_any_value_matches_asterisk(self):
-        self.assertTrue(
-            boot_resources.value_passes_filter(
-                '*', factory.make_name('value')))
-
-
-def make_boot_resource():
-    """Create a fake resource dict."""
-    return {
-        'content_id': factory.make_name('content_id'),
-        'product_name': factory.make_name('product_name'),
-        'version_name': factory.make_name('version_name'),
-        }
-
-
-class TestProductMapping(MAASTestCase):
-    """Tests for `ProductMapping`."""
-
-    def test_initially_empty(self):
-        self.assertEqual({}, boot_resources.ProductMapping().mapping)
-
-    def test_make_key_extracts_identifying_items(self):
-        resource = make_boot_resource()
-        content_id = resource['content_id']
-        product_name = resource['product_name']
-        version_name = resource['version_name']
-        self.assertEqual(
-            (content_id, product_name, version_name),
-            boot_resources.ProductMapping.make_key(resource))
-
-    def test_make_key_ignores_other_items(self):
-        resource = make_boot_resource()
-        resource['other_item'] = factory.make_name('other')
-        self.assertEqual(
-            (
-                resource['content_id'],
-                resource['product_name'],
-                resource['version_name'],
-            ),
-            boot_resources.ProductMapping.make_key(resource))
-
-    def test_make_key_fails_if_key_missing(self):
-        resource = make_boot_resource()
-        del resource['version_name']
-        self.assertRaises(
-            KeyError,
-            boot_resources.ProductMapping.make_key, resource)
-
-    def test_add_creates_subarches_list_if_needed(self):
-        product_dict = boot_resources.ProductMapping()
-        resource = make_boot_resource()
-        subarch = factory.make_name('subarch')
-        product_dict.add(resource, subarch)
-        self.assertEqual(
-            {product_dict.make_key(resource): [subarch]},
-            product_dict.mapping)
-
-    def test_add_appends_to_existing_list(self):
-        product_dict = boot_resources.ProductMapping()
-        resource = make_boot_resource()
-        subarches = [factory.make_name('subarch') for _ in range(2)]
-        for subarch in subarches:
-            product_dict.add(resource, subarch)
-        self.assertEqual(
-            {product_dict.make_key(resource): subarches},
-            product_dict.mapping)
-
-    def test_contains_returns_true_for_stored_item(self):
-        product_dict = boot_resources.ProductMapping()
-        resource = make_boot_resource()
-        subarch = factory.make_name('subarch')
-        product_dict.add(resource, subarch)
-        self.assertTrue(product_dict.contains(resource))
-
-    def test_contains_returns_false_for_unstored_item(self):
-        self.assertFalse(
-            boot_resources.ProductMapping().contains(make_boot_resource()))
-
-    def test_contains_ignores_similar_items(self):
-        product_dict = boot_resources.ProductMapping()
-        resource = make_boot_resource()
-        subarch = factory.make_name('subarch')
-        product_dict.add(resource.copy(), subarch)
-        resource['product_name'] = factory.make_name('other')
-        self.assertFalse(product_dict.contains(resource))
-
-    def test_contains_ignores_extraneous_keys(self):
-        product_dict = boot_resources.ProductMapping()
-        resource = make_boot_resource()
-        subarch = factory.make_name('subarch')
-        product_dict.add(resource.copy(), subarch)
-        resource['other_item'] = factory.make_name('other')
-        self.assertTrue(product_dict.contains(resource))
-
-    def test_get_returns_stored_item(self):
-        product_dict = boot_resources.ProductMapping()
-        resource = make_boot_resource()
-        subarch = factory.make_name('subarch')
-        product_dict.add(resource, subarch)
-        self.assertEqual([subarch], product_dict.get(resource))
-
-    def test_get_fails_for_unstored_item(self):
-        product_dict = boot_resources.ProductMapping()
-        resource = make_boot_resource()
-        subarch = factory.make_name('subarch')
-        product_dict.add(resource.copy(), subarch)
-        resource['content_id'] = factory.make_name('other')
-        self.assertRaises(KeyError, product_dict.get, resource)
-
-    def test_get_ignores_extraneous_keys(self):
-        product_dict = boot_resources.ProductMapping()
-        resource = make_boot_resource()
-        subarch = factory.make_name('subarch')
-        product_dict.add(resource, subarch)
-        resource['other_item'] = factory.make_name('other')
-        self.assertEqual([subarch], product_dict.get(resource))
-
-
 class TestBootReverse(MAASTestCase):
     """Tests for `boot_reverse`."""
 
     def test_maps_empty_dict_to_empty_dict(self):
-        empty_boot_image_dict = boot_resources.BootImageMapping()
+        empty_boot_image_dict = BootImageMapping()
         self.assertEqual(
             {},
             boot_resources.boot_reverse(empty_boot_image_dict).mapping)
@@ -315,7 +75,7 @@
         image1 = make_image_spec()
         image2 = make_image_spec()
         resource = make_boot_resource()
-        boot_dict = boot_resources.BootImageMapping()
+        boot_dict = BootImageMapping()
         # Create two images in boot_dict, both containing the same resource.
         for image in [image1, image2]:
             set_resource(
@@ -334,186 +94,6 @@
             reverse_dict.get(resource))
 
 
-class TestImagePassesFilter(MAASTestCase):
-    """Tests for `image_passes_filter`."""
-
-    def make_filter_from_image(self, image_spec=None):
-        """Create a filter dict that matches the given `ImageSpec`.
-
-        If `image_spec` is not given, creates a random value.
-        """
-        if image_spec is None:
-            image_spec = make_image_spec()
-        return {
-            'arches': [image_spec.arch],
-            'subarches': [image_spec.subarch],
-            'release': image_spec.release,
-            'labels': [image_spec.label],
-            }
-
-    def test_any_image_passes_none_filter(self):
-        arch, subarch, release, label = make_image_spec()
-        self.assertTrue(
-            boot_resources.image_passes_filter(
-                None, arch, subarch, release, label))
-
-    def test_any_image_passes_empty_filter(self):
-        arch, subarch, release, label = make_image_spec()
-        self.assertTrue(
-            boot_resources.image_passes_filter(
-                [], arch, subarch, release, label))
-
-    def test_image_passes_matching_filter(self):
-        image = make_image_spec()
-        self.assertTrue(
-            boot_resources.image_passes_filter(
-                [self.make_filter_from_image(image)],
-                image.arch, image.subarch, image.release, image.label))
-
-    def test_image_does_not_pass_nonmatching_filter(self):
-        image = make_image_spec()
-        self.assertFalse(
-            boot_resources.image_passes_filter(
-                [self.make_filter_from_image()],
-                image.arch, image.subarch, image.release, image.label))
-
-    def test_image_passes_if_one_filter_matches(self):
-        image = make_image_spec()
-        self.assertTrue(
-            boot_resources.image_passes_filter(
-                [
-                    self.make_filter_from_image(),
-                    self.make_filter_from_image(image),
-                    self.make_filter_from_image(),
-                ], image.arch, image.subarch, image.release, image.label))
-
-    def test_filter_checks_release(self):
-        image = make_image_spec()
-        self.assertFalse(
-            boot_resources.image_passes_filter(
-                [
-                    self.make_filter_from_image(image._replace(
-                        release=factory.make_name('other-release')))
-                ], image.arch, image.subarch, image.release, image.label))
-
-    def test_filter_checks_arches(self):
-        image = make_image_spec()
-        self.assertFalse(
-            boot_resources.image_passes_filter(
-                [
-                    self.make_filter_from_image(image._replace(
-                        arch=factory.make_name('other-arch')))
-                ], image.arch, image.subarch, image.release, image.label))
-
-    def test_filter_checks_subarches(self):
-        image = make_image_spec()
-        self.assertFalse(
-            boot_resources.image_passes_filter(
-                [
-                    self.make_filter_from_image(image._replace(
-                        subarch=factory.make_name('other-subarch')))
-                ], image.arch, image.subarch, image.release, image.label))
-
-    def test_filter_checks_labels(self):
-        image = make_image_spec()
-        self.assertFalse(
-            boot_resources.image_passes_filter(
-                [
-                    self.make_filter_from_image(image._replace(
-                        label=factory.make_name('other-label')))
-                ], image.arch, image.subarch, image.release, image.label))
-
-
-def set_resource(boot_dict=None, image_spec=None, resource=None):
-    """Add boot resource to a `BootImageMapping`, creating it if necessary."""
-    if boot_dict is None:
-        boot_dict = boot_resources.BootImageMapping()
-    if image_spec is None:
-        image_spec = make_image_spec()
-    if resource is None:
-        resource = factory.make_name('boot-resource')
-    boot_dict.mapping[image_spec] = resource
-    return boot_dict
-
-
-class TestBootMerge(MAASTestCase):
-    """Tests for `boot_merge`."""
-
-    def test_integrates(self):
-        # End-to-end scenario for boot_merge: start with an empty boot
-        # resources dict, and receive one resource from Simplestreams.
-        total_resources = boot_resources.BootImageMapping()
-        resources_from_repo = set_resource()
-        boot_resources.boot_merge(total_resources, resources_from_repo)
-        # Since we started with an empty dict, the result contains the same
-        # item that we got from Simplestreams, and nothing else.
-        self.assertEqual(resources_from_repo.mapping, total_resources.mapping)
-
-    def test_obeys_filters(self):
-        filters = [
-            {
-                'arches': [factory.make_name('other-arch')],
-                'subarches': [factory.make_name('other-subarch')],
-                'release': factory.make_name('other-release'),
-                'label': [factory.make_name('other-label')],
-            },
-            ]
-        total_resources = boot_resources.BootImageMapping()
-        resources_from_repo = set_resource()
-        boot_resources.boot_merge(
-            total_resources, resources_from_repo, filters=filters)
-        self.assertEqual({}, total_resources.mapping)
-
-    def test_does_not_overwrite_existing_entry(self):
-        image = make_image_spec()
-        total_resources = set_resource(
-            resource="Original resource", image_spec=image)
-        original_resources = total_resources.mapping.copy()
-        resources_from_repo = set_resource(
-            resource="New resource", image_spec=image)
-        boot_resources.boot_merge(total_resources, resources_from_repo)
-        self.assertEqual(original_resources, total_resources.mapping)
-
-
-class TestGetSigningPolicy(MAASTestCase):
-    """Tests for `get_signing_policy`."""
-
-    def test_picks_nonchecking_policy_for_json_index(self):
-        path = 'streams/v1/index.json'
-        policy = boot_resources.get_signing_policy(path)
-        content = factory.getRandomString()
-        self.assertEqual(
-            content,
-            policy(content, path, factory.make_name('keyring')))
-
-    def test_picks_checking_policy_for_sjson_index(self):
-        path = 'streams/v1/index.sjson'
-        content = factory.getRandomString()
-        policy = boot_resources.get_signing_policy(path)
-        self.assertRaises(
-            SignatureMissingException,
-            policy, content, path, factory.make_name('keyring'))
-
-    def test_picks_checking_policy_for_json_gpg_index(self):
-        path = 'streams/v1/index.json.gpg'
-        content = factory.getRandomString()
-        policy = boot_resources.get_signing_policy(path)
-        self.assertRaises(
-            SignatureMissingException,
-            policy, content, path, factory.make_name('keyring'))
-
-    def test_injects_default_keyring_if_passed(self):
-        path = 'streams/v1/index.json.gpg'
-        content = factory.getRandomString()
-        keyring = factory.make_name('keyring')
-        self.patch(boot_resources, 'policy_read_signed')
-        policy = boot_resources.get_signing_policy(path, keyring)
-        policy(content, path)
-        self.assertThat(
-            boot_resources.policy_read_signed,
-            MockCalledOnceWith(mock.ANY, mock.ANY, keyring=keyring))
-
-
 class TestTgtEntry(MAASTestCase):
     """Tests for `tgt_entry`."""
 

=== added file 'src/provisioningserver/import_images/tests/test_download_descriptions.py'
--- src/provisioningserver/import_images/tests/test_download_descriptions.py	1970-01-01 00:00:00 +0000
+++ src/provisioningserver/import_images/tests/test_download_descriptions.py	2014-04-08 10:22:55 +0000
@@ -0,0 +1,209 @@
+# Copyright 2014 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Tests for the `download_descriptions` module."""
+
+from __future__ import (
+    absolute_import,
+    print_function,
+    unicode_literals,
+    )
+
+str = None
+
+__metaclass__ = type
+__all__ = []
+
+from maastesting.factory import factory
+from maastesting.testcase import MAASTestCase
+from provisioningserver.import_images import download_descriptions
+from provisioningserver.import_images.boot_image_mapping import (
+    BootImageMapping,
+    )
+from provisioningserver.import_images.testing.factory import (
+    make_image_spec,
+    set_resource,
+    )
+
+
+class TestValuePassesFilterList(MAASTestCase):
+    """Tests for `value_passes_filter_list`."""
+
+    def test_nothing_passes_empty_list(self):
+        self.assertFalse(
+            download_descriptions.value_passes_filter_list(
+                [], factory.make_name('value')))
+
+    def test_unmatched_value_does_not_pass(self):
+        self.assertFalse(
+            download_descriptions.value_passes_filter_list(
+                [factory.make_name('filter')], factory.make_name('value')))
+
+    def test_matched_value_passes(self):
+        value = factory.make_name('value')
+        self.assertTrue(
+            download_descriptions.value_passes_filter_list([value], value))
+
+    def test_value_passes_if_matched_anywhere_in_filter(self):
+        value = factory.make_name('value')
+        self.assertTrue(
+            download_descriptions.value_passes_filter_list(
+                [
+                    factory.make_name('filter'),
+                    value,
+                    factory.make_name('filter'),
+                ],
+                value))
+
+    def test_any_value_passes_asterisk(self):
+        self.assertTrue(
+            download_descriptions.value_passes_filter_list(
+                ['*'], factory.make_name('value')))
+
+
+class TestValuePassesFilter(MAASTestCase):
+    """Tests for `value_passes_filter`."""
+
+    def test_unmatched_value_does_not_pass(self):
+        self.assertFalse(
+            download_descriptions.value_passes_filter(
+                factory.make_name('filter'), factory.make_name('value')))
+
+    def test_matching_value_passes(self):
+        value = factory.make_name('value')
+        self.assertTrue(
+            download_descriptions.value_passes_filter(value, value))
+
+    def test_any_value_matches_asterisk(self):
+        self.assertTrue(
+            download_descriptions.value_passes_filter(
+                '*', factory.make_name('value')))
+
+
+class TestImagePassesFilter(MAASTestCase):
+    """Tests for `image_passes_filter`."""
+
+    def make_filter_from_image(self, image_spec=None):
+        """Create a filter dict that matches the given `ImageSpec`.
+
+        If `image_spec` is not given, creates a random value.
+        """
+        if image_spec is None:
+            image_spec = make_image_spec()
+        return {
+            'arches': [image_spec.arch],
+            'subarches': [image_spec.subarch],
+            'release': image_spec.release,
+            'labels': [image_spec.label],
+            }
+
+    def test_any_image_passes_none_filter(self):
+        arch, subarch, release, label = make_image_spec()
+        self.assertTrue(
+            download_descriptions.image_passes_filter(
+                None, arch, subarch, release, label))
+
+    def test_any_image_passes_empty_filter(self):
+        arch, subarch, release, label = make_image_spec()
+        self.assertTrue(
+            download_descriptions.image_passes_filter(
+                [], arch, subarch, release, label))
+
+    def test_image_passes_matching_filter(self):
+        image = make_image_spec()
+        self.assertTrue(
+            download_descriptions.image_passes_filter(
+                [self.make_filter_from_image(image)],
+                image.arch, image.subarch, image.release, image.label))
+
+    def test_image_does_not_pass_nonmatching_filter(self):
+        image = make_image_spec()
+        self.assertFalse(
+            download_descriptions.image_passes_filter(
+                [self.make_filter_from_image()],
+                image.arch, image.subarch, image.release, image.label))
+
+    def test_image_passes_if_one_filter_matches(self):
+        image = make_image_spec()
+        self.assertTrue(
+            download_descriptions.image_passes_filter(
+                [
+                    self.make_filter_from_image(),
+                    self.make_filter_from_image(image),
+                    self.make_filter_from_image(),
+                ], image.arch, image.subarch, image.release, image.label))
+
+    def test_filter_checks_release(self):
+        image = make_image_spec()
+        self.assertFalse(
+            download_descriptions.image_passes_filter(
+                [
+                    self.make_filter_from_image(image._replace(
+                        release=factory.make_name('other-release')))
+                ], image.arch, image.subarch, image.release, image.label))
+
+    def test_filter_checks_arches(self):
+        image = make_image_spec()
+        self.assertFalse(
+            download_descriptions.image_passes_filter(
+                [
+                    self.make_filter_from_image(image._replace(
+                        arch=factory.make_name('other-arch')))
+                ], image.arch, image.subarch, image.release, image.label))
+
+    def test_filter_checks_subarches(self):
+        image = make_image_spec()
+        self.assertFalse(
+            download_descriptions.image_passes_filter(
+                [
+                    self.make_filter_from_image(image._replace(
+                        subarch=factory.make_name('other-subarch')))
+                ], image.arch, image.subarch, image.release, image.label))
+
+    def test_filter_checks_labels(self):
+        image = make_image_spec()
+        self.assertFalse(
+            download_descriptions.image_passes_filter(
+                [
+                    self.make_filter_from_image(image._replace(
+                        label=factory.make_name('other-label')))
+                ], image.arch, image.subarch, image.release, image.label))
+
+
+class TestBootMerge(MAASTestCase):
+    """Tests for `boot_merge`."""
+
+    def test_integrates(self):
+        # End-to-end scenario for boot_merge: start with an empty boot
+        # resources dict, and receive one resource from Simplestreams.
+        total_resources = BootImageMapping()
+        resources_from_repo = set_resource()
+        download_descriptions.boot_merge(total_resources, resources_from_repo)
+        # Since we started with an empty dict, the result contains the same
+        # item that we got from Simplestreams, and nothing else.
+        self.assertEqual(resources_from_repo.mapping, total_resources.mapping)
+
+    def test_obeys_filters(self):
+        filters = [
+            {
+                'arches': [factory.make_name('other-arch')],
+                'subarches': [factory.make_name('other-subarch')],
+                'release': factory.make_name('other-release'),
+                'label': [factory.make_name('other-label')],
+            },
+            ]
+        total_resources = BootImageMapping()
+        resources_from_repo = set_resource()
+        download_descriptions.boot_merge(
+            total_resources, resources_from_repo, filters=filters)
+        self.assertEqual({}, total_resources.mapping)
+
+    def test_does_not_overwrite_existing_entry(self):
+        image = make_image_spec()
+        total_resources = set_resource(
+            resource="Original resource", image_spec=image)
+        original_resources = total_resources.mapping.copy()
+        resources_from_repo = set_resource(
+            resource="New resource", image_spec=image)
+        download_descriptions.boot_merge(total_resources, resources_from_repo)
+        self.assertEqual(original_resources, total_resources.mapping)

=== added file 'src/provisioningserver/import_images/tests/test_helpers.py'
--- src/provisioningserver/import_images/tests/test_helpers.py	1970-01-01 00:00:00 +0000
+++ src/provisioningserver/import_images/tests/test_helpers.py	2014-04-08 10:22:55 +0000
@@ -0,0 +1,61 @@
+# Copyright 2014 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Tests for the `helpers` module."""
+
+from __future__ import (
+    absolute_import,
+    print_function,
+    unicode_literals,
+    )
+
+str = None
+
+__metaclass__ = type
+__all__ = []
+
+from maastesting.factory import factory
+from maastesting.matchers import MockCalledOnceWith
+from maastesting.testcase import MAASTestCase
+import mock
+from provisioningserver.import_images import helpers
+from simplestreams.util import SignatureMissingException
+
+
+class TestGetSigningPolicy(MAASTestCase):
+    """Tests for `get_signing_policy`."""
+
+    def test_picks_nonchecking_policy_for_json_index(self):
+        path = 'streams/v1/index.json'
+        policy = helpers.get_signing_policy(path)
+        content = factory.getRandomString()
+        self.assertEqual(
+            content,
+            policy(content, path, factory.make_name('keyring')))
+
+    def test_picks_checking_policy_for_sjson_index(self):
+        path = 'streams/v1/index.sjson'
+        content = factory.getRandomString()
+        policy = helpers.get_signing_policy(path)
+        self.assertRaises(
+            SignatureMissingException,
+            policy, content, path, factory.make_name('keyring'))
+
+    def test_picks_checking_policy_for_json_gpg_index(self):
+        path = 'streams/v1/index.json.gpg'
+        content = factory.getRandomString()
+        policy = helpers.get_signing_policy(path)
+        self.assertRaises(
+            SignatureMissingException,
+            policy, content, path, factory.make_name('keyring'))
+
+    def test_injects_default_keyring_if_passed(self):
+        path = 'streams/v1/index.json.gpg'
+        content = factory.getRandomString()
+        keyring = factory.make_name('keyring')
+        self.patch(helpers, 'policy_read_signed')
+        policy = helpers.get_signing_policy(path, keyring)
+        policy(content, path)
+        self.assertThat(
+            helpers.policy_read_signed,
+            MockCalledOnceWith(mock.ANY, mock.ANY, keyring=keyring))

=== added file 'src/provisioningserver/import_images/tests/test_product_mapping.py'
--- src/provisioningserver/import_images/tests/test_product_mapping.py	1970-01-01 00:00:00 +0000
+++ src/provisioningserver/import_images/tests/test_product_mapping.py	2014-04-08 10:22:55 +0000
@@ -0,0 +1,125 @@
+# Copyright 2014 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Tests for the `ProductMapping` class."""
+
+from __future__ import (
+    absolute_import,
+    print_function,
+    unicode_literals,
+    )
+
+str = None
+
+__metaclass__ = type
+__all__ = []
+
+from maastesting.factory import factory
+from maastesting.testcase import MAASTestCase
+from provisioningserver.import_images.product_mapping import ProductMapping
+from provisioningserver.import_images.testing.factory import (
+    make_boot_resource,
+    )
+
+
+class TestProductMapping(MAASTestCase):
+    """Tests for `ProductMapping`."""
+
+    def test_initially_empty(self):
+        self.assertEqual({}, ProductMapping().mapping)
+
+    def test_make_key_extracts_identifying_items(self):
+        resource = make_boot_resource()
+        content_id = resource['content_id']
+        product_name = resource['product_name']
+        version_name = resource['version_name']
+        self.assertEqual(
+            (content_id, product_name, version_name),
+            ProductMapping.make_key(resource))
+
+    def test_make_key_ignores_other_items(self):
+        resource = make_boot_resource()
+        resource['other_item'] = factory.make_name('other')
+        self.assertEqual(
+            (
+                resource['content_id'],
+                resource['product_name'],
+                resource['version_name'],
+            ),
+            ProductMapping.make_key(resource))
+
+    def test_make_key_fails_if_key_missing(self):
+        resource = make_boot_resource()
+        del resource['version_name']
+        self.assertRaises(
+            KeyError,
+            ProductMapping.make_key, resource)
+
+    def test_add_creates_subarches_list_if_needed(self):
+        product_dict = ProductMapping()
+        resource = make_boot_resource()
+        subarch = factory.make_name('subarch')
+        product_dict.add(resource, subarch)
+        self.assertEqual(
+            {product_dict.make_key(resource): [subarch]},
+            product_dict.mapping)
+
+    def test_add_appends_to_existing_list(self):
+        product_dict = ProductMapping()
+        resource = make_boot_resource()
+        subarches = [factory.make_name('subarch') for _ in range(2)]
+        for subarch in subarches:
+            product_dict.add(resource, subarch)
+        self.assertEqual(
+            {product_dict.make_key(resource): subarches},
+            product_dict.mapping)
+
+    def test_contains_returns_true_for_stored_item(self):
+        product_dict = ProductMapping()
+        resource = make_boot_resource()
+        subarch = factory.make_name('subarch')
+        product_dict.add(resource, subarch)
+        self.assertTrue(product_dict.contains(resource))
+
+    def test_contains_returns_false_for_unstored_item(self):
+        self.assertFalse(
+            ProductMapping().contains(make_boot_resource()))
+
+    def test_contains_ignores_similar_items(self):
+        product_dict = ProductMapping()
+        resource = make_boot_resource()
+        subarch = factory.make_name('subarch')
+        product_dict.add(resource.copy(), subarch)
+        resource['product_name'] = factory.make_name('other')
+        self.assertFalse(product_dict.contains(resource))
+
+    def test_contains_ignores_extraneous_keys(self):
+        product_dict = ProductMapping()
+        resource = make_boot_resource()
+        subarch = factory.make_name('subarch')
+        product_dict.add(resource.copy(), subarch)
+        resource['other_item'] = factory.make_name('other')
+        self.assertTrue(product_dict.contains(resource))
+
+    def test_get_returns_stored_item(self):
+        product_dict = ProductMapping()
+        resource = make_boot_resource()
+        subarch = factory.make_name('subarch')
+        product_dict.add(resource, subarch)
+        self.assertEqual([subarch], product_dict.get(resource))
+
+    def test_get_fails_for_unstored_item(self):
+        product_dict = ProductMapping()
+        resource = make_boot_resource()
+        subarch = factory.make_name('subarch')
+        product_dict.add(resource.copy(), subarch)
+        resource['content_id'] = factory.make_name('other')
+        self.assertRaises(KeyError, product_dict.get, resource)
+
+    def test_get_ignores_extraneous_keys(self):
+        product_dict = ProductMapping()
+        resource = make_boot_resource()
+        subarch = factory.make_name('subarch')
+        product_dict.add(resource, subarch)
+        resource['other_item'] = factory.make_name('other')
+        self.assertEqual([subarch], product_dict.get(resource))