Merge lp:~blake-rouse/maas/fix-repo-dumper-hwe-images-1.7 into lp:maas/1.7

[Blake Rouse]

Fix RepoDumper to handle HWE images correctly. Fix pxeconfig to not use commissioning osystem and distro_series for Ubuntu xinstall.

Merged into trunk: https://code.launchpad.net/~blake-rouse/maas/fix-repo-dumper-hwe-images/+merge/238967

[Gavin Panella (allenap)]

Same as for https://code.launchpad.net/~blake-rouse/maas/fix-1378527-1.7-again/+merge/239050/comments/587151.

review: Approve

=== modified file 'src/maasserver/api/pxeconfig.py'
--- src/maasserver/api/pxeconfig.py	2014-10-09 12:03:53 +0000
+++ src/maasserver/api/pxeconfig.py	2014-10-21 13:19:27 +0000
@@ -212,7 +212,11 @@
     else:
         purpose = node.get_boot_purpose()
         event_log_pxe_request(node, purpose)
-    if purpose == "xinstall":
+
+    # Use only the commissioning osystem and series, for operating systems
+    # other than Ubuntu. As Ubuntu supports HWE kernels, and needs to use
+    # that kernel to perform the installation.
+    if purpose == "xinstall" and osystem != 'ubuntu':
         osystem = Config.objects.get_config('commissioning_osystem')
         series = Config.objects.get_config('commissioning_distro_series')
 

=== modified file 'src/maasserver/api/tests/test_pxeconfig.py'
--- src/maasserver/api/tests/test_pxeconfig.py	2014-10-09 10:08:54 +0000
+++ src/maasserver/api/tests/test_pxeconfig.py	2014-10-21 13:19:27 +0000
@@ -491,3 +491,50 @@
         self.client.get(reverse('pxeconfig'), params)
         node = reload_object(node)
         self.assertEqual(mac, node.pxe_mac)
+
+    def test_pxeconfig_returns_commissioning_os_series_for_other_oses(self):
+        osystem = Config.objects.get_config('default_osystem')
+        release = Config.objects.get_config('default_distro_series')
+        nodegroup = factory.make_NodeGroup()
+        os_image = make_rpc_boot_image(purpose='xinstall')
+        architecture = '%s/%s' % (
+            os_image['architecture'], os_image['subarchitecture'])
+        self.patch(
+            preseed_module,
+            'get_boot_images_for').return_value = [os_image]
+        self.patch(
+            pxeconfig_module,
+            'get_boot_images_for').return_value = [os_image]
+        node = factory.make_Node(
+            mac=True, nodegroup=nodegroup, status=NODE_STATUS.DEPLOYING,
+            osystem=os_image['osystem'],
+            distro_series=os_image['release'],
+            architecture=architecture)
+        params = self.get_default_params()
+        params['cluster_uuid'] = nodegroup.uuid
+        params['mac'] = node.get_primary_mac()
+        params_out = self.get_pxeconfig(params)
+        self.assertEqual(osystem, params_out["osystem"])
+        self.assertEqual(release, params_out["release"])
+
+    def test_pxeconfig_returns_ubuntu_os_series_for_ubuntu_xinstall(self):
+        nodegroup = factory.make_NodeGroup()
+        ubuntu_image = make_rpc_boot_image(
+            osystem='ubuntu', purpose='xinstall')
+        architecture = '%s/%s' % (
+            ubuntu_image['architecture'], ubuntu_image['subarchitecture'])
+        self.patch(
+            preseed_module,
+            'get_boot_images_for').return_value = [ubuntu_image]
+        self.patch(
+            pxeconfig_module,
+            'get_boot_images_for').return_value = [ubuntu_image]
+        node = factory.make_Node(
+            mac=True, nodegroup=nodegroup, status=NODE_STATUS.DEPLOYING,
+            osystem='ubuntu', distro_series=ubuntu_image['release'],
+            architecture=architecture)
+        params = self.get_default_params()
+        params['cluster_uuid'] = nodegroup.uuid
+        params['mac'] = node.get_primary_mac()
+        params_out = self.get_pxeconfig(params)
+        self.assertEqual(ubuntu_image['release'], params_out["release"])

=== modified file 'src/provisioningserver/import_images/boot_image_mapping.py'
--- src/provisioningserver/import_images/boot_image_mapping.py	2014-10-02 00:53:53 +0000
+++ src/provisioningserver/import_images/boot_image_mapping.py	2014-10-21 13:19:27 +0000
@@ -61,6 +61,11 @@
         assert isinstance(image_spec, ImageSpec)
         self.mapping.setdefault(image_spec, item)
 
+    def set(self, image_spec, item):
+        """"Set metadata for `image_spec` to item, even if already set."""
+        assert isinstance(image_spec, ImageSpec)
+        self.mapping[image_spec] = item
+
     def dump_json(self):
         """Produce JSON representing the mapped boot images.
 

=== modified file 'src/provisioningserver/import_images/download_descriptions.py'
--- src/provisioningserver/import_images/download_descriptions.py	2014-10-21 13:19:27 +0000
+++ src/provisioningserver/import_images/download_descriptions.py	2014-10-21 13:19:27 +0000
@@ -64,7 +64,11 @@
     """
 
     def __init__(self, boot_images_dict):
-        super(RepoDumper, self).__init__()
+        super(RepoDumper, self).__init__(config={
+            # Only download the latest version. Without this all versions
+            # will be read, causing miss matches in versions.
+            'max_items': 1,
+            })
         self.boot_images_dict = boot_images_dict
 
     def load_products(self, path=None, content_id=None):
@@ -78,12 +82,27 @@
         """Overridable from `BasicMirrorWriter`."""
         item = products_exdata(src, pedigree)
         os = get_os_from_product(item)
-        arch, subarch = item['arch'], item['subarch']
+        arch, subarches = item['arch'], item['subarches']
         release = item['release']
         label = item['label']
-        base_image = ImageSpec(os, arch, subarch, release, label)
+        base_image = ImageSpec(os, arch, None, release, label)
         compact_item = clean_up_repo_item(item)
-        self.boot_images_dict.setdefault(base_image, compact_item)
+        for subarch in subarches.split(','):
+            self.boot_images_dict.setdefault(
+                base_image._replace(subarch=subarch), compact_item)
+
+        # HWE resources need to map to a specfic resource, and not just to
+        # any of the supported subarchitectures for that resource.
+        subarch = item['subarch']
+        self.boot_images_dict.set(
+            base_image._replace(subarch=subarch), compact_item)
+
+        # HWE resources with generic, should map to the HWE that ships with
+        # that release.
+        hwe_arch = 'hwe-%s' % release[0]
+        if subarch == hwe_arch and 'generic' in subarches:
+            self.boot_images_dict.set(
+                base_image._replace(subarch='generic'), compact_item)
 
 
 def value_passes_filter_list(filter_list, property_value):

=== modified file 'src/provisioningserver/import_images/tests/test_boot_image_mapping.py'
--- src/provisioningserver/import_images/tests/test_boot_image_mapping.py	2014-10-02 00:53:53 +0000
+++ src/provisioningserver/import_images/tests/test_boot_image_mapping.py	2014-10-21 13:19:27 +0000
@@ -63,6 +63,14 @@
         image_dict.setdefault(image, factory.make_name('newresource'))
         self.assertItemsEqual([(image, old_resource)], image_dict.items())
 
+    def test_set_overwrites_item(self):
+        image_dict = BootImageMapping()
+        image = make_image_spec()
+        resource = factory.make_name('resource')
+        image_dict.setdefault(image, factory.make_name('resource'))
+        image_dict.set(image, resource)
+        self.assertItemsEqual([(image, resource)], image_dict.items())
+
     def test_dump_json_is_consistent(self):
         image = make_image_spec()
         resource = factory.make_name('resource')

=== modified file 'src/provisioningserver/import_images/tests/test_download_descriptions.py'
--- src/provisioningserver/import_images/tests/test_download_descriptions.py	2014-09-30 20:54:32 +0000
+++ src/provisioningserver/import_images/tests/test_download_descriptions.py	2014-10-21 13:19:27 +0000
@@ -16,10 +16,15 @@
 
 from maastesting.factory import factory
 from maastesting.testcase import MAASTestCase
+from mock import sentinel
 from provisioningserver.import_images import download_descriptions
 from provisioningserver.import_images.boot_image_mapping import (
     BootImageMapping,
     )
+from provisioningserver.import_images.download_descriptions import (
+    clean_up_repo_item,
+    RepoDumper,
+    )
 from provisioningserver.import_images.testing.factory import (
     make_image_spec,
     set_resource,
@@ -221,3 +226,111 @@
             resource="New resource", image_spec=image)
         download_descriptions.boot_merge(total_resources, resources_from_repo)
         self.assertEqual(original_resources, total_resources.mapping)
+
+
+class TestRepoDumper(MAASTestCase):
+    """Tests for `RepoDumper`."""
+
+    def make_item(self, os=None, release=None, arch=None,
+                  subarch=None, subarches=None, label=None):
+        if os is None:
+            os = factory.make_name('os')
+        if release is None:
+            release = factory.make_name('release')
+        if arch is None:
+            arch = factory.make_name('arch')
+        if subarch is None:
+            subarch = factory.make_name('subarch')
+        if subarches is None:
+            subarches = [factory.make_name('subarch') for _ in range(3)]
+        if subarch not in subarches:
+            subarches.append(subarch)
+        if label is None:
+            label = factory.make_name('label')
+        item = {
+            'content_id': factory.make_name('content_id'),
+            'product_name': factory.make_name('product_name'),
+            'version_name': factory.make_name('version_name'),
+            'path': factory.make_name('path'),
+            'os': os,
+            'release': release,
+            'arch': arch,
+            'subarch': subarch,
+            'subarches': ','.join(subarches),
+            'label': label,
+            }
+        return item, clean_up_repo_item(item)
+
+    def test_insert_item_adds_item_per_subarch(self):
+        boot_images_dict = BootImageMapping()
+        dumper = RepoDumper(boot_images_dict)
+        subarches = [factory.make_name('subarch') for _ in range(3)]
+        item, _ = self.make_item(
+            subarch=subarches.pop(), subarches=subarches)
+        self.patch(
+            download_descriptions, 'products_exdata').return_value = item
+        dumper.insert_item(
+            sentinel.data, sentinel.src, sentinel.target,
+            sentinel.pedigree, sentinel.contentsource)
+        image_specs = [
+            make_image_spec(
+                os=item['os'], release=item['release'],
+                arch=item['arch'], subarch=subarch,
+                label=item['label'])
+            for subarch in subarches
+        ]
+        self.assertItemsEqual(image_specs, boot_images_dict.mapping.keys())
+
+    def test_insert_item_sets_compat_item_specific_to_subarch(self):
+        boot_images_dict = BootImageMapping()
+        dumper = RepoDumper(boot_images_dict)
+        subarches = [factory.make_name('subarch') for _ in range(5)]
+        compat_subarch = subarches.pop()
+        item, _ = self.make_item(subarch=subarches.pop(), subarches=subarches)
+        second_item, compat_item = self.make_item(
+            os=item['os'], release=item['release'], arch=item['arch'],
+            subarch=compat_subarch, subarches=[compat_subarch],
+            label=item['label'])
+        self.patch(
+            download_descriptions,
+            'products_exdata').side_effect = [item, second_item]
+        for _ in range(2):
+            dumper.insert_item(
+                sentinel.data, sentinel.src, sentinel.target,
+                sentinel.pedigree, sentinel.contentsource)
+        image_spec = make_image_spec(
+            os=item['os'], release=item['release'],
+            arch=item['arch'], subarch=compat_subarch,
+            label=item['label'])
+        self.assertEqual(compat_item, boot_images_dict.mapping[image_spec])
+
+    def test_insert_item_sets_generic_to_release_item_for_hwe(self):
+        boot_images_dict = BootImageMapping()
+        dumper = RepoDumper(boot_images_dict)
+        os = 'ubuntu'
+        release = 'precise'
+        arch = 'amd64'
+        label = 'release'
+        hwep_subarch = 'hwe-p'
+        hwep_subarches = ['generic', 'hwe-p']
+        hwes_subarch = 'hwe-s'
+        hwes_subarches = ['generic', 'hwe-p', 'hwe-s']
+        hwep_item, compat_item = self.make_item(
+            os=os, release=release,
+            arch=arch, subarch=hwep_subarch,
+            subarches=hwep_subarches, label=label)
+        hwes_item, _ = self.make_item(
+            os=os, release=release,
+            arch=arch, subarch=hwes_subarch,
+            subarches=hwes_subarches, label=label)
+        self.patch(
+            download_descriptions,
+            'products_exdata').side_effect = [hwep_item, hwes_item]
+        for _ in range(2):
+            dumper.insert_item(
+                sentinel.data, sentinel.src, sentinel.target,
+                sentinel.pedigree, sentinel.contentsource)
+        image_spec = make_image_spec(
+            os=os, release=release, arch=arch, subarch='generic',
+            label=label)
+        self.assertEqual(compat_item, boot_images_dict.mapping[image_spec])