Merge lp:~pwlars/maas/core-simplestreams-rebased into lp:~maas-committers/maas/trunk

[Paul Larson]

Support for importing ubuntu-core images through simple streams

Here's a version of the changes to support ubuntu-core with simplestreams rebased on top of the rename patch, in order to clean up the history a bit. This replaces lp:~pwlars/maas/core-simplestreams

To use this, you'll need the image at http://people.canonical.com/~rharper/ubuntu-core/20170331/curtin-ubuntu-core-16-pc-amd64.img.xz

You'll also need to create the simplestreams data using https://code.launchpad.net/~ltrager/maas-images/create/+merge/321213
Filetype should be root-dd.xz and the image filename should also be root-dd.xz

Using these, and the existing support for ubuntu core in the image, I was able to get it to import and provision with the provided image.

[lp:~pwlars/maas/core-simplestreams-rebased]

5978. By Paul Larson on 2017-04-18

Add some unit tests for ubuntu-core simplestreams

[Paul Larson (pwlars)]

Added a few unit tests

review: Needs Resubmitting

[Blake Rouse (blake-rouse)]

Looks good, but still missing tests for the changes in the provisioningserver.

review: Needs Fixing

[lp:~pwlars/maas/core-simplestreams-rebased]

5979. By Paul Larson on 2017-04-18

Add unit tests for ubuntu core specific items in download_descriptions.py

[Paul Larson (pwlars)]

> Looks good, but still missing tests for the changes in the provisioningserver.
Thanks, I've added a couple of tests to ensure that the weird settings we use for ubuntu-core are set up correctly when os=='ubuntu-core'. Specifically, kflavor should be the kernel_snap and release should be {release}-{gadget}.

review: Needs Resubmitting

[Andres Rodriguez (andreserl)]

Marking this branch as merged provided it landed on bzr6015

=== modified file 'src/maasserver/bootresources.py'
--- src/maasserver/bootresources.py	2017-03-31 19:38:34 +0000
+++ src/maasserver/bootresources.py	2017-04-18 21:57:03 +0000
@@ -35,7 +35,10 @@
     StreamingHttpResponse,
 )
 from django.shortcuts import get_object_or_404
-from maasserver import locks
+from maasserver import (
+    __version__,
+    locks,
+)
 from maasserver.bootsources import (
     cache_boot_sources,
     ensure_boot_source_definition,
@@ -485,7 +488,13 @@
         arch = product['arch']
         kflavor = product.get('kflavor')
         bootloader_type = product.get('bootloader-type')
-        if bootloader_type is None:
+        if os == 'ubuntu-core':
+            kflavor = product.get('kernel_snap', 'generic')
+            release = product['release']
+            gadget = product['gadget_snap']
+            architecture = '%s/generic' % arch
+            series = "%s-%s" % (release, gadget)
+        elif bootloader_type is None:
             # The rack controller assumes the subarch is the kernel. We need to
             # include the kflavor in the subarch otherwise the rack will
             # overwrite the generic kernel with each kernel flavor.
@@ -1042,6 +1051,8 @@
         # Compile a regex to validate Ubuntu product names. This only allows
         # V2 and V3 Ubuntu products.
         self.ubuntu_regex = re.compile('.*:v[23]:.*', re.IGNORECASE)
+        # Compile a regex to validate Ubuntu Core. This only allows v4
+        self.ubuntucore_regex = re.compile('.*:v4:.*', re.IGNORECASE)
 
     def load_products(self, path=None, content_id=None):
         """Overridable from `BasicMirrorWriter`."""
@@ -1057,6 +1068,15 @@
 
     def _validate_ubuntu(self, data, product_name):
         osystem = data.get('os')
+        if osystem == 'ubuntu-core':
+            maas_version = tuple(int(x) for x in __version__.split('.'))
+            supported_version = tuple(
+                int(x) for x in data.get('maas_supported').split('.'))
+            if (self.ubuntucore_regex.search(product_name) and
+               maas_version >= supported_version):
+                return True
+            else:
+                return False
         if 'ubuntu' not in osystem.lower():
             # It's not an Ubuntu product, nothing to validate.
             return True

=== modified file 'src/maasserver/tests/test_bootresources.py'
--- src/maasserver/tests/test_bootresources.py	2017-03-31 20:18:23 +0000
+++ src/maasserver/tests/test_bootresources.py	2017-04-18 21:57:03 +0000
@@ -2342,3 +2342,96 @@
         mock_insert = self.patch(boot_resource_repo_writer.store, 'insert')
         boot_resource_repo_writer.insert_item(data, src, None, pedigree, None)
         self.assertThat(mock_insert, MockNotCalled())
+
+
+class TestBootResourceRepoWriterCore(MAASServerTestCase):
+    """Tests for `BootResourceRepoWriter` on ubuntu-core images."""
+
+    def create_simplestream(self, ftypes, stream_version=None):
+        arch = 'amd64'
+        if stream_version is None:
+            stream_version = 'v4'
+        release = '16'
+        gadget_snap = 'pc'
+        channel = 'stable'
+        product = "com.ubuntu.maas.daily:%s:%s:%s:%s:%s" % (
+            stream_version, release, arch, gadget_snap, channel)
+        version = datetime.now().date().strftime('%Y%m%d.0')
+        versions = {
+            version: {
+                'items': {
+                    ftype: {
+                        'sha256': factory.make_name('sha256'),
+                        'path': factory.make_name('path'),
+                        'ftype': ftype,
+                        'size': random.randint(0, 2**64),
+                    } for ftype in ftypes
+                }
+            }
+        }
+        products = {
+            product: {
+                'label': 'daily',
+                'os': 'ubuntu-core',
+                'os_title': 'Ubuntu Core',
+                'arch': arch,
+                'brand_id': 'canonical',
+                'channel': channel,
+                'gadget_snap': gadget_snap,
+                'gadget_title': 'PC',
+                'kernel_snap': 'pc-kernel',
+                'maas_supported': '2.2',
+                'model_name': 'pc-amd64',
+                'release': release,
+                'release_title': '16',
+                'versions': versions,
+            }
+        }
+        src = {
+            'datatype': 'image-downloads',
+            'format': 'products:1.0',
+            'updated': format_datetime(datetime.now()),
+            'products': products,
+            'content_id': 'com.ubuntu.maas:daily:v4:download'
+        }
+        return src, product, version
+
+    def test_insert_ubuntu_core_allows_ddxz(self):
+        boot_resource_repo_writer = BootResourceRepoWriter(
+            BootResourceStore(), None)
+        src, product, version = self.create_simplestream([
+            BOOT_RESOURCE_FILE_TYPE.ROOT_DDXZ
+        ])
+        data = src['products'][product]['versions'][version]['items'][
+            BOOT_RESOURCE_FILE_TYPE.ROOT_DDXZ]
+        pedigree = (product, version, BOOT_RESOURCE_FILE_TYPE.ROOT_DDXZ)
+        mock_insert = self.patch(boot_resource_repo_writer.store, 'insert')
+        boot_resource_repo_writer.insert_item(data, src, None, pedigree, None)
+        self.assertThat(mock_insert, MockCalledOnce())
+
+    def test_ubuntu_core_requires_v4_stream(self):
+        boot_resource_repo_writer = BootResourceRepoWriter(
+            BootResourceStore(), None)
+        src, product, version = self.create_simplestream([
+            BOOT_RESOURCE_FILE_TYPE.ROOT_DDXZ
+        ], 'v3')
+        data = src['products'][product]['versions'][version]['items'][
+            BOOT_RESOURCE_FILE_TYPE.ROOT_DDXZ]
+        pedigree = (product, version, BOOT_RESOURCE_FILE_TYPE.ROOT_DDXZ)
+        mock_insert = self.patch(boot_resource_repo_writer.store, 'insert')
+        boot_resource_repo_writer.insert_item(data, src, None, pedigree, None)
+        self.assertThat(mock_insert, MockNotCalled())
+
+    def test_ubuntu_core_checks_maas_supported_field(self):
+        boot_resource_repo_writer = BootResourceRepoWriter(
+            BootResourceStore(), None)
+        src, product, version = self.create_simplestream([
+            BOOT_RESOURCE_FILE_TYPE.ROOT_DDXZ
+        ])
+        src['products'][product]['maas_supported'] = '999.999'
+        data = src['products'][product]['versions'][version]['items'][
+            BOOT_RESOURCE_FILE_TYPE.ROOT_DDXZ]
+        pedigree = (product, version, BOOT_RESOURCE_FILE_TYPE.ROOT_DDXZ)
+        mock_insert = self.patch(boot_resource_repo_writer.store, 'insert')
+        boot_resource_repo_writer.insert_item(data, src, None, pedigree, None)
+        self.assertThat(mock_insert, MockNotCalled())

=== modified file 'src/provisioningserver/import_images/download_descriptions.py'
--- src/provisioningserver/import_images/download_descriptions.py	2017-01-28 00:51:47 +0000
+++ src/provisioningserver/import_images/download_descriptions.py	2017-04-18 21:57:03 +0000
@@ -91,24 +91,36 @@
         label = item['label']
         base_image = ImageSpec(os, arch, None, kflavor, release, label)
         compact_item = clean_up_repo_item(item)
-        for subarch in subarches.split(','):
+
+        if os == 'ubuntu-core':
+            # For Ubuntu Core we only want one entry per release/arch/gadget
+            subarch = item.get('gadget_snap', 'generic')
+            kflavor = item.get('kernel_snap', 'generic')
+            release = "%s-%s" % (release, subarch)
             self.boot_images_dict.setdefault(
+                base_image._replace(
+                    subarch=subarch, kflavor=kflavor,
+                    release=release), compact_item)
+        else:
+            for subarch in subarches.split(','):
+                self.boot_images_dict.setdefault(
+                    base_image._replace(subarch=subarch), compact_item)
+
+            # HWE resources need to map to a specfic resource, and not just to
+            # any of the supported subarchitectures for that resource.
+            subarch = item.get('subarch', 'generic')
+            self.boot_images_dict.set(
                 base_image._replace(subarch=subarch), compact_item)
 
-        # HWE resources need to map to a specfic resource, and not just to
-        # any of the supported subarchitectures for that resource.
-        subarch = item.get('subarch', 'generic')
-        self.boot_images_dict.set(
-            base_image._replace(subarch=subarch), compact_item)
-
-        if os == 'ubuntu' and item.get('version') is not None:
-            # HWE resources with generic, should map to the HWE that ships with
-            # that release. Starting with Xenial kernels changed from using the
-            # naming format hwe-<letter> to ga-<version>. Look for both.
-            hwe_archs = ["ga-%s" % item['version'], "hwe-%s" % release[0]]
-            if subarch in hwe_archs and 'generic' in subarches:
-                self.boot_images_dict.set(
-                    base_image._replace(subarch='generic'), compact_item)
+            if os == 'ubuntu' and item.get('version') is not None:
+                # HWE resources with generic, should map to the HWE that ships
+                # with that release. Starting with Xenial kernels changed from
+                # using the naming format hwe-<letter> to ga-<version>. Look
+                # for both.
+                hwe_archs = ["ga-%s" % item['version'], "hwe-%s" % release[0]]
+                if subarch in hwe_archs and 'generic' in subarches:
+                    self.boot_images_dict.set(
+                        base_image._replace(subarch='generic'), compact_item)
 
     def sync(self, reader, path):
         try:

=== modified file 'src/provisioningserver/import_images/tests/test_download_descriptions.py'
--- src/provisioningserver/import_images/tests/test_download_descriptions.py	2017-01-28 00:51:47 +0000
+++ src/provisioningserver/import_images/tests/test_download_descriptions.py	2017-04-18 21:57:03 +0000
@@ -236,7 +236,7 @@
 
     def make_item(self, os=None, release=None, version=None, arch=None,
                   subarch=None, subarches=None, label=None,
-                  bootloader_type=None):
+                  bootloader_type=None, gadget_snap=None, kernel_snap=None):
         if os is None:
             os = factory.make_name('os')
         if release is None:
@@ -266,10 +266,48 @@
             'subarches': ','.join(subarches),
             'label': label,
             }
+        if gadget_snap:
+            item['gadget_snap'] = gadget_snap
+        if kernel_snap:
+            item['kernel_snap'] = kernel_snap
         if bootloader_type is not None:
             item['bootloader-type'] = bootloader_type
         return item, clean_up_repo_item(item)
 
+    def test_insert_item_ubuntu_core_kflavor_is_ksnap(self):
+        boot_images_dict = BootImageMapping()
+        kernel_snap = 'pc-kernel'
+        dumper = RepoDumper(boot_images_dict)
+        item, _ = self.make_item(
+            os='ubuntu-core', release='16', gadget_snap='pc',
+            kernel_snap=kernel_snap)
+        self.patch(
+            download_descriptions, 'products_exdata').return_value = item
+        dumper.insert_item(
+            sentinel.data, sentinel.src, sentinel.target,
+            sentinel.pedigree, sentinel.contentsource)
+        image_spec = list(boot_images_dict.mapping.keys())[0]
+        self.assertEqual(kernel_snap, image_spec.kflavor)
+
+    def test_insert_item_ubuntu_core_release_combines_gadget(self):
+        boot_images_dict = BootImageMapping()
+        dumper = RepoDumper(boot_images_dict)
+        gadget_snap = 'pc'
+        release = '16'
+        item, _ = self.make_item(
+            os='ubuntu-core', release=release, gadget_snap=gadget_snap,
+            kernel_snap='pc-kernel')
+        # For ubuntu-core, the release should be a combination of the
+        # gadget snap and the release
+        expected_release = '%s-%s' % (release, gadget_snap)
+        self.patch(
+            download_descriptions, 'products_exdata').return_value = item
+        dumper.insert_item(
+            sentinel.data, sentinel.src, sentinel.target,
+            sentinel.pedigree, sentinel.contentsource)
+        image_spec = list(boot_images_dict.mapping.keys())[0]
+        self.assertEqual(expected_release, image_spec.release)
+
     def test_insert_item_adds_item_per_subarch(self):
         boot_images_dict = BootImageMapping()
         dumper = RepoDumper(boot_images_dict)