Merge lp:~ltrager/maas-images/lp1618709 into lp:maas-images

[Lee Trager]

Read the meph yaml config value 'squashfs' in stream.py to determine whether to include the root-image and manifest or the squashfs image in the product version. This ensures both meph2-build and meph2-cloudimg-sync publish the correct files for the configuration file they are given.

[Scott Moser (smoser)]

one comment.
we should publish the manifest for the squashfs path. even if we're building everything we could eitehr take it as a '--manifest' argument or we could generate it from
  mount-image-callback <image> -- chroot _MOUNTPOINT_ dpkg-query --show

[lp:~ltrager/maas-images/lp1618709]

328. By Lee Trager on 2016-09-02

Add manifest for SquashFS image

[Lee Trager (ltrager)]

Updated to include a manifest for SquashFS images generated with the --manifest argument. This works when using the meph2-build or meph2-cloudimg-sync command.

[Scott Moser (smoser)]

looks good. thank you, Lee.

review: Approve

=== modified file 'meph2/commands/cloudimg_sync.py'
--- meph2/commands/cloudimg_sync.py	2016-08-24 05:05:47 +0000
+++ meph2/commands/cloudimg_sync.py	2016-09-02 01:23:15 +0000
@@ -11,7 +11,6 @@
 from meph2.stream import CONTENT_ID, create_version
 
 import argparse
-import glob
 import copy
 import os
 import sys
@@ -146,39 +145,6 @@
 
         for prodname, items in cvret.items():
             for i in items:
-                filename = os.path.join(self.out_d, items[i]['path'])
-                if i == 'squashfs':
-                    # Verify upstream SHA256 of SquashFS image.
-                    if items[i]['sha256'] != flat['sha256']:
-                        raise ValueError(
-                            'Expected SHA256 %s got %s on %s' %
-                            (flat['sha256'], items[i]['sha256'], filename))
-                    if not self.squashfs:
-                        # If we're not publishing the SquashFS image but one
-                        # was used to generate root-image.gz delete it.
-                        if os.path.exists(filename):
-                            os.remove(filename)
-                        continue
-                elif i == 'root-image.gz' and self.squashfs:
-                    # If we're publishing the SquashFS image we don't need the
-                    # root-image after its been used to extract the kernels.
-                    # Older Ubuntu releases (<16.04) don't have SquashFS images
-                    # published, so only remove if a SquashFS file exists.
-                    squashfs_image = os.path.join(
-                        os.path.dirname(filename), '*squashfs')
-                    if len(glob.glob(squashfs_image)) > 0:
-                        if os.path.exists(filename):
-                            os.remove(filename)
-                        continue
-                elif i == 'manifest' and self.squashfs:
-                    # If we're publishing the SquashFS image we don't need the
-                    # root-image manifest either.
-                    squashfs_image = os.path.join(
-                        os.path.dirname(filename), '*squashfs')
-                    if len(glob.glob(squashfs_image)) > 0:
-                        if os.path.exists(filename):
-                            os.remove(filename)
-                        continue
                 sutil.products_set(
                     self.content_t, items[i], (prodname, vername, i))
 

=== modified file 'meph2/stream.py'
--- meph2/stream.py	2016-08-25 02:38:40 +0000
+++ meph2/stream.py	2016-09-02 01:23:15 +0000
@@ -19,8 +19,10 @@
 DI_COMMON = PATH_COMMON + "di/%(di_version)s/%(krel)s/%(flavor)s"
 PATH_FORMATS = {
     'root-image.gz': PATH_COMMON + "%(version_name)s/root-image.gz",
-    'manifest': PATH_COMMON + "%(version_name)s/root-image.manifest",
+    'root-image.manifest': (
+        PATH_COMMON + "%(version_name)s/root-image.manifest"),
     'squashfs': PATH_COMMON + "%(version_name)s/squashfs",
+    'squashfs.manifest': PATH_COMMON + "%(version_name)s/squashfs.manifest",
     'boot-dtb': BOOT_COMMON + "/boot-dtb%(suffix)s",
     'boot-kernel': BOOT_COMMON + "/boot-kernel%(suffix)s",
     'boot-initrd': BOOT_COMMON + "/boot-initrd%(suffix)s",
@@ -120,16 +122,24 @@
             'version_name': version_name, 'version': version}
 
     rootimg_path = PATH_FORMATS['root-image.gz'] % subs
-    manifest_path = PATH_FORMATS['manifest'] % subs
+
+    krd_packs = []
+    squashfs = cfgdata.get('squashfs', False)
+    boot_keys = ['boot-kernel', 'boot-initrd']
+    if squashfs and img_url.endswith('.squashfs'):
+        base_ikeys = boot_keys + ['squashfs', 'squashfs.manifest']
+        manifest_path = PATH_FORMATS['squashfs.manifest'] % subs
+        newpaths = set((PATH_FORMATS['squashfs'] % subs, manifest_path))
+    else:
+        base_ikeys = boot_keys + ['root-image.gz', 'root-image.manifest']
+        manifest_path = PATH_FORMATS['root-image.manifest'] % subs
+        newpaths = set((rootimg_path, manifest_path))
 
     gencmd = ([mci2e] + mci2e_flags +
               [bkparm, "--arch=%s" % arch,
                "--manifest=%s" % os.path.join(out_d, manifest_path),
                img_url, os.path.join(out_d, rootimg_path)])
 
-    krd_packs = []
-    newpaths = set((rootimg_path, manifest_path,))
-
     kdata_defaults = {'suffix': "", 'di-format': "default", 'dtb': ""}
 
     for info in rdata['kernels']:
@@ -158,11 +168,7 @@
                      'psubarch': product_psubarch,
                      'suffix': kdata["suffix"]})
 
-        boot_keys = ['boot-kernel', 'boot-initrd']
-        ikeys = boot_keys + ['root-image.gz', 'manifest']
-        if img_url.endswith('.squashfs'):
-            ikeys += ['squashfs']
-            newpaths.add(PATH_FORMATS['squashfs'] % subs)
+        ikeys = base_ikeys
 
         dtb = kdata.get('dtb')
         if dtb:
@@ -204,9 +210,15 @@
 
         items = {}
         for i in ikeys:
-            items[i] = {'ftype': i, 'path': PATH_FORMATS[i] % subs,
-                        'size': None, 'sha256': None}
-            items[i].update(common)
+            # Allow root-image.manifest and squashfs.image to have different
+            # filenames but keep the same ftype.
+            if 'manifest' in i:
+                ftype = 'manifest'
+            else:
+                ftype = i
+            items[ftype] = {'ftype': ftype, 'path': PATH_FORMATS[i] % subs,
+                            'size': None, 'sha256': None}
+            items[ftype].update(common)
 
         for key in di_keys:
             items[key]['sha256'] = curdi[key]['sha256']
@@ -248,12 +260,23 @@
         subprocess.check_call(gencmd)
         LOG.info("finished: %s" % gencmd)
 
-    # If we're downloading a SquashFS file rename it to its filetype
     if img_url.endswith('squashfs'):
         base_dir = os.path.join(out_d, release, arch, version_name)
         src_squash = os.path.join(base_dir, os.path.basename(img_url))
-        dst_squash = os.path.join(base_dir, 'squashfs')
-        os.rename(src_squash, dst_squash)
+        if squashfs:
+            # If we're publishing a SquashFS file rename it to its filetype.
+            dst_squash = os.path.join(base_dir, 'squashfs')
+            os.rename(src_squash, dst_squash)
+            # The root-img is used to generate the kernels and initrds. If
+            # we're publishing the SquashFS image then we don't want to publish
+            # the root-img, we can safely clean it up.
+            src_rootimg_path = os.path.join(
+                base_dir, os.path.basename(rootimg_path))
+            os.remove(src_rootimg_path)
+        else:
+            # If we're not publishing the SquashFS image but used it to
+            # generate the root-img clean it up.
+            os.remove(src_squash)
 
     # get checksum and size of new files created
     file_info = {}