Merge lp:~rcj/simplestreams/lxd into lp:simplestreams

[Robert C Jennings]

[Dan Watkins (oddbloke)]

Overall approach looks good; comments inline.

review: Needs Fixing

[Dan Watkins (oddbloke)]

Looks much better; one remaining question.

review: Needs Information

[Robert C Jennings (rcj)]

fixed.

[Dan Watkins (oddbloke)]

SHIP IT

review: Approve

[lp:~rcj/simplestreams/lxd]

437. By Robert C Jennings on 2016-06-22

Create chksum for LXD metadata+root for squashfs (LP: #1577922)

This adds a 'combined_sqashfs_sha256' element to items that have
both a LXD metadata tar and a squashfs root. Additionally, we
have 'combined_rootxz_sha256' which wil have the same value as
'combined_sha256' and allow for sunsetting that ambiguous name.

[Robert C Jennings (rcj)]

I combined the revisions for a clean upstream merge with a good commit message. No code changes. Merging now.

=== modified file 'tools/make-test-data'
--- tools/make-test-data	2016-05-17 13:48:54 +0000
+++ tools/make-test-data	2016-06-22 19:05:52 +0000
@@ -288,23 +288,34 @@
         del fdata['content']
         FAKE_DATA[item['ftype']].update(fdata)
 
-    # create a combined_sha256 for lxd.tar.xz from the -root.tar.gz also
+    # create a combined sha256 for lxd.tar.xz (metadata) and the root fs
+    # - combined_sha256 and combined_rootxz_sha256 for the -root.tar.gz
+    # - combined_squashfs_sha256 for the squashfs
     ftype = item['ftype']
-    if ftype == "lxd.tar.xz" and 'combined_sha256' not in FAKE_DATA[ftype]:
-        rootpath = item['path'].replace("-lxd.tar.xz", "-root.tar.xz")
-        # make sure the root.tar.xz exists
-        if not os.path.exists(os.path.join(prefix, rootpath)):
-            rootitem = item.copy()
-            rootitem['ftype'] = "root.tar.xz"
-            rootitem['path'] = rootpath
-            create_fake_file(prefix, rootitem)
-
-        # and sha256 hash the combined file, .xz first
-        chash = hashlib.new('sha256')
-        for member in [fpath, os.path.join(prefix, rootpath)]:
-            with open(member, "rb") as fp:
-                chash.update(fp.read())
-        data.update({'combined_sha256': chash.hexdigest()})
+    for name, extension in (('rootxz', '-root.tar.xz'),
+                            ('squashfs', '.squashfs')):
+        if (ftype == "lxd.tar.xz" and
+                'combined_{}_sha256'.format(name) not in FAKE_DATA[ftype]):
+            rootpath = item['path'].replace("-lxd.tar.xz", extension)
+            if not os.path.exists(os.path.join(prefix, rootpath)):
+                rootitem = item.copy()
+                rootitem['ftype'] = extension.lstrip('-.')
+                rootitem['path'] = rootpath
+                create_fake_file(prefix, rootitem)
+
+            # and sha256 hash the combined file
+            chash = hashlib.new('sha256')
+            for member in [fpath, os.path.join(prefix, rootpath)]:
+                with open(member, "rb") as fp:
+                    chash.update(fp.read())
+            data.update(
+                {'combined_{}_sha256'.format(name): chash.hexdigest()})
+
+    # Add legacy combined_sha256 if combined_rootxz_sha256 exists
+    if (ftype == "lxd.tar.xz" and 'combined_sha256' not in FAKE_DATA[ftype]
+            and 'combined_rootxz_sha256' in FAKE_DATA[ftype]):
+        data.update({'combined_sha256':
+                     FAKE_DATA[ftype]['combined_rootxz_sha256']})
 
     item.update(data)
 
@@ -545,13 +556,22 @@
         path = item['path']
         item.update(get_cloud_images_file_info(path))
         if path.endswith('-lxd.tar.xz'):
-            # find calculated combined checksum
             dirname = os.path.dirname(path)
             lxd = os.path.basename(path)
-            root = lxd.replace('-lxd', '-root')
-            combined = os.path.join(dirname, ','.join([lxd, root]))
-            item.update({'combined_sha256':
-                         get_cache_data(combined, 'sha256')})
+
+            # find calculated combined checksums
+            for name, extension in (('rootxz', '-root.tar.xz'),
+                                    ('squashfs', '.squashfs')):
+                root = lxd.replace('-lxd.tar.xz', extension)
+                combined = os.path.join(dirname, ','.join([lxd, root]))
+                value = get_cache_data(combined, 'sha256')
+                if value:
+                    item.update({'combined_{}_sha256'.format(name): value})
+
+            # Add legacy combined_sha256 if combined_rootxz_sha256 exists
+            if 'combined_rootxz_sha256' in item:
+                value = item['combined_rootxz_sha256']
+                item.update({'combined_sha256': value})
 
     cid_fmt = "com.ubuntu.cloud:%s:download"
     for stream in tree:

=== modified file 'tools/toolutil.py'
--- tools/toolutil.py	2016-05-20 19:39:13 +0000
+++ tools/toolutil.py	2016-06-22 19:05:52 +0000
@@ -101,7 +101,8 @@
             return False
         if codename_cmp(rel, "<", "precise"):
             return False
-        if codename_cmp(rel, ">=", "yakkety"):
+        # -root.tar.xz is replaced by squashfs for yakkety
+        if codename_cmp(rel, ">=", "yakkety") and suffix == '-root.tar.xz':
             return False
 
     if suffix == '.squashfs' or suffix == '.squashfs.manifest':