Merge lp:~abentley/juju-release-tools/image-compatibility into lp:juju-release-tools

[Aaron Bentley]

Write image streams in a way compatible with old jujus.

Improve the compatibility of our image streams with old jujus.

Old jujus expected to find root_store and virt at the item level, but we were storing it at a higher level. The simplestreams authoring tools support forcing an attribute to be stored at the item level with the 'sticky' parameter.

Unfortunately, the 'sticky' parameter was not exposed by write_juju_streams, so I had to copy its implementation in order to set 'sticky'.

Let me know if you want more unit tests for write_juju_streams.

[Curtis Hovey (sinzui)]

Thank you.

review: Approve (code)

[Aaron Bentley (abentley)]

On 2016-05-10 03:38 PM, Curtis Hovey wrote:
> Is your tree stale? I reviewed this some time ago and it is in the tree.

I forgot to push after I merged it locally. I'm not sure why it was in
your tree.

=== added file 'build-juju'
--- build-juju	1970-01-01 00:00:00 +0000
+++ build-juju	2016-05-10 19:18:04 +0000
@@ -0,0 +1,30 @@
+#!/bin/bash
+set -eu
+usage() {
+    cat <<EOT
+usage: $0 BRANCH [REPO_PATH]"
+Build the specified juju branch to a package.  Intended for manual testing.
+
+BRANCH is the branch to build.
+REPO_PATH is end of the github repo URL.  It defaults to juju/juju.
+
+Packages are built for the current machine's series and the amd64 architecture.
+DEBEMAIL and DEBFULLNAME are derived from "bzr whoami".
+EOT
+exit 1
+}
+test $# -gt 0 || usage
+set -x
+
+TOOLS_DIR=$(dirname $(readlink -f $0))
+PATH=$TOOLS_DIR:$PATH
+BRANCH=$1
+REPO=https://github.com/${2-juju/juju}
+DEBEMAIL=$(bzr whoami --email -d $TOOLS_DIR)
+DEBFULLNAME=$(bzr whoami -d $TOOLS_DIR|sed 's/ *<.*>//')
+SERIES=$(lsb_release --codename -s)
+export PATH DEBEMAIL DEBFULLNAME
+make-release-tarball.bash $BRANCH $REPO
+build_package.py -v source ./juju-core_*.tar.gz ./ $SERIES 999999999
+build_package.py -v binary juju-build-$SERIES-all/juju-core_*.dsc \
+  ./build-binary-$SERIES-amd64 $SERIES amd64

=== modified file 'make_aws_image_streams.py'
--- make_aws_image_streams.py	2016-03-18 17:38:16 +0000
+++ make_aws_image_streams.py	2016-05-10 19:18:04 +0000
@@ -2,6 +2,7 @@
 from __future__ import print_function
 
 from argparse import ArgumentParser
+from copy import deepcopy
 from datetime import datetime
 import os
 import sys
@@ -10,13 +11,16 @@
 
 from boto import ec2
 from simplestreams.generate_simplestreams import (
+    generate_index,
     items2content_trees,
+    json_dump,
     )
 from simplestreams.json2streams import (
     Item,
-    write_juju_streams,
+    JujuFileNamer,
+    write_release_index,
     )
-from simplestreams.util import timestamp
+from simplestreams import util
 
 
 def get_parameters(argv=None):
@@ -190,10 +194,35 @@
     """
     items = [make_item(i, now) for i in iter_centos_images(
         credentials, china_credentials)]
-    updated = timestamp()
+    updated = util.timestamp()
     data = {'updated': updated, 'datatype': 'image-ids'}
     trees = items2content_trees(items, data)
-    write_juju_streams(streams, trees, updated)
+    write_juju_streams(streams, trees, updated, [
+        'path', 'sha256', 'md5', 'size', 'virt', 'root_store'])
+
+
+def write_juju_streams(out_d, trees, updated, sticky):
+    # Based on simplestreams.json2streams.write_juju_streams +
+    # simplestreams.generate_simplestreams.write_streams,
+    # but allows sticky to be specified.
+    namer = JujuFileNamer
+    index = generate_index(trees, updated, namer)
+    to_write = [(namer.get_index_path(), index,)]
+    # Don't let products_condense modify the input
+    trees = deepcopy(trees)
+    for content_id in trees:
+        util.products_condense(
+            trees[content_id], sticky=sticky)
+        content = trees[content_id]
+        to_write.append((index['index'][content_id]['path'], content,))
+    out_filenames = []
+    for (outfile, data) in to_write:
+        filef = os.path.join(out_d, outfile)
+        util.mkdir_p(os.path.dirname(filef))
+        json_dump(data, filef)
+        out_filenames.append(filef)
+    out_filenames.append(write_release_index(out_d))
+    return out_filenames
 
 
 def main():

=== modified file 'tests/test_make_aws_image_streams.py'
--- tests/test_make_aws_image_streams.py	2016-03-18 17:34:57 +0000
+++ tests/test_make_aws_image_streams.py	2016-05-10 19:18:04 +0000
@@ -179,7 +179,7 @@
 
     def test_happy_path(self):
         image = make_mock_image()
-        now = datetime(2001, 02, 03)
+        now = datetime(2001, 2, 3)
         item = make_item(image, now)
         self.assertEqual(item.content_id, 'com.ubuntu.cloud.released:aws')
         self.assertEqual(item.product_name,
@@ -204,7 +204,7 @@
     def test_china(self):
         image = make_mock_image()
         image.region.endpoint = 'foo.amazonaws.com.cn'
-        now = datetime(2001, 02, 03)
+        now = datetime(2001, 2, 3)
         item = make_item(image, now)
         self.assertEqual(item.content_id, 'com.ubuntu.cloud.released:aws-cn')
         self.assertEqual(item.data['endpoint'], 'https://foo.amazonaws.com.cn')
@@ -212,7 +212,7 @@
     def test_not_x86_64(self):
         image = make_mock_image()
         image.architecture = 'ppc128'
-        now = datetime(2001, 02, 03)
+        now = datetime(2001, 2, 3)
         with self.assertRaisesRegexp(ValueError,
                                      'Architecture is "ppc128", not'
                                      ' "x86_64".'):
@@ -221,7 +221,7 @@
     def test_not_centos_7(self):
         image = make_mock_image()
         image.name = 'CentOS Linux 8'
-        now = datetime(2001, 02, 03)
+        now = datetime(2001, 2, 3)
         with self.assertRaisesRegexp(ValueError,
                                      'Name "CentOS Linux 8" does not begin'
                                      ' with "CentOS Linux 7".'):
@@ -264,7 +264,7 @@
 class TestWriteStreams(TestCase):
 
     def test_write_streams(self):
-        now = datetime(2001, 02, 03)
+        now = datetime(2001, 2, 3)
         credentials = {'name': 'aws'}
         china_credentials = {'name': 'aws-cn'}
         east_conn = Mock()
@@ -277,7 +277,7 @@
             with patch('make_aws_image_streams.iter_region_connection',
                        return_value=[east_conn, west_conn],
                        autospec=True) as irc_mock:
-                with patch('make_aws_image_streams.timestamp',
+                with patch('simplestreams.util.timestamp',
                            return_value='now'):
                     with patch('sys.stderr'):
                         write_streams(credentials, china_credentials, now,
@@ -304,21 +304,27 @@
             'updated': 'now',
             'datatype': 'image-ids',
             'products': {'com.ubuntu.cloud:server:centos7:amd64': {
-                'root_store': 'ebs',
                 'endpoint': 'https://foo',
                 'arch': 'amd64',
                 'release_title': 'Centos 7',
                 'label': 'release',
                 'release_codename': 'centos7',
                 'version': 'centos7',
-                'virt': 'hvm',
                 'release': 'centos7',
                 'os': 'centos',
                 'id': 'qux',
                 'versions': {'20010203': {
                     'items': {
-                        'usww1he': {'region': 'us-west-1'},
-                        'usee1he': {'region': 'us-east-1'},
+                        'usww1he': {
+                            'region': 'us-west-1',
+                            'root_store': 'ebs',
+                            'virt': 'hvm',
+                            },
+                        'usee1he': {
+                            'region': 'us-east-1',
+                            'root_store': 'ebs',
+                            'virt': 'hvm',
+                            },
                         }
                     }},
                 }},