Merge lp:~abentley/juju-release-tools/append-images into lp:juju-release-tools

[Aaron Bentley]

Support generating image streams for Centos7.

This branch implements support for generating Centos7 image streams.

It generates streams directly, instead of generating the intermediate JSON representation. This avoids two issues that prevented json2streams from working:
- It always uses "content-download" as the datatype.
- It fails if the JSON does not contain a 'size', even if it has no path.

It takes advantage of the Juju 2.0 credentials format, which is more convenient, even giving a standardized way of finding AWS credentials.

The product code aw0evgkw8e5c1q413zgy5pjce is used to find images. This is the official CentOS product code: https://wiki.centos.org/Cloud/AWS

This apppears to be the only way we can be certain to select the official images.

We have no aws-gov credentials, so these regions are skipped. http://cloud-images.ubuntu.com/releases/streams/v1/com.ubuntu.cloud:released:aws.json does not appear to have any existing gov entries, either.

[Curtis Hovey (sinzui)]

Thank you.

review: Approve (code)

=== added file 'make_aws_image_streams.py'
--- make_aws_image_streams.py	1970-01-01 00:00:00 +0000
+++ make_aws_image_streams.py	2016-03-17 14:53:09 +0000
@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+from __future__ import print_function
+
+from argparse import ArgumentParser
+from datetime import datetime
+import os
+import sys
+from textwrap import dedent
+import yaml
+
+from boto import ec2
+from simplestreams.generate_simplestreams import (
+    items2content_trees,
+    )
+from simplestreams.json2streams import (
+    Item,
+    write_juju_streams,
+    )
+from simplestreams.util import timestamp
+
+
+def get_parameters(argv=None):
+    """Return streams, creds_filename for this invocation.
+
+    streams is the directory to write streams into.
+    creds_filename is the filename to get credentials from.
+    """
+    parser = ArgumentParser(description=dedent("""
+        Write image streams for AWS images.  Only CentOS 7 is currently
+        supported."""))
+    parser.add_argument('streams', help='The directory to write streams to.')
+    args = parser.parse_args(argv)
+    try:
+        juju_data = os.environ['JUJU_DATA']
+    except KeyError:
+        print('JUJU_DATA must be set to a directory containing'
+              ' credentials.yaml.', file=sys.stderr)
+        sys.exit(1)
+    creds_filename = os.path.join(juju_data, 'credentials.yaml')
+    return args.streams, creds_filename
+
+
+def make_aws_credentials(creds):
+    """Convert credentials from juju format to AWS/Boto format."""
+    for creds in creds.values():
+        return {
+            'aws_access_key_id': creds['access-key'],
+            'aws_secret_access_key': creds['secret-key'],
+            }
+    else:
+        raise LookupError('No credentials found!')
+
+
+def is_china(region):
+    """Determine whether the supplied region is in AWS-China."""
+    return region.endpoint.endswith('.amazonaws.com.cn')
+
+
+def iter_region_connection(credentials, china_credentials):
+    """Iterate through connections for all regions except gov.
+
+    AWS-China regions will be connected using china_credentials.
+    US-GOV regions will be skipped.
+    All other regions will be connected using credentials.
+    """
+    regions = ec2.regions()
+    for region in regions:
+        if 'us-gov' in region.name:
+            continue
+        if is_china(region):
+            yield region.connect(**china_credentials)
+        else:
+            yield region.connect(**credentials)
+
+
+def iter_centos_images(credentials, china_credentials):
+    """Iterate through CentOS 7 images in standard AWS and AWS China."""
+    for conn in iter_region_connection(credentials, china_credentials):
+        images = conn.get_all_images(filters={
+            'owner_alias': 'aws-marketplace',
+            'product_code': 'aw0evgkw8e5c1q413zgy5pjce',
+            # 'name': 'CentOS Linux 7*',
+            })
+        for image in images:
+            yield image
+
+
+def make_item_name(region, vtype, store):
+    """Determine the item_name, given an image's attributes.
+
+    :param region: The region name
+    :param vtype: The virtualization type
+    :param store: The root device type.
+    """
+    # This is a port of code from simplestreams/tools/make-test-data, which is
+    # not provided as library code.
+    dmap = {
+        "north": "nn",
+        "northeast": "ne",
+        "east": "ee",
+        "southeast": "se",
+        "south": "ss",
+        "southwest": "sw",
+        "west": "ww",
+        "northwest": "nw",
+        "central": "cc",
+    }
+    itmap = {
+        'pv': {'instance': "pi", "ebs": "pe", "ssd": "es", "io1": "eo"},
+        'hvm': {'instance': "hi", "ebs": "he", "ssd": "hs", "io1": "ho"}
+    }
+    if store == "instance-store":
+        store = 'instance'
+    elif '-' in store:
+        store = store.split('-')[-1]
+    if vtype == "paravirtual":
+        vtype = "pv"
+
+    # create the item key:
+    #  - 2 letter country code (us) . 3 for govcloud (gww)
+    #  - 2 letter direction (nn=north, nw=northwest, cc=central)
+    #  - 1 digit number
+    #  - 1 char for virt type
+    #  - 1 char for root-store type
+
+    # Handle special case of 'gov' regions
+    pre_cc = ""
+    _region = region
+    if '-gov-' in region:
+        _region = region.replace('gov-', '')
+        pre_cc = "g"
+
+    (cc, direction, num) = _region.split("-")
+
+    ikey = pre_cc + cc + dmap[direction] + num + itmap[vtype][store]
+    return ikey
+
+
+def make_item(image, now):
+    """Convert Centos 7 Boto image to simplestreams Item.
+
+    :param now: the current datetime.
+    """
+    if image.architecture != 'x86_64':
+        raise ValueError(
+            'Architecture is "{}", not "x86_64".'.format(image.architecture))
+    if not image.name.startswith('CentOS Linux 7 '):
+        raise ValueError(
+            'Name "{}" does not begin with "CentOS Linux 7".'.format(
+                image.name))
+    item_name = make_item_name(image.region.name, image.virtualization_type,
+                               image.root_device_type)
+    version_name = now.strftime('%Y%m%d')
+    content_id = 'com.ubuntu.cloud.released:aws'
+    if is_china(image.region):
+        content_id = 'com.ubuntu.cloud.released:aws-cn'
+    else:
+        content_id = 'com.ubuntu.cloud.released:aws'
+    return Item(
+        content_id, 'com.ubuntu.cloud:server:centos7:amd64', version_name,
+        item_name, {
+            'endpoint': 'https://{}'.format(image.region.endpoint),
+            'region': image.region.name,
+            'arch': 'amd64',
+            'os': 'centos',
+            'virt': image.virtualization_type,
+            'id': image.id,
+            'version': 'centos7',
+            'label': 'release',
+            'release': 'centos7',
+            'release_codename': 'centos7',
+            'release_title': 'Centos 7',
+            'root_store': image.root_device_type,
+        })
+
+
+def write_streams(credentials, china_credentials, now, streams):
+    """Write image streams for Centos 7.
+
+    :param credentials: The standard AWS credentials.
+    :param china_credentials: The AWS China crentials.
+    :param now: The current datetime.
+    :param streams: The directory to store streams metadata in.
+    """
+    items = [make_item(i, now) for i in iter_centos_images(
+        credentials, china_credentials)]
+    updated = timestamp()
+    data = {'updated': updated, 'datatype': 'image-ids'}
+    trees = items2content_trees(items, data)
+    write_juju_streams(streams, trees, updated)
+
+
+def main():
+    streams, creds_filename = get_parameters()
+    with open(creds_filename) as creds_file:
+        all_credentials = yaml.safe_load(creds_file)['credentials']
+    credentials = make_aws_credentials(all_credentials['aws'])
+    china_credentials = make_aws_credentials(all_credentials['aws-china'])
+    now = datetime.utcnow()
+    write_streams(credentials, china_credentials, now, streams)
+
+
+if __name__ == '__main__':
+    main()

=== added file 'tests/test_make_aws_image_streams.py'
--- tests/test_make_aws_image_streams.py	1970-01-01 00:00:00 +0000
+++ tests/test_make_aws_image_streams.py	2016-03-17 14:53:09 +0000
@@ -0,0 +1,311 @@
+from datetime import datetime
+import json
+import os
+from StringIO import StringIO
+from unittest import TestCase
+
+from mock import (
+    Mock,
+    patch,
+    )
+
+from make_aws_image_streams import (
+    is_china,
+    iter_centos_images,
+    iter_region_connection,
+    get_parameters,
+    make_aws_credentials,
+    make_item,
+    make_item_name,
+    write_streams,
+    )
+from utils import temp_dir
+
+
+class TestIsChina(TestCase):
+
+    def test_is_china(self):
+        region = Mock()
+        region.endpoint = 'foo.amazonaws.com.cn'
+        self.assertIs(True, is_china(region))
+        region.endpoint = 'foo.amazonaws.com'
+        self.assertIs(False, is_china(region))
+
+
+def make_mock_region(stem, name=None, endpoint=None):
+    if endpoint is None:
+        endpoint = '{}-end'.format(stem)
+    region = Mock(endpoint=endpoint)
+    if name is None:
+        name = '{}-name'.format(stem)
+    region.name = name
+    return region
+
+
+class IterRegionConnection(TestCase):
+
+    def test_iter_region_connection(self):
+        east = make_mock_region('east')
+        west = make_mock_region('west')
+        aws = {}
+        with patch('make_aws_image_streams.ec2.regions', autospec=True,
+                   return_value=[east, west]) as regions_mock:
+            connections = [x for x in iter_region_connection(aws, None)]
+        regions_mock.assert_called_once_with()
+        self.assertEqual(
+            [east.connect.return_value, west.connect.return_value],
+            connections)
+        east.connect.assert_called_once_with(**aws)
+        west.connect.assert_called_once_with(**aws)
+
+    def test_gov_region(self):
+        east = make_mock_region('east')
+        gov = make_mock_region('west', name='foo-us-gov-bar')
+        aws = {}
+        with patch('make_aws_image_streams.ec2.regions', autospec=True,
+                   return_value=[east, gov]) as regions_mock:
+            connections = [x for x in iter_region_connection(aws, None)]
+        regions_mock.assert_called_once_with()
+        self.assertEqual(
+            [east.connect.return_value], connections)
+        east.connect.assert_called_once_with(**aws)
+        self.assertEqual(0, gov.connect.call_count)
+
+    def test_china_region(self):
+        east = make_mock_region('east')
+        west = make_mock_region('west', endpoint='west-end.amazonaws.com.cn')
+        east.name = 'east-name'
+        west.name = 'west-name'
+        aws = {'name': 'aws'}
+        aws_cn = {'name': 'aws-cn'}
+        with patch('make_aws_image_streams.ec2.regions', autospec=True,
+                   return_value=[east, west]) as regions_mock:
+            connections = [x for x in iter_region_connection(aws, aws_cn)]
+        regions_mock.assert_called_once_with()
+        self.assertEqual(
+            [east.connect.return_value, west.connect.return_value],
+            connections)
+        east.connect.assert_called_once_with(**aws)
+        west.connect.assert_called_once_with(**aws_cn)
+
+
+class IterCentosImages(TestCase):
+
+    def test_iter_centos_images(self):
+        aws = {'name': 'aws'}
+        aws_cn = {'name': 'aws-cn'}
+        east_imgs = ['east-1', 'east-2']
+        west_imgs = ['west-1', 'west-2']
+        east_conn = Mock()
+        east_conn.get_all_images.return_value = east_imgs
+        west_conn = Mock()
+        west_conn.get_all_images.return_value = west_imgs
+        with patch('make_aws_image_streams.iter_region_connection',
+                   return_value=[east_conn, west_conn],
+                   autospec=True) as irc_mock:
+            imgs = list(iter_centos_images(aws, aws_cn))
+        self.assertEqual(east_imgs + west_imgs, imgs)
+        irc_mock.assert_called_once_with(aws, aws_cn)
+        east_conn.get_all_images.assert_called_once_with(filters={
+            'owner_alias': 'aws-marketplace',
+            'product_code': 'aw0evgkw8e5c1q413zgy5pjce',
+            })
+        west_conn.get_all_images.assert_called_once_with(filters={
+            'owner_alias': 'aws-marketplace',
+            'product_code': 'aw0evgkw8e5c1q413zgy5pjce',
+            })
+
+
+class TestMakeAWSCredentials(TestCase):
+
+    def test_happy_path(self):
+        aws_credentials = make_aws_credentials({'credentials': {
+            'access-key': 'foo',
+            'secret-key': 'bar',
+            }})
+        self.assertEqual({
+            'aws_access_key_id': 'foo',
+            'aws_secret_access_key': 'bar',
+            }, aws_credentials)
+
+    def test_no_credentials(self):
+        with self.assertRaisesRegexp(LookupError, 'No credentials found!'):
+            make_aws_credentials({})
+
+    def test_multiple_credentials(self):
+        # If multiple credentials are present, an arbitrary credential will be
+        # used.
+        aws_credentials = make_aws_credentials({
+            'credentials-1': {
+                'access-key': 'foo',
+                'secret-key': 'bar',
+                },
+            'credentials-2': {
+                'access-key': 'baz',
+                'secret-key': 'qux',
+                },
+            })
+        self.assertIn(aws_credentials, [
+            {'aws_access_key_id': 'foo', 'aws_secret_access_key': 'bar'},
+            {'aws_access_key_id': 'baz', 'aws_secret_access_key': 'qux'},
+            ])
+
+
+def make_mock_image(region_name='us-northeast-3'):
+    image = Mock(virtualization_type='hvm', id='qux',
+                 root_device_type='ebs', architecture='x86_64')
+    image.name = 'CentOS Linux 7 foo'
+    image.region.endpoint = 'foo'
+    image.region.name = region_name
+    return image
+
+
+class TetMakeItem(TestCase):
+
+    def test_happy_path(self):
+        image = make_mock_image()
+        now = datetime(2001, 02, 03)
+        item = make_item(image, now)
+        self.assertEqual(item.content_id, 'com.ubuntu.cloud.released:aws')
+        self.assertEqual(item.product_name,
+                         'com.ubuntu.cloud:server:centos7:amd64')
+        self.assertEqual(item.item_name, 'usne3he')
+        self.assertEqual(item.version_name, '20010203')
+        self.assertEqual(item.data, {
+            'endpoint': 'https://foo',
+            'region': 'us-northeast-3',
+            'arch': 'amd64',
+            'os': 'centos',
+            'virt': 'hvm',
+            'id': 'qux',
+            'version': 'centos7',
+            'label': 'release',
+            'release': 'centos7',
+            'release_codename': 'centos7',
+            'release_title': 'Centos 7',
+            'root_store': 'ebs',
+            })
+
+    def test_china(self):
+        image = make_mock_image()
+        image.region.endpoint = 'foo.amazonaws.com.cn'
+        now = datetime(2001, 02, 03)
+        item = make_item(image, now)
+        self.assertEqual(item.content_id, 'com.ubuntu.cloud.released:aws-cn')
+        self.assertEqual(item.data['endpoint'], 'https://foo.amazonaws.com.cn')
+
+    def test_not_x86_64(self):
+        image = make_mock_image()
+        image.architecture = 'ppc128'
+        now = datetime(2001, 02, 03)
+        with self.assertRaisesRegexp(ValueError,
+                                     'Architecture is "ppc128", not'
+                                     ' "x86_64".'):
+            make_item(image, now)
+
+    def test_not_centos_7(self):
+        image = make_mock_image()
+        image.name = 'CentOS Linux 8'
+        now = datetime(2001, 02, 03)
+        with self.assertRaisesRegexp(ValueError,
+                                     'Name "CentOS Linux 8" does not begin'
+                                     ' with "CentOS Linux 7".'):
+            make_item(image, now)
+
+
+class TestGetParameters(TestCase):
+
+    def test_happy_path(self):
+        with patch.dict(os.environ, {'JUJU_DATA': 'foo'}):
+            streams, creds_filename = get_parameters(['bar'])
+        self.assertEqual(creds_filename, 'foo/credentials.yaml')
+        self.assertEqual(streams, 'bar')
+
+    def test_no_juju_data(self):
+        stderr = StringIO()
+        with self.assertRaises(SystemExit):
+            with patch('sys.stderr', stderr):
+                get_parameters(['bar'])
+        self.assertEqual(
+            stderr.getvalue(),
+            'JUJU_DATA must be set to a directory containing'
+            ' credentials.yaml.\n')
+
+
+class TestMakeItemName(TestCase):
+
+    def test_make_item_name(self):
+        item_name = make_item_name('us-east-1', 'paravirtual', 'instance')
+        self.assertEqual(item_name, 'usee1pi')
+        item_name = make_item_name('cn-northwest-3', 'hvm', 'ebs')
+        self.assertEqual(item_name, 'cnnw3he')
+
+
+def load_json(parent, filename):
+    with open(os.path.join(parent, 'streams', 'v1', filename)) as f:
+        return json.load(f)
+
+
+class TestWriteStreams(TestCase):
+
+    def test_write_streams(self):
+        now = datetime(2001, 02, 03)
+        credentials = {'name': 'aws'}
+        china_credentials = {'name': 'aws-cn'}
+        east_conn = Mock()
+        east_image = make_mock_image(region_name='us-east-1')
+        east_conn.get_all_images.return_value = [east_image]
+        west_conn = Mock()
+        west_image = make_mock_image(region_name='us-west-1')
+        west_conn.get_all_images.return_value = [west_image]
+        with temp_dir() as streams:
+            with patch('make_aws_image_streams.iter_region_connection',
+                       return_value=[east_conn, west_conn],
+                       autospec=True) as irc_mock:
+                with patch('make_aws_image_streams.timestamp',
+                           return_value='now'):
+                    with patch('sys.stderr'):
+                        write_streams(credentials, china_credentials, now,
+                                      streams)
+            index = load_json(streams, 'index.json')
+            index2 = load_json(streams, 'index2.json')
+            releases = load_json(streams, 'com.ubuntu.cloud.released-aws.json')
+            irc_mock.assert_called_once_with(credentials, china_credentials)
+        self.assertEqual(
+            {'format': 'index:1.0', 'updated': 'now', 'index': {}}, index)
+        self.assertEqual(
+            {'format': 'index:1.0', 'updated': 'now', 'index': {
+                'com.ubuntu.cloud.released:aws': {
+                    'format': 'products:1.0',
+                    'updated': 'now',
+                    'datatype': 'image-ids',
+                    'path': 'streams/v1/com.ubuntu.cloud.released-aws.json',
+                    'products': ['com.ubuntu.cloud:server:centos7:amd64'],
+                    }
+                }}, index2)
+        expected = {
+            'content_id': 'com.ubuntu.cloud.released:aws',
+            'format': 'products:1.0',
+            'updated': 'now',
+            'datatype': 'image-ids',
+            'products': {'com.ubuntu.cloud:server:centos7:amd64': {
+                'root_store': 'ebs',
+                'endpoint': 'https://foo',
+                'arch': 'amd64',
+                'release_title': 'Centos 7',
+                'label': 'release',
+                'release_codename': 'centos7',
+                'version': 'centos7',
+                'virt': 'hvm',
+                'release': 'centos7',
+                'os': 'centos',
+                'id': 'qux',
+                'versions': {'20010203': {
+                    'items': {
+                        'usww1he': {'region': 'us-west-1'},
+                        'usee1he': {'region': 'us-east-1'},
+                        }
+                    }},
+                }},
+            }
+        self.assertEqual(releases, expected)