Merge lp:~abentley/juju-release-tools/azure-arm-streams-latest into lp:juju-release-tools

[Aaron Bentley]

Get ubuntu images by sku

This branch changes the azurm-arm streams to use sku to look up ubuntu images, rather than attempting to convert cloudware images.

It also causes them to use 'latest' rather than a specific version, which matches the current Juju behaviour. Supplying Ubuntu images this way is intended as a stopgap until Cloudware can publish Azure streams.

It also forces the id to be provided at the lowest level, even in cases where this is redundant. This works around a Juju bug.

Importing simplestreams has the side effect of configuring logging (in a not-helpful way, i.e. disabling all output). This branch works around this by running logging.basicConfig before simplestreams is imported.

It also adds some windows image types and expected-missing cloudware images.

[Curtis Hovey (sinzui)]

Thank you.

review: Approve (code)

=== modified file 'azure_image_streams.py'
--- azure_image_streams.py	2016-12-12 14:24:50 +0000
+++ azure_image_streams.py	2016-12-12 14:24:50 +0000
@@ -1,6 +1,5 @@
 import logging
 import re
-import sys
 
 # Tested on Azure 2.0 rc5 API
 from azure.common.credentials import ServicePrincipalCredentials
@@ -16,6 +15,8 @@
 from simplestreams import mirrors
 from simplestreams import util
 
+from build_package import juju_series
+
 
 CANONICAL = 'Canonical'
 MS_VSTUDIO = 'MicrosoftVisualStudio'
@@ -24,10 +25,12 @@
 WINDOWS = 'Windows'
 WINDOWS_SERVER = 'WindowsServer'
 IMAGE_SPEC = [
-    ('win81', MS_VSTUDIO, WINDOWS, '8.1-Enterprise-N'),
-    ('win10', MS_VSTUDIO, WINDOWS, '10-Enterprise-N'),
+    ('win81', MS_VSTUDIO, WINDOWS, 'Win8.1-Ent-N'),
+    ('win10', MS_VSTUDIO, WINDOWS, 'Windows-10-N-x64'),
     ('win2012', MS_SERVER, WINDOWS_SERVER, '2012-Datacenter'),
     ('win2012r2', MS_SERVER, WINDOWS_SERVER, '2012-R2-Datacenter'),
+    ('win2016', MS_SERVER, WINDOWS_SERVER, '2016-Datacenter'),
+    ('win2016nano', MS_SERVER, WINDOWS_SERVER, '2016-Nano-Server'),
     ('centos7', 'OpenLogic', 'CentOS', '7.1'),
 ]
 
@@ -64,9 +67,17 @@
 }
 
 
+def logger():
+    return logging.getLogger('azure_image_streams')
+
+
 # Thorough investigation has not found an equivalent for these in the
 # Azure-ARM image repository.
-EXPECTED_MISSING = frozenset({('12.04.2-LTS', '12.04.201212180')})
+EXPECTED_MISSING = frozenset({
+    ('12.04.2-LTS', '12.04.201212180'),
+    ('16.04.0-LTS', '16.04.201611220'),
+    ('16.04.0-LTS', '16.04.201611300'),
+    })
 
 
 class MissingImage(Exception):
@@ -202,14 +213,14 @@
     spec = full_spec[1:]
     location_versions = {}
     for location in locations:
-        logging.info('Retrieving image data in {}'.format(
+        logger().debug('Retrieving image data in {}'.format(
             location.display_name))
         try:
             versions = client.virtual_machine_images.list(location.name,
                                                           *spec)
         except CloudError:
             template = 'Could not find {} {} {} in {location}'
-            logging.warning(template.format(
+            logger().warning(template.format(
                 *spec, location=location.display_name))
             continue
         for version in versions:
@@ -226,7 +237,8 @@
                             endpoint)
 
 
-def make_item(version_name, urn_version, full_spec, location_name, endpoint):
+def make_item(version_name, urn_version, full_spec, location_name, endpoint,
+              stream='released', item_version=None, release=None):
     """Make a simplestreams Item for a version.
 
     Version name is the simplestreams version_name.
@@ -239,8 +251,12 @@
     """
     URN = ':'.join(full_spec[1:] + (urn_version,))
     product_name = 'com.ubuntu.cloud:server:{}:amd64'.format(full_spec[0])
+    if release is None:
+        release = full_spec[0]
+    if item_version is None:
+        item_version = full_spec[0]
     return Item(
-        'com.ubuntu.cloud:released:azure',
+        'com.ubuntu.cloud:{}:azure'.format(stream),
         product_name,
         version_name, ITEM_NAMES[location_name], {
             'arch': 'amd64',
@@ -249,8 +265,8 @@
             'id': URN,
             'label': 'release',
             'endpoint': endpoint,
-            'release': full_spec[0],
-            'version': full_spec[0],
+            'release': release,
+            'version': item_version,
             }
         )
 
@@ -292,16 +308,56 @@
     sub_client = SubscriptionClient(credentials)
     client = ComputeManagementClient(credentials, subscription_id)
     locations = sub_client.subscriptions.list_locations(subscription_id)
-    ci_items = ItemList.items_from_url(
-        'http://cloud-images.ubuntu.com/releases/streams/v1')
-    items, unknown_locations = convert_cloud_images_items(
-        client, locations, ci_items)
-    sys.stderr.write('Unknown locations: {}\n'.format(', '.join(
-        sorted(unknown_locations))))
+    items = find_ubuntu_items(client, locations)
     items.extend(find_spec_items(client, locations))
     return items
 
 
+def make_ubuntu_item(endpoint, location_name, sku_name):
+    match = re.match(r'(\d\d\.\d\d)(\.\d+)?-?(.*)', sku_name)
+    if match is None:
+        logger().info('Skipping {}'.format(sku_name))
+        return None
+    tag = match.group(3)
+    if tag in ('DAILY', 'DAILY-LTS'):
+        stream = 'daily'
+    elif tag in ('', 'LTS'):
+        stream = 'released'
+    else:
+        logger().info('Skipping {}'.format(sku_name))
+        return None
+    minor_version = match.group(1)
+    try:
+        release = juju_series.get_name(minor_version)
+    except KeyError:
+        logger().warning("Can't find name for {}".format(release))
+        return None
+    full_spec = (minor_version, CANONICAL, UBUNTU_SERVER, sku_name)
+    return make_item(sku_name, 'latest', full_spec, location_name,
+                     endpoint, stream=stream, release=release)
+
+
+def find_ubuntu_items(client, locations):
+    """Make simplestreams Items for existing Azure images.
+
+    All versions of all images matching IMAGE_SPEC will be returned.
+
+    all_credentials is a dict of credentials in the credentials.yaml
+    structure, used to create Azure credentials.
+    """
+    items = []
+    for location in locations:
+        skus = client.virtual_machine_images.list_skus(
+            location.name, CANONICAL, UBUNTU_SERVER)
+        for sku in skus:
+            item = make_ubuntu_item(client.config.base_url, location.name,
+                                    sku.name)
+            if item is None:
+                continue
+            items.append(item)
+    return items
+
+
 def find_spec_items(client, locations):
     """Make simplestreams Items for existing Azure images.
 

=== modified file 'make_image_streams.py'
--- make_image_streams.py	2016-07-21 19:33:35 +0000
+++ make_image_streams.py	2016-12-12 14:24:50 +0000
@@ -4,6 +4,9 @@
 from argparse import ArgumentParser
 from copy import deepcopy
 from datetime import datetime
+import logging
+# Done early to prevent Simplestreams from messing with the log configuration.
+logging.basicConfig(level=logging.INFO)
 import os
 import sys
 from textwrap import dedent
@@ -22,6 +25,8 @@
     )
 from simplestreams import util
 
+log = logging.getLogger(
+    "requests.packages.urllib3.connectionpool").setLevel(logging.WARNING)
 
 AWS = 'aws'
 AZURE = 'azure'
@@ -215,7 +220,7 @@
     data = {'updated': updated, 'datatype': 'image-ids'}
     trees = items2content_trees(items, data)
     write_juju_streams(out_dir, trees, updated, [
-        'path', 'sha256', 'md5', 'size', 'virt', 'root_store'])
+        'path', 'sha256', 'md5', 'size', 'virt', 'root_store', 'id'])
 
 
 def write_juju_streams(out_d, trees, updated, sticky):

=== modified file 'tests/test_azure_image_streams.py'
--- tests/test_azure_image_streams.py	2016-12-12 14:24:50 +0000
+++ tests/test_azure_image_streams.py	2016-12-12 14:24:50 +0000
@@ -13,12 +13,14 @@
     CANONICAL,
     convert_cloud_images_items,
     convert_item_to_arm,
+    find_ubuntu_items,
     get_azure_credentials,
     IMAGE_SPEC,
     make_spec_items,
     MissingImage,
     make_item,
     make_azure_items,
+    make_ubuntu_item,
     parse_id,
     UBUNTU_SERVER,
     UnexpectedImage,
@@ -287,7 +289,11 @@
     client = Mock(spec=['config', 'virtual_machine_images'])
     client.virtual_machine_images.list.return_value = [
         mock_version(v) for v in versions]
+    client.virtual_machine_images.list_skus.return_value = [
+        mock_sku('12.04.2-LTS'),
+        ]
     client.config.base_url = 'http://example.com/arm'
+
     return client
 
 
@@ -303,6 +309,12 @@
     return location
 
 
+def mock_sku(name):
+    sku = Mock()
+    sku.name = name
+    return sku
+
+
 def make_expected(client, versions, specs):
     expected_items = []
     expected_calls = []
@@ -351,19 +363,89 @@
         expected_calls, expected_items = make_expected(client, ['3'],
                                                        IMAGE_SPEC)
         location = mock_location('canadaeast', 'Canada East')
-        old_item, spec, expected_item = make_item_expected(
-            region=location.display_name, endpoint=client.config.base_url)
-        expected_items.insert(0, expected_item)
-        with self.mai_cxt(location, client, [old_item]):
+        expected_items.insert(
+            0, make_item('12.04.2-LTS', 'latest', (
+                '12.04', CANONICAL, UBUNTU_SERVER, '12.04.2-LTS'
+                ), 'canadaeast', client.config.base_url, release='precise'))
+        with self.mai_cxt(location, client, []):
             items = make_azure_items(all_credentials)
         self.assertEqual(expected_items, items)
 
     def test_make_azure_items_no_ubuntu(self):
         all_credentials = make_all_credentials()
         client = mock_compute_client(['3'])
+        client.virtual_machine_images.list_skus.return_value = []
         expected_calls, expected_items = make_expected(client, ['3'],
                                                        IMAGE_SPEC)
         location = mock_location('canadaeast', 'Canada East')
         with self.mai_cxt(location, client, []):
             items = make_azure_items(all_credentials)
         self.assertEqual(expected_items, items)
+
+
+class TestMakeUbuntuItem(TestCase):
+
+    def make_item(self, full_version='12.04.5-LTS', daily=False):
+        stream = 'daily' if daily else 'released'
+        return make_item(full_version, 'latest', (
+            '12.04', CANONICAL, UBUNTU_SERVER, full_version,
+            ), 'canadaeast', 'http://example.com', release='precise',
+            stream=stream)
+
+    def test_make_ubuntu_item(self):
+        item = make_ubuntu_item('http://example.com', 'canadaeast',
+                                '12.04.5-LTS')
+        self.assertEqual(item, self.make_item())
+
+    def test_no_lts(self):
+        item = make_ubuntu_item('http://example.com', 'canadaeast',
+                                '12.04.5')
+        self.assertEqual(item, self.make_item('12.04.5'))
+
+    def test_daily(self):
+        item = make_ubuntu_item('http://example.com', 'canadaeast',
+                                '12.04.5-DAILY')
+        self.assertEqual(item.content_id, 'com.ubuntu.cloud:daily:azure')
+
+    def test_daily_lts(self):
+        item = make_ubuntu_item('http://example.com', 'canadaeast',
+                                '12.04.5-DAILY-LTS')
+        self.assertEqual(item.content_id, 'com.ubuntu.cloud:daily:azure')
+
+    def test_unknown_tag(self):
+        item = make_ubuntu_item('http://example.com', 'canadaeast',
+                                '12.04.5-FOOBAR')
+        self.assertIs(item, None)
+
+    def test_not_a_version(self):
+        item = make_ubuntu_item('http://example.com', 'canadaeast',
+                                '12.q.5')
+        self.assertIs(item, None)
+
+    def test_xenial(self):
+        item = make_ubuntu_item('http://example.com', 'canadaeast',
+                                '16.04.5-LTS')
+        self.assertEqual('xenial', item.data['release'])
+
+    def test_version(self):
+        item = make_ubuntu_item('http://example.com', 'canadaeast',
+                                '16.04.5-LTS')
+        self.assertEqual('16.04', item.data['version'])
+        self.assertIn(':16.04:', item.product_name)
+
+
+class TestFindUbuntuItems(TestCase):
+
+    def test_find_ubuntu_items(self):
+        sku = Mock()
+        sku.name = '16.04.3-LTS'
+        bad_sku = Mock()
+        bad_sku.name = '16.0q.3-LTS'
+        client = Mock()
+        client.virtual_machine_images.list_skus.return_value = [sku, bad_sku]
+        client.config.base_url = 'http://example.com'
+        location = Mock()
+        location.name = 'canadaeast'
+        items = find_ubuntu_items(client, [location])
+        self.assertEqual(items, [make_ubuntu_item(
+            'http://example.com', 'canadaeast', '16.04.3-LTS')])

=== modified file 'tests/test_make_image_streams.py'
--- tests/test_make_image_streams.py	2016-07-20 14:00:52 +0000
+++ tests/test_make_image_streams.py	2016-12-12 14:24:50 +0000
@@ -355,15 +355,16 @@
                 'version': 'centos7',
                 'release': 'centos7',
                 'os': 'centos',
-                'id': 'qux',
                 'versions': {'20010203': {
                     'items': {
                         'usww1he': {
+                            'id': 'qux',
                             'region': 'us-west-1',
                             'root_store': 'ebs',
                             'virt': 'hvm',
                             },
                         'usee1he': {
+                            'id': 'qux',
                             'region': 'us-east-1',
                             'root_store': 'ebs',
                             'virt': 'hvm',