Merge lp:~oddbloke/cloud-init/fix-gce-az into lp:~cloud-init-dev/cloud-init/trunk

[Dan Watkins]

This makes the full data source available to the mirror selection code, which means we can implement region logic on the data source for everything to use rather than it being mirror-selection-specific.

It also implements that region logic for EC2 and GCE.

[lp:~oddbloke/cloud-init/fix-gce-az]

1124. By Dan Watkins on 2015-07-22

Make full data source available to code that handles mirror selection.

1125. By Dan Watkins on 2015-07-22

Add DataSource.region and use it in mirror selection.

Also implement DataSource.region for EC2 and GCE data sources.

=== modified file 'cloudinit/distros/__init__.py'
--- cloudinit/distros/__init__.py	2015-06-02 20:27:57 +0000
+++ cloudinit/distros/__init__.py	2015-07-22 12:06:45 +0000
@@ -117,12 +117,11 @@
             arch = self.get_primary_arch()
         return _get_arch_package_mirror_info(mirror_info, arch)
 
-    def get_package_mirror_info(self, arch=None,
-                                availability_zone=None):
+    def get_package_mirror_info(self, arch=None, data_source=None):
         # This resolves the package_mirrors config option
         # down to a single dict of {mirror_name: mirror_url}
         arch_info = self._get_arch_package_mirror_info(arch)
-        return _get_package_mirror_info(availability_zone=availability_zone,
+        return _get_package_mirror_info(data_source=data_source,
                                         mirror_info=arch_info)
 
     def apply_network(self, settings, bring_up=True):
@@ -556,7 +555,7 @@
                 LOG.info("Added user '%s' to group '%s'" % (member, name))
 
 
-def _get_package_mirror_info(mirror_info, availability_zone=None,
+def _get_package_mirror_info(mirror_info, data_source=None,
                              mirror_filter=util.search_for_mirror):
     # given a arch specific 'mirror_info' entry (from package_mirrors)
     # search through the 'search' entries, and fallback appropriately
@@ -572,11 +571,14 @@
     ec2_az_re = ("^[a-z][a-z]-(%s)-[1-9][0-9]*[a-z]$" % directions_re)
 
     subst = {}
-    if availability_zone:
-        subst['availability_zone'] = availability_zone
-
-    if availability_zone and re.match(ec2_az_re, availability_zone):
-        subst['ec2_region'] = "%s" % availability_zone[0:-1]
+    if data_source and data_source.availability_zone:
+        subst['availability_zone'] = data_source.availability_zone
+
+        if re.match(ec2_az_re, data_source.availability_zone):
+            subst['ec2_region'] = "%s" % data_source.availability_zone[0:-1]
+
+    if data_source and data_source.region:
+        subst['region'] = data_source.region
 
     results = {}
     for (name, mirror) in mirror_info.get('failsafe', {}).items():

=== modified file 'cloudinit/sources/DataSourceEc2.py'
--- cloudinit/sources/DataSourceEc2.py	2015-01-21 22:56:53 +0000
+++ cloudinit/sources/DataSourceEc2.py	2015-07-22 12:06:45 +0000
@@ -197,6 +197,13 @@
         except KeyError:
             return None
 
+    @property
+    def region(self):
+        az = self.availability_zone
+        if az is not None:
+            return az[:-1]
+        return None
+
 # Used to match classes to dependencies
 datasources = [
   (DataSourceEc2, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),

=== modified file 'cloudinit/sources/DataSourceGCE.py'
--- cloudinit/sources/DataSourceGCE.py	2015-07-06 14:33:33 +0000
+++ cloudinit/sources/DataSourceGCE.py	2015-07-22 12:06:45 +0000
@@ -152,6 +152,10 @@
     def availability_zone(self):
         return self.metadata['availability-zone']
 
+    @property
+    def region(self):
+        return self.availability_zone.rsplit('-', 1)[0]
+
 # Used to match classes to dependencies
 datasources = [
     (DataSourceGCE, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),

=== modified file 'cloudinit/sources/__init__.py'
--- cloudinit/sources/__init__.py	2015-01-21 22:56:53 +0000
+++ cloudinit/sources/__init__.py	2015-07-22 12:06:45 +0000
@@ -157,6 +157,10 @@
         return self.metadata.get('availability-zone',
                                  self.metadata.get('availability_zone'))
 
+    @property
+    def region(self):
+        return self.metadata.get('region')
+
     def get_instance_id(self):
         if not self.metadata or 'instance-id' not in self.metadata:
             # Return a magic not really instance id string
@@ -210,8 +214,7 @@
             return hostname
 
     def get_package_mirror_info(self):
-        return self.distro.get_package_mirror_info(
-            availability_zone=self.availability_zone)
+        return self.distro.get_package_mirror_info(data_source=self)
 
 
 def normalize_pubkey_data(pubkey_data):

=== modified file 'config/cloud.cfg'
--- config/cloud.cfg	2015-03-04 15:10:11 +0000
+++ config/cloud.cfg	2015-07-22 12:06:45 +0000
@@ -104,6 +104,7 @@
          primary:
            - http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/
            - http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/
+           - http://%(region)s.clouds.archive.ubuntu.com/ubuntu/
          security: []
      - arches: [armhf, armel, default]
        failsafe:

=== modified file 'tests/unittests/test_distros/test_generic.py'
--- tests/unittests/test_distros/test_generic.py	2015-06-03 17:18:38 +0000
+++ tests/unittests/test_distros/test_generic.py	2015-07-22 12:06:45 +0000
@@ -7,6 +7,11 @@
 import shutil
 import tempfile
 
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+
 unknown_arch_info = {
     'arches': ['default'],
     'failsafe': {'primary': 'http://fs-primary-default',
@@ -144,33 +149,35 @@
 
     def test_get_package_mirror_info_az_ec2(self):
         arch_mirrors = gapmi(package_mirrors, arch="amd64")
+        data_source_mock = mock.Mock(availability_zone="us-east-1a")
 
-        results = gpmi(arch_mirrors, availability_zone="us-east-1a",
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
                        mirror_filter=self.return_first)
         self.assertEqual(results,
                          {'primary': 'http://us-east-1.ec2/',
                           'security': 'http://security-mirror1-intel'})
 
-        results = gpmi(arch_mirrors, availability_zone="us-east-1a",
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
                        mirror_filter=self.return_second)
         self.assertEqual(results,
                          {'primary': 'http://us-east-1a.clouds/',
                           'security': 'http://security-mirror2-intel'})
 
-        results = gpmi(arch_mirrors, availability_zone="us-east-1a",
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
                        mirror_filter=self.return_none)
         self.assertEqual(results, package_mirrors[0]['failsafe'])
 
     def test_get_package_mirror_info_az_non_ec2(self):
         arch_mirrors = gapmi(package_mirrors, arch="amd64")
+        data_source_mock = mock.Mock(availability_zone="nova.cloudvendor")
 
-        results = gpmi(arch_mirrors, availability_zone="nova.cloudvendor",
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
                        mirror_filter=self.return_first)
         self.assertEqual(results,
                          {'primary': 'http://nova.cloudvendor.clouds/',
                           'security': 'http://security-mirror1-intel'})
 
-        results = gpmi(arch_mirrors, availability_zone="nova.cloudvendor",
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
                        mirror_filter=self.return_last)
         self.assertEqual(results,
                          {'primary': 'http://nova.cloudvendor.clouds/',
@@ -178,17 +185,18 @@
 
     def test_get_package_mirror_info_none(self):
         arch_mirrors = gapmi(package_mirrors, arch="amd64")
+        data_source_mock = mock.Mock(availability_zone=None)
 
         # because both search entries here replacement based on
         # availability-zone, the filter will be called with an empty list and
         # failsafe should be taken.
-        results = gpmi(arch_mirrors, availability_zone=None,
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
                        mirror_filter=self.return_first)
         self.assertEqual(results,
                          {'primary': 'http://fs-primary-intel',
                           'security': 'http://security-mirror1-intel'})
 
-        results = gpmi(arch_mirrors, availability_zone=None,
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
                        mirror_filter=self.return_last)
         self.assertEqual(results,
                          {'primary': 'http://fs-primary-intel',