Merge lp:~oddbloke/ubuntu/precise/cloud-init/lp1374600 into lp:ubuntu/precise-proposed/cloud-init

[Dan Watkins]

This backports the GCE data source from trusty; the intent is to SRU it in the near future.

[Scott Moser (smoser)]

2 nit picks, other than that, assuming this is tested it looks fine.
The headers you're putting on the patch are described at http://dep.debian.net/deps/dep3/
so:
a.) add 'Origin: upstream'
    you can probably remove 'Author:' here (i realize we're probably not consistent in other patches on this).

b.) the bug number you referenced is a private bug (1374600), while there is a public bug (bug 1378441). Seems to make sense to reference the public bug.

c.) reference the ubuntu bug in the changelog message.

so make those changes, and i'll upload for you.

review: Needs Fixing

[lp:~oddbloke/ubuntu/precise/cloud-init/lp1374600]

212. By Dan Watkins on 2014-10-10

debian/patches/lp-1378441-backport-gce-data-source.patch: backported
Google Compute Engine data source. (LP: #1378441)

[Dan Watkins (oddbloke)]

I've just pushed up those changes.

=== modified file '.pc/applied-patches'
--- .pc/applied-patches	2014-09-17 19:44:07 +0000
+++ .pc/applied-patches	2014-10-10 08:21:23 +0000
@@ -24,3 +24,4 @@
 lp-1302229-fix_futils_azure.patch
 lp-1363260-add-cloudsigma_ds.patch
 lp-1336855-grub_xvda.patch
+lp-1378441-backport-gce-data-source.patch

=== added directory '.pc/lp-1378441-backport-gce-data-source.patch'
=== added directory '.pc/lp-1378441-backport-gce-data-source.patch/cloudinit'
=== added file '.pc/lp-1378441-backport-gce-data-source.patch/cloudinit/DataSourceGCE.py'
=== added file '.pc/lp-1378441-backport-gce-data-source.patch/cloudinit/url_helper.py'
=== added file 'cloudinit/DataSourceGCE.py'
--- cloudinit/DataSourceGCE.py	1970-01-01 00:00:00 +0000
+++ cloudinit/DataSourceGCE.py	2014-10-10 08:21:23 +0000
@@ -0,0 +1,156 @@
+# vi: ts=4 expandtab
+#
+#    Author: Vaidas Jablonskis <[e-mail]>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import urllib2
+
+from cloudinit import util
+from cloudinit import DataSource as sources
+
+LOG = logging.getLogger(__name__)
+
+BUILTIN_DS_CONFIG = {
+    'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/'
+}
+REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname')
+
+
+class _FakeResponse(object):
+
+    def __init__(self, code, contents):
+        self.code = code
+        self.contents = contents
+
+
+class url_helper(object):
+    # This is a shim designed to look like url_helper.py from future versions
+    # of cloud-init.
+
+    @staticmethod
+    def readurl(url, headers):
+        request = urllib2.Request(url, headers=headers)
+        response = urllib2.urlopen(request)
+        return _FakeResponse(code=response.getcode(), contents=response.read())
+
+    UrlError = urllib2.URLError
+
+
+class DataSourceGCE(sources.DataSource):
+    def __init__(self, sys_cfg):
+        sources.DataSource.__init__(self, sys_cfg)
+        self.metadata = dict()
+        self.ds_cfg = util.mergedict(
+            util.get_cfg_by_path(sys_cfg, ["datasource", "GCE"], {}),
+            BUILTIN_DS_CONFIG)
+        self.metadata_address = self.ds_cfg['metadata_url']
+
+    # GCE takes sshKeys attribute in the format of '<user>:<public_key>'
+    # so we have to trim each key to remove the username part
+    def _trim_key(self, public_key):
+        try:
+            index = public_key.index(':')
+            if index > 0:
+                return public_key[(index + 1):]
+        except:
+            return public_key
+
+    def get_data(self):
+        # GCE metadata server requires a custom header since v1
+        headers = {'X-Google-Metadata-Request': 'True'}
+
+        # url_map: (our-key, path, required)
+        url_map = [
+            ('instance-id', 'instance/id', True),
+            ('availability-zone', 'instance/zone', True),
+            ('local-hostname', 'instance/hostname', True),
+            ('public-keys', 'project/attributes/sshKeys', False),
+            ('user-data', 'instance/attributes/user-data', False),
+        ]
+
+        # if we cannot resolve the metadata server, then no point in trying
+        if not util.is_resolvable_url(self.metadata_address):
+            LOG.debug("%s is not resolvable", self.metadata_address)
+            return False
+
+        # iterate over url_map keys to get metadata items
+        found = False
+        for (mkey, path, required) in url_map:
+            try:
+                resp = url_helper.readurl(url=self.metadata_address + path,
+                                          headers=headers)
+                if resp.code == 200:
+                    found = True
+                    self.metadata[mkey] = resp.contents
+                else:
+                    if required:
+                        msg = "required url %s returned code %s. not GCE"
+                        if not found:
+                            LOG.debug(msg, path, resp.code)
+                        else:
+                            LOG.warn(msg, path, resp.code)
+                        return False
+                    else:
+                        self.metadata[mkey] = None
+            except url_helper.UrlError as e:
+                if required:
+                    msg = "required url %s raised exception %s. not GCE"
+                    if not found:
+                        LOG.debug(msg, path, e)
+                    else:
+                        LOG.warn(msg, path, e)
+                    return False
+                msg = "Failed to get %s metadata item: %s."
+                LOG.debug(msg, path, e)
+
+                self.metadata[mkey] = None
+
+        if self.metadata['public-keys']:
+            lines = self.metadata['public-keys'].splitlines()
+            self.metadata['public-keys'] = [self._trim_key(k) for k in lines]
+
+        return found
+
+    @property
+    def launch_index(self):
+        # GCE does not provide lauch_index property
+        return None
+
+    def get_instance_id(self):
+        return self.metadata['instance-id']
+
+    def get_public_ssh_keys(self):
+        return self.metadata['public-keys']
+
+    def get_hostname(self, fqdn=False, _resolve_ip=False):
+        return self.metadata['local-hostname']
+
+    @property
+    def userdata_raw(self):
+        return self.metadata['user-data'] or ''
+
+    @property
+    def availability_zone(self):
+        return self.metadata['availability-zone']
+
+# Used to match classes to dependencies
+datasources = [
+    (DataSourceGCE, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+]
+
+
+# Return a list of data sources that match this set of dependencies
+def get_datasource_list(depends):
+    return sources.list_from_depends(depends, datasources)

=== modified file 'debian/changelog'
--- debian/changelog	2014-09-18 22:26:16 +0000
+++ debian/changelog	2014-10-10 08:21:23 +0000
@@ -1,3 +1,10 @@
+cloud-init (0.6.3-0ubuntu1.15) precise; urgency=medium
+
+  * debian/patches/lp-1378441-backport-gce-data-source.patch: backported
+    Google Compute Engine data source. (LP: #1378441)
+
+ -- Daniel Watkins <[e-mail]>  Thu, 10 Oct 2014 09:24:57 +0100
+
 cloud-init (0.6.3-0ubuntu1.14) precise-proposed; urgency=medium
 
   * debian/patches/lp-1363260-add-cloudsigma_ds.patch: backport from

=== modified file 'debian/cloud-init.templates'
--- debian/cloud-init.templates	2014-09-17 19:44:07 +0000
+++ debian/cloud-init.templates	2014-10-10 08:21:23 +0000
@@ -1,8 +1,8 @@
 Template: cloud-init/datasources
 Type: multiselect
-Default: NoCloud, ConfigDrive, Azure, OVF, MAAS, CloudSigma
-Choices-C: NoCloud, ConfigDrive, Azure, OVF, MAAS, Ec2, SmartOS, CloudSigma
-Choices: NoCloud: Reads info from /var/lib/cloud/seed only, ConfigDrive: Reads data from Openstack Config Drive, Azure: read from MS Azure cdrom. Requires walinux-agent, OVF: Reads data from OVF Transports, MAAS: Reads data from Ubuntu MAAS, Ec2: reads data from EC2 Metadata service, SmartOS: reads data from serial console, CloudSigma: reads data from serial console
+Default: NoCloud, ConfigDrive, Azure, OVF, MAAS, CloudSigma, GCE
+Choices-C: NoCloud, ConfigDrive, Azure, OVF, MAAS, Ec2, SmartOS, CloudSigma, GCE
+Choices: NoCloud: Reads info from /var/lib/cloud/seed only, ConfigDrive: Reads data from Openstack Config Drive, Azure: read from MS Azure cdrom. Requires walinux-agent, OVF: Reads data from OVF Transports, MAAS: Reads data from Ubuntu MAAS, Ec2: reads data from EC2 Metadata service, SmartOS: reads data from serial console, CloudSigma: reads data from serial console, GCE: reads data from Google Compute Engine metadata service
 Description: Which data sources should be searched?
  Cloud-init supports searching different "Data Sources" for information
  that it uses to configure a cloud instance.

=== added file 'debian/patches/lp-1378441-backport-gce-data-source.patch'
--- debian/patches/lp-1378441-backport-gce-data-source.patch	1970-01-01 00:00:00 +0000
+++ debian/patches/lp-1378441-backport-gce-data-source.patch	2014-10-10 08:21:23 +0000
@@ -0,0 +1,170 @@
+Origin: upstream
+Bug: https://launchpad.net/bugs/1378441
+Forwarded: not-needed
+Description: Add support for Google Compute Engine
+ In order to use cloud-init on GCE, we need a GCE data source. This backports
+ the GCE datasource from trusty, DataSourceGCE.py.
+ .
+ This is accompanied by a debian/cloud-init.templates change to enable debconf
+ to select the 'GCE' datasource.
+Index: precise-proposed/cloudinit/DataSourceGCE.py
+===================================================================
+--- /dev/null	1970-01-01 00:00:00.000000000 +0000
++++ precise-proposed/cloudinit/DataSourceGCE.py	2014-10-07 15:43:05.054129022 +0100
+@@ -0,0 +1,156 @@
++# vi: ts=4 expandtab
++#
++#    Author: Vaidas Jablonskis <[e-mail]>
++#
++#    This program is free software: you can redistribute it and/or modify
++#    it under the terms of the GNU General Public License version 3, as
++#    published by the Free Software Foundation.
++#
++#    This program is distributed in the hope that it will be useful,
++#    but WITHOUT ANY WARRANTY; without even the implied warranty of
++#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
++#    GNU General Public License for more details.
++#
++#    You should have received a copy of the GNU General Public License
++#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
++
++import logging
++import urllib2
++
++from cloudinit import util
++from cloudinit import DataSource as sources
++
++LOG = logging.getLogger(__name__)
++
++BUILTIN_DS_CONFIG = {
++    'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/'
++}
++REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname')
++
++
++class _FakeResponse(object):
++
++    def __init__(self, code, contents):
++        self.code = code
++        self.contents = contents
++
++
++class url_helper(object):
++    # This is a shim designed to look like url_helper.py from future versions
++    # of cloud-init.
++
++    @staticmethod
++    def readurl(url, headers):
++        request = urllib2.Request(url, headers=headers)
++        response = urllib2.urlopen(request)
++        return _FakeResponse(code=response.getcode(), contents=response.read())
++
++    UrlError = urllib2.URLError
++
++
++class DataSourceGCE(sources.DataSource):
++    def __init__(self, sys_cfg):
++        sources.DataSource.__init__(self, sys_cfg)
++        self.metadata = dict()
++        self.ds_cfg = util.mergedict(
++            util.get_cfg_by_path(sys_cfg, ["datasource", "GCE"], {}),
++            BUILTIN_DS_CONFIG)
++        self.metadata_address = self.ds_cfg['metadata_url']
++
++    # GCE takes sshKeys attribute in the format of '<user>:<public_key>'
++    # so we have to trim each key to remove the username part
++    def _trim_key(self, public_key):
++        try:
++            index = public_key.index(':')
++            if index > 0:
++                return public_key[(index + 1):]
++        except:
++            return public_key
++
++    def get_data(self):
++        # GCE metadata server requires a custom header since v1
++        headers = {'X-Google-Metadata-Request': 'True'}
++
++        # url_map: (our-key, path, required)
++        url_map = [
++            ('instance-id', 'instance/id', True),
++            ('availability-zone', 'instance/zone', True),
++            ('local-hostname', 'instance/hostname', True),
++            ('public-keys', 'project/attributes/sshKeys', False),
++            ('user-data', 'instance/attributes/user-data', False),
++        ]
++
++        # if we cannot resolve the metadata server, then no point in trying
++        if not util.is_resolvable_url(self.metadata_address):
++            LOG.debug("%s is not resolvable", self.metadata_address)
++            return False
++
++        # iterate over url_map keys to get metadata items
++        found = False
++        for (mkey, path, required) in url_map:
++            try:
++                resp = url_helper.readurl(url=self.metadata_address + path,
++                                          headers=headers)
++                if resp.code == 200:
++                    found = True
++                    self.metadata[mkey] = resp.contents
++                else:
++                    if required:
++                        msg = "required url %s returned code %s. not GCE"
++                        if not found:
++                            LOG.debug(msg, path, resp.code)
++                        else:
++                            LOG.warn(msg, path, resp.code)
++                        return False
++                    else:
++                        self.metadata[mkey] = None
++            except url_helper.UrlError as e:
++                if required:
++                    msg = "required url %s raised exception %s. not GCE"
++                    if not found:
++                        LOG.debug(msg, path, e)
++                    else:
++                        LOG.warn(msg, path, e)
++                    return False
++                msg = "Failed to get %s metadata item: %s."
++                LOG.debug(msg, path, e)
++
++                self.metadata[mkey] = None
++
++        if self.metadata['public-keys']:
++            lines = self.metadata['public-keys'].splitlines()
++            self.metadata['public-keys'] = [self._trim_key(k) for k in lines]
++
++        return found
++
++    @property
++    def launch_index(self):
++        # GCE does not provide lauch_index property
++        return None
++
++    def get_instance_id(self):
++        return self.metadata['instance-id']
++
++    def get_public_ssh_keys(self):
++        return self.metadata['public-keys']
++
++    def get_hostname(self, fqdn=False, _resolve_ip=False):
++        return self.metadata['local-hostname']
++
++    @property
++    def userdata_raw(self):
++        return self.metadata['user-data'] or ''
++
++    @property
++    def availability_zone(self):
++        return self.metadata['availability-zone']
++
++# Used to match classes to dependencies
++datasources = [
++    (DataSourceGCE, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
++]
++
++
++# Return a list of data sources that match this set of dependencies
++def get_datasource_list(depends):
++    return sources.list_from_depends(depends, datasources)

=== modified file 'debian/patches/series'
--- debian/patches/series	2014-09-17 19:44:07 +0000
+++ debian/patches/series	2014-10-10 08:21:23 +0000
@@ -24,3 +24,4 @@
 lp-1302229-fix_futils_azure.patch
 lp-1363260-add-cloudsigma_ds.patch
 lp-1336855-grub_xvda.patch
+lp-1378441-backport-gce-data-source.patch