Merge lp:~harlowja/cloud-init/boto-ec2-replace into lp:~cloud-init-dev/cloud-init/trunk

[Joshua Harlow]

[lp:~harlowja/cloud-init/boto-ec2-replace]

906. By Joshua Harlow on 2014-01-11

Add ec2 utils tests and httpretty requirement for http mocking

907. By Joshua Harlow on 2014-01-11

Add a maybe_json helper function

908. By Joshua Harlow on 2014-01-11

Only check for json objects instead of also arrays

909. By Joshua Harlow on 2014-01-11

Updated non-json message

=== modified file 'Requires'
--- Requires	2013-09-09 05:36:28 +0000
+++ Requires	2014-01-11 01:39:20 +0000
@@ -29,8 +29,8 @@
 # Requests handles ssl correctly!
 requests
 
-# Boto for ec2
-boto
-
 # For patching pieces of cloud-config together
 jsonpatch
+
+# For http testing (only needed for testing)
+httpretty>=0.7.1

=== modified file 'cloudinit/ec2_utils.py'
--- cloudinit/ec2_utils.py	2013-03-20 12:34:19 +0000
+++ cloudinit/ec2_utils.py	2014-01-11 01:39:20 +0000
@@ -16,48 +16,163 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import boto.utils as boto_utils
-
-# Versions of boto >= 2.6.0 (and possibly 2.5.2)
-# try to lazily load the metadata backing, which
-# doesn't work so well in cloud-init especially
-# since the metadata is serialized and actions are
-# performed where the metadata server may be blocked
-# (thus the datasource will start failing) resulting
-# in url exceptions when fields that do exist (or
-# would have existed) do not exist due to the blocking
-# that occurred.
-
-# TODO(harlowja): https://github.com/boto/boto/issues/1401
-# When boto finally moves to using requests, we should be able
-# to provide it ssl details, it does not yet, so we can't provide them...
-
-
-def _unlazy_dict(mp):
-    if not isinstance(mp, (dict)):
-        return mp
-    # Walk over the keys/values which
-    # forces boto to unlazy itself and
-    # has no effect on dictionaries that
-    # already have there items.
-    for (_k, v) in mp.items():
-        _unlazy_dict(v)
-    return mp
-
-
-def get_instance_userdata(api_version, metadata_address):
-    # Note: boto.utils.get_instance_metadata returns '' for empty string
-    # so the change from non-true to '' is not specifically necessary, but
-    # this way cloud-init will get consistent behavior even if boto changed
-    # in the future to return a None on "no user-data provided".
-    ud = boto_utils.get_instance_userdata(api_version, None, metadata_address)
-    if not ud:
-        ud = ''
-    return ud
-
-
-def get_instance_metadata(api_version, metadata_address):
-    metadata = boto_utils.get_instance_metadata(api_version, metadata_address)
-    if not isinstance(metadata, (dict)):
-        metadata = {}
-    return _unlazy_dict(metadata)
+from urlparse import (urlparse, urlunparse)
+
+import functools
+import json
+import urllib
+
+from cloudinit import log as logging
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+
+def maybe_json_object(text):
+    if not text:
+        return False
+    text = text.strip()
+    if text.startswith("{") and text.endswith("}"):
+        return True
+    return False
+
+
+def combine_url(base, add_on):
+    base_parsed = list(urlparse(base))
+    path = base_parsed[2]
+    if path and not path.endswith("/"):
+        path += "/"
+    path += urllib.quote(str(add_on), safe="/:")
+    base_parsed[2] = path
+    return urlunparse(base_parsed)
+
+
+# See: http://bit.ly/TyoUQs
+#
+# Since boto metadata reader uses the old urllib which does not
+# support ssl, we need to ahead and create our own reader which
+# works the same as the boto one (for now).
+class MetadataMaterializer(object):
+    def __init__(self, blob, base_url, caller):
+        self._blob = blob
+        self._md = None
+        self._base_url = base_url
+        self._caller = caller
+
+    def _parse(self, blob):
+        leaves = {}
+        children = []
+        if not blob:
+            return (leaves, children)
+
+        def has_children(item):
+            if item.endswith("/"):
+                return True
+            else:
+                return False
+
+        def get_name(item):
+            if item.endswith("/"):
+                return item.rstrip("/")
+            return item
+
+        for field in blob.splitlines():
+            field = field.strip()
+            field_name = get_name(field)
+            if not field or not field_name:
+                continue
+            if has_children(field):
+                if field_name not in children:
+                    children.append(field_name)
+            else:
+                contents = field.split("=", 1)
+                resource = field_name
+                if len(contents) > 1:
+                    # What a PITA...
+                    (ident, sub_contents) = contents
+                    ident = util.safe_int(ident)
+                    if ident is not None:
+                        resource = "%s/openssh-key" % (ident)
+                        field_name = sub_contents
+                leaves[field_name] = resource
+        return (leaves, children)
+
+    def materialize(self):
+        if self._md is not None:
+            return self._md
+        self._md = self._materialize(self._blob, self._base_url)
+        return self._md
+
+    def _decode_leaf_blob(self, field, blob):
+        if not blob:
+            return blob
+        if maybe_json_object(blob):
+            try:
+                # Assume it's json, unless it fails parsing...
+                return json.loads(blob)
+            except (ValueError, TypeError) as e:
+                LOG.warn("Field %s looked like a json object, but it was"
+                         " not: %s", field, e)
+        if blob.find("\n") != -1:
+            return blob.splitlines()
+        return blob
+
+    def _materialize(self, blob, base_url):
+        (leaves, children) = self._parse(blob)
+        child_contents = {}
+        for c in children:
+            child_url = combine_url(base_url, c)
+            if not child_url.endswith("/"):
+                child_url += "/"
+            child_blob = str(self._caller(child_url))
+            child_contents[c] = self._materialize(child_blob, child_url)
+        leaf_contents = {}
+        for (field, resource) in leaves.items():
+            leaf_url = combine_url(base_url, resource)
+            leaf_blob = str(self._caller(leaf_url))
+            leaf_contents[field] = self._decode_leaf_blob(field, leaf_blob)
+        joined = {}
+        joined.update(child_contents)
+        for field in leaf_contents.keys():
+            if field in joined:
+                LOG.warn("Duplicate key found in results from %s", base_url)
+            else:
+                joined[field] = leaf_contents[field]
+        return joined
+
+
+def get_instance_userdata(api_version='latest',
+                          metadata_address='http://169.254.169.254',
+                          ssl_details=None, timeout=5, retries=5):
+    ud_url = combine_url(metadata_address, api_version)
+    ud_url = combine_url(ud_url, 'user-data')
+    try:
+        response = util.read_file_or_url(ud_url,
+                                         ssl_details=ssl_details,
+                                         timeout=timeout,
+                                         retries=retries)
+        return str(response)
+    except Exception:
+        util.logexc(LOG, "Failed fetching userdata from url %s", ud_url)
+        return ''
+
+
+def get_instance_metadata(api_version='latest',
+                          metadata_address='http://169.254.169.254',
+                          ssl_details=None, timeout=5, retries=5):
+    md_url = combine_url(metadata_address, api_version)
+    md_url = combine_url(md_url, 'meta-data')
+    caller = functools.partial(util.read_file_or_url,
+                               ssl_details=ssl_details, timeout=timeout,
+                               retries=retries)
+
+    try:
+        response = caller(md_url)
+        materializer = MetadataMaterializer(str(response), md_url, caller)
+        md = materializer.materialize()
+        if not isinstance(md, (dict)):
+            md = {}
+        return md
+    except Exception:
+        util.logexc(LOG, "Failed fetching metadata from url %s", md_url)
+        return {}

=== added file 'tests/unittests/test_ec2_util.py'
--- tests/unittests/test_ec2_util.py	1970-01-01 00:00:00 +0000
+++ tests/unittests/test_ec2_util.py	2014-01-11 01:39:20 +0000
@@ -0,0 +1,133 @@
+from tests.unittests import helpers
+
+from cloudinit import ec2_utils as eu
+
+import httpretty as hp
+
+
+class TestEc2Util(helpers.TestCase):
+    VERSION = 'latest'
+
+    @hp.activate
+    def test_userdata_fetch(self):
+        hp.register_uri(hp.GET,
+                        'http://169.254.169.254/%s/user-data' % (self.VERSION),
+                        body='stuff',
+                        status=200)
+        userdata = eu.get_instance_userdata(self.VERSION)
+        self.assertEquals('stuff', userdata)
+
+    @hp.activate
+    def test_userdata_fetch_fail_not_found(self):
+        hp.register_uri(hp.GET,
+                        'http://169.254.169.254/%s/user-data' % (self.VERSION),
+                        status=404)
+        userdata = eu.get_instance_userdata(self.VERSION, retries=0)
+        self.assertEquals('', userdata)
+
+
+    @hp.activate
+    def test_userdata_fetch_fail_server_dead(self):
+        hp.register_uri(hp.GET,
+                        'http://169.254.169.254/%s/user-data' % (self.VERSION),
+                        status=500)
+        userdata = eu.get_instance_userdata(self.VERSION, retries=0)
+        self.assertEquals('', userdata)
+
+    @hp.activate
+    def test_metadata_fetch_no_keys(self):
+        base_url = 'http://169.254.169.254/%s/meta-data' % (self.VERSION)
+        hp.register_uri(hp.GET, base_url, status=200,
+                        body="\n".join(['hostname',
+                                        'instance-id',
+                                        'ami-launch-index']))
+        hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'),
+                        status=200, body='ec2.fake.host.name.com')
+        hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'),
+                        status=200, body='123')
+        hp.register_uri(hp.GET, eu.combine_url(base_url, 'ami-launch-index'),
+                        status=200, body='1')
+        md = eu.get_instance_metadata(self.VERSION, retries=0)
+        self.assertEquals(md['hostname'], 'ec2.fake.host.name.com')
+        self.assertEquals(md['instance-id'], '123')
+        self.assertEquals(md['ami-launch-index'], '1')
+
+    @hp.activate
+    def test_metadata_fetch_key(self):
+        base_url = 'http://169.254.169.254/%s/meta-data' % (self.VERSION)
+        hp.register_uri(hp.GET, base_url, status=200,
+                        body="\n".join(['hostname',
+                                        'instance-id',
+                                        'public-keys/']))
+        hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'),
+                        status=200, body='ec2.fake.host.name.com')
+        hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'),
+                        status=200, body='123')
+        hp.register_uri(hp.GET, eu.combine_url(base_url, 'public-keys/'),
+                        status=200, body='0=my-public-key')
+        hp.register_uri(hp.GET, eu.combine_url(base_url, 'public-keys/'),
+                        status=200, body='0=my-public-key')
+        hp.register_uri(hp.GET,
+                        eu.combine_url(base_url, 'public-keys/0/openssh-key'),
+                        status=200, body='ssh-rsa AAAA.....wZEf my-public-key')
+        md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
+        self.assertEquals(md['hostname'], 'ec2.fake.host.name.com')
+        self.assertEquals(md['instance-id'], '123')
+        self.assertEquals(1, len(md['public-keys']))
+
+    @hp.activate
+    def test_metadata_fetch_key(self):
+        base_url = 'http://169.254.169.254/%s/meta-data' % (self.VERSION)
+        hp.register_uri(hp.GET, base_url, status=200,
+                        body="\n".join(['hostname',
+                                        'instance-id',
+                                        'public-keys/']))
+        hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'),
+                        status=200, body='ec2.fake.host.name.com')
+        hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'),
+                        status=200, body='123')
+        hp.register_uri(hp.GET, eu.combine_url(base_url, 'public-keys/'),
+                        status=200,
+                        body="\n".join(['0=my-public-key', '1=my-other-key']))
+        hp.register_uri(hp.GET,
+                        eu.combine_url(base_url, 'public-keys/0/openssh-key'),
+                        status=200, body='ssh-rsa AAAA.....wZEf my-public-key')
+        hp.register_uri(hp.GET,
+                        eu.combine_url(base_url, 'public-keys/1/openssh-key'),
+                        status=200, body='ssh-rsa AAAA.....wZEf my-other-key')
+        md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
+        self.assertEquals(md['hostname'], 'ec2.fake.host.name.com')
+        self.assertEquals(md['instance-id'], '123')
+        self.assertEquals(2, len(md['public-keys']))
+
+    @hp.activate
+    def test_metadata_fetch_bdm(self):
+        base_url = 'http://169.254.169.254/%s/meta-data' % (self.VERSION)
+        hp.register_uri(hp.GET, base_url, status=200,
+                        body="\n".join(['hostname',
+                                        'instance-id',
+                                        'block-device-mapping/']))
+        hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'),
+                        status=200, body='ec2.fake.host.name.com')
+        hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'),
+                        status=200, body='123')
+        hp.register_uri(hp.GET,
+                        eu.combine_url(base_url, 'block-device-mapping/'),
+                        status=200,
+                        body="\n".join(['ami', 'ephemeral0']))
+        hp.register_uri(hp.GET,
+                        eu.combine_url(base_url, 'block-device-mapping/ami'),
+                        status=200,
+                        body="sdb")
+        hp.register_uri(hp.GET,
+                        eu.combine_url(base_url,
+                                       'block-device-mapping/ephemeral0'),
+                        status=200,
+                        body="sdc")
+        md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1)
+        self.assertEquals(md['hostname'], 'ec2.fake.host.name.com')
+        self.assertEquals(md['instance-id'], '123')
+        bdm = md['block-device-mapping']
+        self.assertEquals(2, len(bdm))
+        self.assertEquals(bdm['ami'], 'sdb')
+        self.assertEquals(bdm['ephemeral0'], 'sdc')