Merge ~chad.smith/cloud-init:ubuntu/devel into cloud-init:ubuntu/devel

[Chad Smith]

Sync upstream master for Bionic release.

[Chad Smith (chad.smith)]

tox is happy with this branch. Thanks for the fix smoser.
  py27: commands succeeded
  py3: commands succeeded
  flake8: commands succeeded
  xenial: commands succeeded
  pylint: commands succeeded
  congratulations :)

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:23919e31120edf37c7ff7f9cba3eda4f46b6f5cb
https://jenkins.ubuntu.com/server/job/cloud-init-ci/727/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Unit & Style Tests
    SUCCESS: Ubuntu LTS: Build
    SUCCESS: Ubuntu LTS: Integration
    SUCCESS: MAAS Compatability Testing
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/727/rebuild

review: Approve (continuous-integration)

diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 870b368..b8db626 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -25,7 +25,7 @@ DEFAULT_METADATA = {
     "instance-id": DEFAULT_IID,
 }
 FS_TYPES = ('vfat', 'iso9660')
-LABEL_TYPES = ('config-2',)
+LABEL_TYPES = ('config-2', 'CONFIG-2')
 POSSIBLE_MOUNTS = ('sr', 'cd')
 OPTICAL_DEVICES = tuple(('/dev/%s%s' % (z, i) for z in POSSIBLE_MOUNTS
                         for i in range(0, 2)))
@@ -224,7 +224,7 @@ def find_candidate_devs(probe_optical=True):
     config drive v2:
        Disk should be:
         * either vfat or iso9660 formated
-        * labeled with 'config-2'
+        * labeled with 'config-2' or 'CONFIG-2'
     """
     # query optical drive to get it in blkid cache for 2.6 kernels
     if probe_optical:
diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py
index ad6dae3..2da34a9 100644
--- a/cloudinit/sources/DataSourceGCE.py
+++ b/cloudinit/sources/DataSourceGCE.py
@@ -2,8 +2,12 @@
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
+import datetime
+import json
+
 from base64 import b64decode
 
+from cloudinit.distros import ug_util
 from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import url_helper
@@ -17,16 +21,18 @@ REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname')
 
 
 class GoogleMetadataFetcher(object):
-    headers = {'X-Google-Metadata-Request': 'True'}
+    headers = {'Metadata-Flavor': 'Google'}
 
     def __init__(self, metadata_address):
         self.metadata_address = metadata_address
 
-    def get_value(self, path, is_text):
+    def get_value(self, path, is_text, is_recursive=False):
         value = None
         try:
-            resp = url_helper.readurl(url=self.metadata_address + path,
-                                      headers=self.headers)
+            url = self.metadata_address + path
+            if is_recursive:
+                url += '/?recursive=True'
+            resp = url_helper.readurl(url=url, headers=self.headers)
         except url_helper.UrlError as exc:
             msg = "url %s raised exception %s"
             LOG.debug(msg, path, exc)
@@ -35,7 +41,7 @@ class GoogleMetadataFetcher(object):
                 if is_text:
                     value = util.decode_binary(resp.contents)
                 else:
-                    value = resp.contents
+                    value = resp.contents.decode('utf-8')
             else:
                 LOG.debug("url %s returned code %s", path, resp.code)
         return value
@@ -47,6 +53,10 @@ class DataSourceGCE(sources.DataSource):
 
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
+        self.default_user = None
+        if distro:
+            (users, _groups) = ug_util.normalize_users_groups(sys_cfg, distro)
+            (self.default_user, _user_config) = ug_util.extract_default(users)
         self.metadata = dict()
         self.ds_cfg = util.mergemanydict([
             util.get_cfg_by_path(sys_cfg, ["datasource", "GCE"], {}),
@@ -70,17 +80,18 @@ class DataSourceGCE(sources.DataSource):
 
     @property
     def launch_index(self):
-        # GCE does not provide lauch_index property
+        # GCE does not provide lauch_index property.
         return None
 
     def get_instance_id(self):
         return self.metadata['instance-id']
 
     def get_public_ssh_keys(self):
-        return self.metadata['public-keys']
+        public_keys_data = self.metadata['public-keys-data']
+        return _parse_public_keys(public_keys_data, self.default_user)
 
     def get_hostname(self, fqdn=False, resolve_ip=False):
-        # GCE has long FDQN's and has asked for short hostnames
+        # GCE has long FDQN's and has asked for short hostnames.
         return self.metadata['local-hostname'].split('.')[0]
 
     @property
@@ -92,15 +103,58 @@ class DataSourceGCE(sources.DataSource):
         return self.availability_zone.rsplit('-', 1)[0]
 
 
-def _trim_key(public_key):
-    # GCE takes sshKeys attribute in the format of '<user>:<public_key>'
-    # so we have to trim each key to remove the username part
+def _has_expired(public_key):
+    # Check whether an SSH key is expired. Public key input is a single SSH
+    # public key in the GCE specific key format documented here:
+    # https://cloud.google.com/compute/docs/instances/adding-removing-ssh-keys#sshkeyformat
+    try:
+        # Check for the Google-specific schema identifier.
+        schema, json_str = public_key.split(None, 3)[2:]
+    except (ValueError, AttributeError):
+        return False
+
+    # Do not expire keys if they do not have the expected schema identifier.
+    if schema != 'google-ssh':
+        return False
+
+    try:
+        json_obj = json.loads(json_str)
+    except ValueError:
+        return False
+
+    # Do not expire keys if there is no expriation timestamp.
+    if 'expireOn' not in json_obj:
+        return False
+
+    expire_str = json_obj['expireOn']
+    format_str = '%Y-%m-%dT%H:%M:%S+0000'
     try:
-        index = public_key.index(':')
-        if index > 0:
-            return public_key[(index + 1):]
-    except Exception:
-        return public_key
+        expire_time = datetime.datetime.strptime(expire_str, format_str)
+    except ValueError:
+        return False
+
+    # Expire the key if and only if we have exceeded the expiration timestamp.
+    return datetime.datetime.utcnow() > expire_time
+
+
+def _parse_public_keys(public_keys_data, default_user=None):
+    # Parse the SSH key data for the default user account. Public keys input is
+    # a list containing SSH public keys in the GCE specific key format
+    # documented here:
+    # https://cloud.google.com/compute/docs/instances/adding-removing-ssh-keys#sshkeyformat
+    public_keys = []
+    if not public_keys_data:
+        return public_keys
+    for public_key in public_keys_data:
+        if not public_key or not all(ord(c) < 128 for c in public_key):
+            continue
+        split_public_key = public_key.split(':', 1)
+        if len(split_public_key) != 2:
+            continue
+        user, key = split_public_key
+        if user in ('cloudinit', default_user) and not _has_expired(key):
+            public_keys.append(key)
+    return public_keys
 
 
 def read_md(address=None, platform_check=True):
@@ -116,31 +170,28 @@ def read_md(address=None, platform_check=True):
         ret['reason'] = "Not running on GCE."
         return ret
 
-    # if we cannot resolve the metadata server, then no point in trying
+    # If we cannot resolve the metadata server, then no point in trying.
     if not util.is_resolvable_url(address):
         LOG.debug("%s is not resolvable", address)
         ret['reason'] = 'address "%s" is not resolvable' % address
         return ret
 
-    # url_map: (our-key, path, required, is_text)
+    # url_map: (our-key, path, required, is_text, is_recursive)
     url_map = [
-        ('instance-id', ('instance/id',), True, True),
-        ('availability-zone', ('instance/zone',), True, True),
-        ('local-hostname', ('instance/hostname',), True, True),
-        ('public-keys', ('project/attributes/sshKeys',
-                         'instance/attributes/ssh-keys'), False, True),
-        ('user-data', ('instance/attributes/user-data',), False, False),
-        ('user-data-encoding', ('instance/attributes/user-data-encoding',),
-         False, True),
+        ('instance-id', ('instance/id',), True, True, False),
+        ('availability-zone', ('instance/zone',), True, True, False),
+        ('local-hostname', ('instance/hostname',), True, True, False),
+        ('instance-data', ('instance/attributes',), False, False, True),
+        ('project-data', ('project/attributes',), False, False, True),
     ]
 
     metadata_fetcher = GoogleMetadataFetcher(address)
     md = {}
-    # iterate over url_map keys to get metadata items
-    for (mkey, paths, required, is_text) in url_map:
+    # Iterate over url_map keys to get metadata items.
+    for (mkey, paths, required, is_text, is_recursive) in url_map:
         value = None
         for path in paths:
-            new_value = metadata_fetcher.get_value(path, is_text)
+            new_value = metadata_fetcher.get_value(path, is_text, is_recursive)
             if new_value is not None:
                 value = new_value
         if required and value is None:
@@ -149,17 +200,23 @@ def read_md(address=None, platform_check=True):
             return ret
         md[mkey] = value
 
-    if md['public-keys']:
-        lines = md['public-keys'].splitlines()
-        md['public-keys'] = [_trim_key(k) for k in lines]
+    instance_data = json.loads(md['instance-data'] or '{}')
+    project_data = json.loads(md['project-data'] or '{}')
+    valid_keys = [instance_data.get('sshKeys'), instance_data.get('ssh-keys')]
+    block_project = instance_data.get('block-project-ssh-keys', '').lower()
+    if block_project != 'true' and not instance_data.get('sshKeys'):
+        valid_keys.append(project_data.get('ssh-keys'))
+        valid_keys.append(project_data.get('sshKeys'))
+    public_keys_data = '\n'.join([key for key in valid_keys if key])
+    md['public-keys-data'] = public_keys_data.splitlines()
 
     if md['availability-zone']:
         md['availability-zone'] = md['availability-zone'].split('/')[-1]
 
-    encoding = md.get('user-data-encoding')
+    encoding = instance_data.get('user-data-encoding')
     if encoding:
         if encoding == 'base64':
-            md['user-data'] = b64decode(md['user-data'])
+            md['user-data'] = b64decode(instance_data.get('user-data'))
         else:
             LOG.warning('unknown user-data-encoding: %s, ignoring', encoding)
 
@@ -188,20 +245,19 @@ def platform_reports_gce():
     return False
 
 
-# Used to match classes to dependencies
+# Used to match classes to dependencies.
 datasources = [
     (DataSourceGCE, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
 ]
 
 
-# Return a list of data sources that match this set of dependencies
+# Return a list of data sources that match this set of dependencies.
 def get_datasource_list(depends):
     return sources.list_from_depends(depends, datasources)
 
 
 if __name__ == "__main__":
     import argparse
-    import json
     import sys
 
     from base64 import b64encode
@@ -217,7 +273,7 @@ if __name__ == "__main__":
     data = read_md(address=args.endpoint, platform_check=args.platform_check)
     if 'user-data' in data:
         # user-data is bytes not string like other things. Handle it specially.
-        # if it can be represented as utf-8 then do so.  Otherwise print base64
+        # If it can be represented as utf-8 then do so. Otherwise print base64
         # encoded value in the key user-data-b64.
         try:
             data['user-data'] = data['user-data'].decode()
@@ -225,7 +281,7 @@ if __name__ == "__main__":
             sys.stderr.write("User-data cannot be decoded. "
                              "Writing as base64\n")
             del data['user-data']
-            # b64encode returns a bytes value. decode to get the string.
+            # b64encode returns a bytes value. Decode to get the string.
             data['user-data-b64'] = b64encode(data['user-data']).decode()
 
     print(json.dumps(data, indent=1, sort_keys=True, separators=(',', ': ')))
diff --git a/cloudinit/util.py b/cloudinit/util.py
index e42498d..df0aa5d 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -253,12 +253,18 @@ class ProcessExecutionError(IOError):
             self.exit_code = exit_code
 
         if not stderr:
-            self.stderr = self.empty_attr
+            if stderr is None:
+                self.stderr = self.empty_attr
+            else:
+                self.stderr = stderr
         else:
             self.stderr = self._indent_text(stderr)
 
         if not stdout:
-            self.stdout = self.empty_attr
+            if stdout is None:
+                self.stdout = self.empty_attr
+            else:
+                self.stdout = stdout
         else:
             self.stdout = self._indent_text(stdout)
 
@@ -1829,58 +1835,60 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False,
         env = env.copy()
         env.update(update_env)
 
-    try:
-        if target_path(target) != "/":
-            args = ['chroot', target] + list(args)
+    if target_path(target) != "/":
+        args = ['chroot', target] + list(args)
 
-        if not logstring:
-            LOG.debug(("Running command %s with allowed return codes %s"
-                       " (shell=%s, capture=%s)"), args, rcs, shell, capture)
-        else:
-            LOG.debug(("Running hidden command to protect sensitive "
-                       "input/output logstring: %s"), logstring)
-
-        stdin = None
-        stdout = None
-        stderr = None
-        if capture:
-            stdout = subprocess.PIPE
-            stderr = subprocess.PIPE
-        if data is None:
-            # using devnull assures any reads get null, rather
-            # than possibly waiting on input.
-            devnull_fp = open(os.devnull)
-            stdin = devnull_fp
-        else:
-            stdin = subprocess.PIPE
-            if not isinstance(data, bytes):
-                data = data.encode()
+    if not logstring:
+        LOG.debug(("Running command %s with allowed return codes %s"
+                   " (shell=%s, capture=%s)"), args, rcs, shell, capture)
+    else:
+        LOG.debug(("Running hidden command to protect sensitive "
+                   "input/output logstring: %s"), logstring)
+
+    stdin = None
+    stdout = None
+    stderr = None
+    if capture:
+        stdout = subprocess.PIPE
+        stderr = subprocess.PIPE
+    if data is None:
+        # using devnull assures any reads get null, rather
+        # than possibly waiting on input.
+        devnull_fp = open(os.devnull)
+        stdin = devnull_fp
+    else:
+        stdin = subprocess.PIPE
+        if not isinstance(data, bytes):
+            data = data.encode()
 
+    try:
         sp = subprocess.Popen(args, stdout=stdout,
                               stderr=stderr, stdin=stdin,
                               env=env, shell=shell)
         (out, err) = sp.communicate(data)
-
-        # Just ensure blank instead of none.
-        if not out and capture:
-            out = b''
-        if not err and capture:
-            err = b''
-        if decode:
-            def ldecode(data, m='utf-8'):
-                if not isinstance(data, bytes):
-                    return data
-                return data.decode(m, decode)
-
-            out = ldecode(out)
-            err = ldecode(err)
     except OSError as e:
-        raise ProcessExecutionError(cmd=args, reason=e,
-                                    errno=e.errno)
+        raise ProcessExecutionError(
+            cmd=args, reason=e, errno=e.errno,
+            stdout="-" if decode else b"-",
+            stderr="-" if decode else b"-")
     finally:
         if devnull_fp:
             devnull_fp.close()
 
+    # Just ensure blank instead of none.
+    if not out and capture:
+        out = b''
+    if not err and capture:
+        err = b''
+    if decode:
+        def ldecode(data, m='utf-8'):
+            if not isinstance(data, bytes):
+                return data
+            return data.decode(m, decode)
+
+        out = ldecode(out)
+        err = ldecode(err)
+
     rc = sp.returncode
     if rc not in rcs:
         raise ProcessExecutionError(stdout=out, stderr=err,
diff --git a/debian/changelog b/debian/changelog
index fc09908..093ed09 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,17 @@
+cloud-init (17.2-20-g32a6a176-0ubuntu1) bionic; urgency=medium
+
+  * New upstream snapshot.
+    - tests: Fix EC2 Platform to return console output as bytes.
+    - tests: Fix attempted use of /run in a test case.
+    - GCE: Improvements and changes to ssh key behavior for default user.
+      [Max Illfelder] (LP: #1670456, #1707033, #1707037, #1707039)
+    - subp: make ProcessExecutionError have expected types in stderr, stdout.
+    - tests: when querying ntp server, do not do dns resolution.
+    - Recognize uppercase vfat disk labels [James Penick] (LP: #1598783)
+    - tests: remove zesty as supported OS to test
+
+ -- Chad Smith <[e-mail]>  Tue, 23 Jan 2018 20:10:44 -0700
+
 cloud-init (17.2-13-g6299e8d0-0ubuntu1) bionic; urgency=medium
 
   * New upstream snapshot.
diff --git a/tests/cloud_tests/platforms/ec2/instance.py b/tests/cloud_tests/platforms/ec2/instance.py
index 4ba737a..ab6037b 100644
--- a/tests/cloud_tests/platforms/ec2/instance.py
+++ b/tests/cloud_tests/platforms/ec2/instance.py
@@ -46,9 +46,15 @@ class EC2Instance(Instance):
         may return empty string.
         """
         try:
-            return self.instance.console_output()['Output'].encode()
+            # OutputBytes comes from platform._decode_console_output_as_bytes
+            response = self.instance.console_output()
+            return response['OutputBytes']
         except KeyError:
-            return b''
+            if 'Output' in response:
+                msg = ("'OutputBytes' did not exist in console_output() but "
+                       "'Output' did: %s..." % response['Output'][0:128])
+                raise util.PlatformError('console_log', msg)
+            return ('No Console Output [%s]' % self.instance).encode()
 
     def destroy(self):
         """Clean up instance."""
diff --git a/tests/cloud_tests/platforms/ec2/platform.py b/tests/cloud_tests/platforms/ec2/platform.py
index fdb17ba..f188c27 100644
--- a/tests/cloud_tests/platforms/ec2/platform.py
+++ b/tests/cloud_tests/platforms/ec2/platform.py
@@ -6,6 +6,8 @@ import os
 
 import boto3
 import botocore
+from botocore import session, handlers
+import base64
 
 from ..platforms import Platform
 from .image import EC2Image
@@ -28,9 +30,10 @@ class EC2Platform(Platform):
         self.instance_type = config['instance-type']
 
         try:
-            self.ec2_client = boto3.client('ec2')
-            self.ec2_resource = boto3.resource('ec2')
-            self.ec2_region = boto3.Session().region_name
+            b3session = get_session()
+            self.ec2_client = b3session.client('ec2')
+            self.ec2_resource = b3session.resource('ec2')
+            self.ec2_region = b3session.region_name
             self.key_name = self._upload_public_key(config)
         except botocore.exceptions.NoRegionError:
             raise RuntimeError(
@@ -228,4 +231,28 @@ class EC2Platform(Platform):
 
         return self.tag
 
+
+def _decode_console_output_as_bytes(parsed, **kwargs):
+    """Provide console output as bytes in OutputBytes.
+
+       For this to be useful, the session has to have had the
+       decode_console_output handler unregistered already.
+
+       https://github.com/boto/botocore/issues/1351 ."""
+    if 'Output' not in parsed:
+        return
+    orig = parsed['Output']
+    handlers.decode_console_output(parsed, **kwargs)
+    parsed['OutputBytes'] = base64.b64decode(orig)
+
+
+def get_session():
+    mysess = session.get_session()
+    mysess.unregister('after-call.ec2.GetConsoleOutput',
+                      handlers.decode_console_output)
+    mysess.register('after-call.ec2.GetConsoleOutput',
+                    _decode_console_output_as_bytes)
+    return boto3.Session(botocore_session=mysess)
+
+
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/releases.yaml b/tests/cloud_tests/releases.yaml
index 0a9fa60..d8bc170 100644
--- a/tests/cloud_tests/releases.yaml
+++ b/tests/cloud_tests/releases.yaml
@@ -158,22 +158,6 @@ releases:
             alias: artful
             setup_overrides: null
             override_templates: false
-    zesty:
-        # EOL: Jan 2018
-        default:
-            enabled: true
-            release: zesty
-            version: 17.04
-            os: ubuntu
-            feature_groups:
-                - base
-                - debian_base
-                - ubuntu_specific
-        lxd:
-            sstreams_server: https://cloud-images.ubuntu.com/daily
-            alias: zesty
-            setup_overrides: null
-            override_templates: false
     xenial:
         # EOL: Apr 2021
         default:
diff --git a/tests/cloud_tests/testcases/modules/ntp_pools.yaml b/tests/cloud_tests/testcases/modules/ntp_pools.yaml
index 3a93faa..d490b22 100644
--- a/tests/cloud_tests/testcases/modules/ntp_pools.yaml
+++ b/tests/cloud_tests/testcases/modules/ntp_pools.yaml
@@ -26,6 +26,6 @@ collect_scripts:
     grep '^pool' /etc/ntp.conf
   ntpq_servers: |
     #!/bin/sh
-    ntpq -p -w
+    ntpq -p -w -n
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/testcases/modules/ntp_servers.yaml b/tests/cloud_tests/testcases/modules/ntp_servers.yaml
index d59d45a..6b13b70 100644
--- a/tests/cloud_tests/testcases/modules/ntp_servers.yaml
+++ b/tests/cloud_tests/testcases/modules/ntp_servers.yaml
@@ -22,6 +22,6 @@ collect_scripts:
     grep '^server' /etc/ntp.conf
   ntpq_servers: |
     #!/bin/sh
-    ntpq -p -w
+    ntpq -p -w -n
 
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py
index 6ef5a35..68400f2 100644
--- a/tests/unittests/test_datasource/test_configdrive.py
+++ b/tests/unittests/test_datasource/test_configdrive.py
@@ -458,6 +458,12 @@ class TestConfigDriveDataSource(CiTestCase):
             self.assertEqual(["/dev/vdb3"],
                              ds.find_candidate_devs())
 
+            # Verify that uppercase labels are also found.
+            devs_with_answers = {"TYPE=vfat": [],
+                                 "TYPE=iso9660": ["/dev/vdb"],
+                                 "LABEL=CONFIG-2": ["/dev/vdb"]}
+            self.assertEqual(["/dev/vdb"], ds.find_candidate_devs())
+
         finally:
             util.find_devs_with = orig_find_devs_with
             util.is_partition = orig_is_partition
diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py
index 82c788d..f77c2c4 100644
--- a/tests/unittests/test_datasource/test_gce.py
+++ b/tests/unittests/test_datasource/test_gce.py
@@ -4,13 +4,16 @@
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
+import datetime
 import httpretty
+import json
 import mock
 import re
 
 from base64 import b64encode, b64decode
 from six.moves.urllib_parse import urlparse
 
+from cloudinit import distros
 from cloudinit import helpers
 from cloudinit import settings
 from cloudinit.sources import DataSourceGCE
@@ -21,10 +24,7 @@ from cloudinit.tests import helpers as test_helpers
 GCE_META = {
     'instance/id': '123',
     'instance/zone': 'foo/bar',
-    'project/attributes/sshKeys': 'user:ssh-rsa AA2..+aRD0fyVw== root@server',
     'instance/hostname': 'server.project-foo.local',
-    # UnicodeDecodeError below if set to ds.userdata instead of userdata_raw
-    'instance/attributes/user-data': b'/bin/echo \xff\n',
 }
 
 GCE_META_PARTIAL = {
@@ -37,11 +37,13 @@ GCE_META_ENCODING = {
     'instance/id': '12345',
     'instance/hostname': 'server.project-baz.local',
     'instance/zone': 'baz/bang',
-    'instance/attributes/user-data': b64encode(b'/bin/echo baz\n'),
-    'instance/attributes/user-data-encoding': 'base64',
+    'instance/attributes': {
+        'user-data': b64encode(b'/bin/echo baz\n').decode('utf-8'),
+        'user-data-encoding': 'base64',
+    }
 }
 
-HEADERS = {'X-Google-Metadata-Request': 'True'}
+HEADERS = {'Metadata-Flavor': 'Google'}
 MD_URL_RE = re.compile(
     r'http://metadata.google.internal/computeMetadata/v1/.*')
 
@@ -54,10 +56,15 @@ def _set_mock_metadata(gce_meta=None):
         url_path = urlparse(uri).path
         if url_path.startswith('/computeMetadata/v1/'):
             path = url_path.split('/computeMetadata/v1/')[1:][0]
+            recursive = path.endswith('/')
+            path = path.rstrip('/')
         else:
             path = None
         if path in gce_meta:
-            return (200, headers, gce_meta.get(path))
+            response = gce_meta.get(path)
+            if recursive:
+                response = json.dumps(response)
+            return (200, headers, response)
         else:
             return (404, headers, '')
 
@@ -69,6 +76,16 @@ def _set_mock_metadata(gce_meta=None):
 @httpretty.activate
 class TestDataSourceGCE(test_helpers.HttprettyTestCase):
 
+    def _make_distro(self, dtype, def_user=None):
+        cfg = dict(settings.CFG_BUILTIN)
+        cfg['system_info']['distro'] = dtype
+        paths = helpers.Paths(cfg['system_info']['paths'])
+        distro_cls = distros.fetch(dtype)
+        if def_user:
+            cfg['system_info']['default_user'] = def_user.copy()
+        distro = distro_cls(dtype, cfg['system_info'], paths)
+        return distro
+
     def setUp(self):
         tmp = self.tmp_dir()
         self.ds = DataSourceGCE.DataSourceGCE(
@@ -90,6 +107,10 @@ class TestDataSourceGCE(test_helpers.HttprettyTestCase):
         self.assertDictContainsSubset(HEADERS, req_header)
 
     def test_metadata(self):
+        # UnicodeDecodeError if set to ds.userdata instead of userdata_raw
+        meta = GCE_META.copy()
+        meta['instance/attributes/user-data'] = b'/bin/echo \xff\n'
+
         _set_mock_metadata()
         self.ds.get_data()
 
@@ -118,8 +139,8 @@ class TestDataSourceGCE(test_helpers.HttprettyTestCase):
         _set_mock_metadata(GCE_META_ENCODING)
         self.ds.get_data()
 
-        decoded = b64decode(
-            GCE_META_ENCODING.get('instance/attributes/user-data'))
+        instance_data = GCE_META_ENCODING.get('instance/attributes')
+        decoded = b64decode(instance_data.get('user-data'))
         self.assertEqual(decoded, self.ds.get_userdata_raw())
 
     def test_missing_required_keys_return_false(self):
@@ -131,33 +152,124 @@ class TestDataSourceGCE(test_helpers.HttprettyTestCase):
             self.assertEqual(False, self.ds.get_data())
             httpretty.reset()
 
-    def test_project_level_ssh_keys_are_used(self):
+    def test_no_ssh_keys_metadata(self):
         _set_mock_metadata()
         self.ds.get_data()
+        self.assertEqual([], self.ds.get_public_ssh_keys())
+
+    def test_cloudinit_ssh_keys(self):
+        valid_key = 'ssh-rsa VALID {0}'
+        invalid_key = 'ssh-rsa INVALID {0}'
+        project_attributes = {
+            'sshKeys': '\n'.join([
+                'cloudinit:{0}'.format(valid_key.format(0)),
+                'user:{0}'.format(invalid_key.format(0)),
+            ]),
+            'ssh-keys': '\n'.join([
+                'cloudinit:{0}'.format(valid_key.format(1)),
+                'user:{0}'.format(invalid_key.format(1)),
+            ]),
+        }
+        instance_attributes = {
+            'ssh-keys': '\n'.join([
+                'cloudinit:{0}'.format(valid_key.format(2)),
+                'user:{0}'.format(invalid_key.format(2)),
+            ]),
+            'block-project-ssh-keys': 'False',
+        }
+
+        meta = GCE_META.copy()
+        meta['project/attributes'] = project_attributes
+        meta['instance/attributes'] = instance_attributes
+
+        _set_mock_metadata(meta)
+        self.ds.get_data()
+
+        expected = [valid_key.format(key) for key in range(3)]
+        self.assertEqual(set(expected), set(self.ds.get_public_ssh_keys()))
+
+    @mock.patch("cloudinit.sources.DataSourceGCE.ug_util")
+    def test_default_user_ssh_keys(self, mock_ug_util):
+        mock_ug_util.normalize_users_groups.return_value = None, None
+        mock_ug_util.extract_default.return_value = 'ubuntu', None
+        ubuntu_ds = DataSourceGCE.DataSourceGCE(
+            settings.CFG_BUILTIN, self._make_distro('ubuntu'),
+            helpers.Paths({'run_dir': self.tmp_dir()}))
+
+        valid_key = 'ssh-rsa VALID {0}'
+        invalid_key = 'ssh-rsa INVALID {0}'
+        project_attributes = {
+            'sshKeys': '\n'.join([
+                'ubuntu:{0}'.format(valid_key.format(0)),
+                'user:{0}'.format(invalid_key.format(0)),
+            ]),
+            'ssh-keys': '\n'.join([
+                'ubuntu:{0}'.format(valid_key.format(1)),
+                'user:{0}'.format(invalid_key.format(1)),
+            ]),
+        }
+        instance_attributes = {
+            'ssh-keys': '\n'.join([
+                'ubuntu:{0}'.format(valid_key.format(2)),
+                'user:{0}'.format(invalid_key.format(2)),
+            ]),
+            'block-project-ssh-keys': 'False',
+        }
 
-        # we expect a list of public ssh keys with user names stripped
-        self.assertEqual(['ssh-rsa AA2..+aRD0fyVw== root@server'],
-                         self.ds.get_public_ssh_keys())
+        meta = GCE_META.copy()
+        meta['project/attributes'] = project_attributes
+        meta['instance/attributes'] = instance_attributes
+
+        _set_mock_metadata(meta)
+        ubuntu_ds.get_data()
+
+        expected = [valid_key.format(key) for key in range(3)]
+        self.assertEqual(set(expected), set(ubuntu_ds.get_public_ssh_keys()))
+
+    def test_instance_ssh_keys_override(self):
+        valid_key = 'ssh-rsa VALID {0}'
+        invalid_key = 'ssh-rsa INVALID {0}'
+        project_attributes = {
+            'sshKeys': 'cloudinit:{0}'.format(invalid_key.format(0)),
+            'ssh-keys': 'cloudinit:{0}'.format(invalid_key.format(1)),
+        }
+        instance_attributes = {
+            'sshKeys': 'cloudinit:{0}'.format(valid_key.format(0)),
+            'ssh-keys': 'cloudinit:{0}'.format(valid_key.format(1)),
+            'block-project-ssh-keys': 'False',
+        }
 
-    def test_instance_level_ssh_keys_are_used(self):
-        key_content = 'ssh-rsa JustAUser root@server'
         meta = GCE_META.copy()
-        meta['instance/attributes/ssh-keys'] = 'user:{0}'.format(key_content)
+        meta['project/attributes'] = project_attributes
+        meta['instance/attributes'] = instance_attributes
 
         _set_mock_metadata(meta)
         self.ds.get_data()
 
-        self.assertIn(key_content, self.ds.get_public_ssh_keys())
+        expected = [valid_key.format(key) for key in range(2)]
+        self.assertEqual(set(expected), set(self.ds.get_public_ssh_keys()))
+
+    def test_block_project_ssh_keys_override(self):
+        valid_key = 'ssh-rsa VALID {0}'
+        invalid_key = 'ssh-rsa INVALID {0}'
+        project_attributes = {
+            'sshKeys': 'cloudinit:{0}'.format(invalid_key.format(0)),
+            'ssh-keys': 'cloudinit:{0}'.format(invalid_key.format(1)),
+        }
+        instance_attributes = {
+            'ssh-keys': 'cloudinit:{0}'.format(valid_key.format(0)),
+            'block-project-ssh-keys': 'True',
+        }
 
-    def test_instance_level_keys_replace_project_level_keys(self):
-        key_content = 'ssh-rsa JustAUser root@server'
         meta = GCE_META.copy()
-        meta['instance/attributes/ssh-keys'] = 'user:{0}'.format(key_content)
+        meta['project/attributes'] = project_attributes
+        meta['instance/attributes'] = instance_attributes
 
         _set_mock_metadata(meta)
         self.ds.get_data()
 
-        self.assertEqual([key_content], self.ds.get_public_ssh_keys())
+        expected = [valid_key.format(0)]
+        self.assertEqual(set(expected), set(self.ds.get_public_ssh_keys()))
 
     def test_only_last_part_of_zone_used_for_availability_zone(self):
         _set_mock_metadata()
@@ -172,5 +284,44 @@ class TestDataSourceGCE(test_helpers.HttprettyTestCase):
         self.assertEqual(False, ret)
         m_fetcher.assert_not_called()
 
+    def test_has_expired(self):
+
+        def _get_timestamp(days):
+            format_str = '%Y-%m-%dT%H:%M:%S+0000'
+            today = datetime.datetime.now()
+            timestamp = today + datetime.timedelta(days=days)
+            return timestamp.strftime(format_str)
+
+        past = _get_timestamp(-1)
+        future = _get_timestamp(1)
+        ssh_keys = {
+            None: False,
+            '': False,
+            'Invalid': False,
+            'user:ssh-rsa key [e-mail]': False,
+            'user:ssh-rsa key google {"expireOn":"%s"}' % past: False,
+            'user:ssh-rsa key google-ssh': False,
+            'user:ssh-rsa key google-ssh {invalid:json}': False,
+            'user:ssh-rsa key google-ssh {"userName":"user"}': False,
+            'user:ssh-rsa key google-ssh {"expireOn":"invalid"}': False,
+            'user:xyz key google-ssh {"expireOn":"%s"}' % future: False,
+            'user:xyz key google-ssh {"expireOn":"%s"}' % past: True,
+        }
+
+        for key, expired in ssh_keys.items():
+            self.assertEqual(DataSourceGCE._has_expired(key), expired)
+
+    def test_parse_public_keys_non_ascii(self):
+        public_key_data = [
+            'cloudinit:rsa ssh-ke%s invalid' % chr(165),
+            'use%sname:rsa ssh-key' % chr(174),
+            'cloudinit:test 1',
+            'default:test 2',
+            'user:test 3',
+        ]
+        expected = ['test 1', 'test 2']
+        found = DataSourceGCE._parse_public_keys(
+            public_key_data, default_user='default')
+        self.assertEqual(sorted(found), sorted(expected))
 
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index c9234ed..ad6c5cf 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -232,6 +232,11 @@ class TestDsIdentify(CiTestCase):
         self._test_ds_found('ConfigDrive')
         return
 
+    def test_config_drive_upper(self):
+        """ConfigDrive datasource has a disk with LABEL=CONFIG-2."""
+        self._test_ds_found('ConfigDriveUpper')
+        return
+
     def test_policy_disabled(self):
         """A Builtin policy of 'disabled' should return not found.
 
@@ -503,6 +508,18 @@ VALID_CFG = {
              },
         ],
     },
+    'ConfigDriveUpper': {
+        'ds': 'ConfigDrive',
+        'mocks': [
+            {'name': 'blkid', 'ret': 0,
+             'out': blkid_out(
+                 [{'DEVNAME': 'vda1', 'TYPE': 'vfat', 'PARTUUID': uuid4()},
+                  {'DEVNAME': 'vda2', 'TYPE': 'ext4',
+                   'LABEL': 'cloudimg-rootfs', 'PARTUUID': uuid4()},
+                  {'DEVNAME': 'vdb', 'TYPE': 'vfat', 'LABEL': 'CONFIG-2'}])
+             },
+        ],
+    },
 }
 
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
index d63b760..4a92e74 100644
--- a/tests/unittests/test_util.py
+++ b/tests/unittests/test_util.py
@@ -623,6 +623,7 @@ class TestSubp(helpers.CiTestCase):
     utf8_valid = b'start \xc3\xa9 end'
     utf8_valid_2 = b'd\xc3\xa9j\xc8\xa7'
     printenv = [BASH, '-c', 'for n in "$@"; do echo "$n=${!n}"; done', '--']
+    bogus_command = 'this-is-not-expected-to-be-a-program-name'
 
     def printf_cmd(self, *args):
         # bash's printf supports \xaa.  So does /usr/bin/printf
@@ -712,6 +713,20 @@ class TestSubp(helpers.CiTestCase):
         self.assertIsNone(err)
         self.assertIsNone(out)
 
+    def test_exception_has_out_err_are_bytes_if_decode_false(self):
+        """Raised exc should have stderr, stdout as bytes if no decode."""
+        with self.assertRaises(util.ProcessExecutionError) as cm:
+            util.subp([self.bogus_command], decode=False)
+        self.assertTrue(isinstance(cm.exception.stdout, bytes))
+        self.assertTrue(isinstance(cm.exception.stderr, bytes))
+
+    def test_exception_has_out_err_are_bytes_if_decode_true(self):
+        """Raised exc should have stderr, stdout as string if no decode."""
+        with self.assertRaises(util.ProcessExecutionError) as cm:
+            util.subp([self.bogus_command], decode=True)
+        self.assertTrue(isinstance(cm.exception.stdout, six.string_types))
+        self.assertTrue(isinstance(cm.exception.stderr, six.string_types))
+
     def test_bunch_of_slashes_in_path(self):
         self.assertEqual("/target/my/path/",
                          util.target_path("/target/", "//my/path/"))
diff --git a/tools/ds-identify b/tools/ds-identify
index 5893a76..374c3ad 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -579,6 +579,8 @@ dscheck_NoCloud() {
 check_configdrive_v2() {
     if has_fs_with_label "config-2"; then
         return ${DS_FOUND}
+    elif has_fs_with_label "CONFIG-2"; then
+        return ${DS_FOUND}
     fi
     # look in /config-drive <vlc>/seed/config_drive for a directory
     # openstack/YYYY-MM-DD format with a file meta_data.json
@@ -666,7 +668,7 @@ is_cdrom_ovf() {
 
     # explicitly skip known labels of other types. rd_rdfe is azure.
     case "$label" in
-        config-2|rd_rdfe_stable*|cidata) return 1;;
+        config-2|CONFIG-2|rd_rdfe_stable*|cidata) return 1;;
     esac
 
     local idstr="http://schemas.dmtf.org/ovf/environment/1"