Merge ~chad.smith/cloud-init:ubuntu/devel into cloud-init:ubuntu/devel

[Chad Smith]

Sync tip of master for release in Bionic with new IBMCloud datasource, Azure timeout fixes and ubuntu-advantage cloud-config module.

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:b6f9fc66c73915f81b51fdf24a77dec6df274d21
https://jenkins.ubuntu.com/server/job/cloud-init-ci/924/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Unit & Style Tests
    SUCCESS: Ubuntu LTS: Build
    SUCCESS: Ubuntu LTS: Integration
    SUCCESS: MAAS Compatability Testing
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/924/rebuild

review: Approve (continuous-integration)

[Scott Moser (smoser)]

I merged this at 405e9518755df26f492252c143ae370429c1d6c8
and then added the IBMCloud datasource and released.

thanks Chad.

review: Approve

diff --git a/cloudinit/config/cc_snap.py b/cloudinit/config/cc_snap.py
index db96529..34a53fd 100644
--- a/cloudinit/config/cc_snap.py
+++ b/cloudinit/config/cc_snap.py
@@ -11,6 +11,7 @@ from cloudinit import log as logging
 from cloudinit.config.schema import (
     get_schema_doc, validate_cloudconfig_schema)
 from cloudinit.settings import PER_INSTANCE
+from cloudinit.subp import prepend_base_command
 from cloudinit import util
 
 
@@ -160,50 +161,6 @@ def add_assertions(assertions):
     util.subp(snap_cmd + [ASSERTIONS_FILE], capture=True)
 
 
-def prepend_snap_commands(commands):
-    """Ensure user-provided commands start with SNAP_CMD, warn otherwise.
-
-    Each command is either a list or string. Perform the following:
-       - When the command is a list, pop the first element if it is None
-       - When the command is a list, insert SNAP_CMD as the first element if
-         not present.
-       - When the command is a string containing a non-snap command, warn.
-
-    Support cut-n-paste snap command sets from public snappy documentation.
-    Allow flexibility to provide non-snap environment/config setup if needed.
-
-    @commands: List of commands. Each command element is a list or string.
-
-    @return: List of 'fixed up' snap commands.
-    @raise: TypeError on invalid config item type.
-    """
-    warnings = []
-    errors = []
-    fixed_commands = []
-    for command in commands:
-        if isinstance(command, list):
-            if command[0] is None:  # Avoid warnings by specifying None
-                command = command[1:]
-            elif command[0] != SNAP_CMD:  # Automatically prepend SNAP_CMD
-                command.insert(0, SNAP_CMD)
-        elif isinstance(command, str):
-            if not command.startswith('%s ' % SNAP_CMD):
-                warnings.append(command)
-        else:
-            errors.append(str(command))
-            continue
-        fixed_commands.append(command)
-
-    if warnings:
-        LOG.warning(
-            'Non-snap commands in snap config:\n%s', '\n'.join(warnings))
-    if errors:
-        raise TypeError(
-            'Invalid snap config.'
-            ' These commands are not a string or list:\n' + '\n'.join(errors))
-    return fixed_commands
-
-
 def run_commands(commands):
     """Run the provided commands provided in snap:commands configuration.
 
@@ -224,7 +181,7 @@ def run_commands(commands):
             'commands parameter was not a list or dict: {commands}'.format(
                 commands=commands))
 
-    fixed_snap_commands = prepend_snap_commands(commands)
+    fixed_snap_commands = prepend_base_command('snap', commands)
 
     cmd_failures = []
     for command in fixed_snap_commands:
diff --git a/cloudinit/config/cc_ubuntu_advantage.py b/cloudinit/config/cc_ubuntu_advantage.py
new file mode 100644
index 0000000..16b1868
--- /dev/null
+++ b/cloudinit/config/cc_ubuntu_advantage.py
@@ -0,0 +1,173 @@
+# Copyright (C) 2018 Canonical Ltd.
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Ubuntu advantage: manage ubuntu-advantage offerings from Canonical."""
+
+import sys
+from textwrap import dedent
+
+from cloudinit import log as logging
+from cloudinit.config.schema import (
+    get_schema_doc, validate_cloudconfig_schema)
+from cloudinit.settings import PER_INSTANCE
+from cloudinit.subp import prepend_base_command
+from cloudinit import util
+
+
+distros = ['ubuntu']
+frequency = PER_INSTANCE
+
+LOG = logging.getLogger(__name__)
+
+schema = {
+    'id': 'cc_ubuntu_advantage',
+    'name': 'Ubuntu Advantage',
+    'title': 'Install, configure and manage ubuntu-advantage offerings',
+    'description': dedent("""\
+        This module provides configuration options to setup ubuntu-advantage
+        subscriptions.
+
+        .. note::
+            Both ``commands`` value can be either a dictionary or a list. If
+            the configuration provided is a dictionary, the keys are only used
+            to order the execution of the commands and the dictionary is
+            merged with any vendor-data ubuntu-advantage configuration
+            provided. If a ``commands`` is provided as a list, any vendor-data
+            ubuntu-advantage ``commands`` are ignored.
+
+        Ubuntu-advantage ``commands`` is a dictionary or list of
+        ubuntu-advantage commands to run on the deployed machine.
+        These commands can be used to enable or disable subscriptions to
+        various ubuntu-advantage products. See 'man ubuntu-advantage' for more
+        information on supported subcommands.
+
+        .. note::
+           Each command item can be a string or list. If the item is a list,
+           'ubuntu-advantage' can be omitted and it will automatically be
+           inserted as part of the command.
+        """),
+    'distros': distros,
+    'examples': [dedent("""\
+        # Enable Extended Security Maintenance using your service auth token
+        ubuntu-advantage:
+            commands:
+              00: ubuntu-advantage enable-esm <token>
+    """), dedent("""\
+        # Enable livepatch by providing your livepatch token
+        ubuntu-advantage:
+            commands:
+                00: ubuntu-advantage enable-livepatch <livepatch-token>
+
+    """), dedent("""\
+        # Convenience: the ubuntu-advantage command can be omitted when
+        # specifying commands as a list and 'ubuntu-advantage' will
+        # automatically be prepended.
+        # The following commands are equivalent
+        ubuntu-advantage:
+            commands:
+                00: ['enable-livepatch', 'my-token']
+                01: ['ubuntu-advantage', 'enable-livepatch', 'my-token']
+                02: ubuntu-advantage enable-livepatch my-token
+                03: 'ubuntu-advantage enable-livepatch my-token'
+    """)],
+    'frequency': PER_INSTANCE,
+    'type': 'object',
+    'properties': {
+        'ubuntu-advantage': {
+            'type': 'object',
+            'properties': {
+                'commands': {
+                    'type': ['object', 'array'],  # Array of strings or dict
+                    'items': {
+                        'oneOf': [
+                            {'type': 'array', 'items': {'type': 'string'}},
+                            {'type': 'string'}]
+                    },
+                    'additionalItems': False,  # Reject non-string & non-list
+                    'minItems': 1,
+                    'minProperties': 1,
+                    'uniqueItems': True
+                }
+            },
+            'additionalProperties': False,  # Reject keys not in schema
+            'required': ['commands']
+        }
+    }
+}
+
+# TODO schema for 'assertions' and 'commands' are too permissive at the moment.
+# Once python-jsonschema supports schema draft 6 add support for arbitrary
+# object keys with 'patternProperties' constraint to validate string values.
+
+__doc__ = get_schema_doc(schema)  # Supplement python help()
+
+UA_CMD = "ubuntu-advantage"
+
+
+def run_commands(commands):
+    """Run the commands provided in ubuntu-advantage:commands config.
+
+     Commands are run individually. Any errors are collected and reported
+     after attempting all commands.
+
+     @param commands: A list or dict containing commands to run. Keys of a
+         dict will be used to order the commands provided as dict values.
+     """
+    if not commands:
+        return
+    LOG.debug('Running user-provided ubuntu-advantage commands')
+    if isinstance(commands, dict):
+        # Sort commands based on dictionary key
+        commands = [v for _, v in sorted(commands.items())]
+    elif not isinstance(commands, list):
+        raise TypeError(
+            'commands parameter was not a list or dict: {commands}'.format(
+                commands=commands))
+
+    fixed_ua_commands = prepend_base_command('ubuntu-advantage', commands)
+
+    cmd_failures = []
+    for command in fixed_ua_commands:
+        shell = isinstance(command, str)
+        try:
+            util.subp(command, shell=shell, status_cb=sys.stderr.write)
+        except util.ProcessExecutionError as e:
+            cmd_failures.append(str(e))
+    if cmd_failures:
+        msg = (
+            'Failures running ubuntu-advantage commands:\n'
+            '{cmd_failures}'.format(
+                cmd_failures=cmd_failures))
+        util.logexc(LOG, msg)
+        raise RuntimeError(msg)
+
+
+def maybe_install_ua_tools(cloud):
+    """Install ubuntu-advantage-tools if not present."""
+    if util.which('ubuntu-advantage'):
+        return
+    try:
+        cloud.distro.update_package_sources()
+    except Exception as e:
+        util.logexc(LOG, "Package update failed")
+        raise
+    try:
+        cloud.distro.install_packages(['ubuntu-advantage-tools'])
+    except Exception as e:
+        util.logexc(LOG, "Failed to install ubuntu-advantage-tools")
+        raise
+
+
+def handle(name, cfg, cloud, log, args):
+    cfgin = cfg.get('ubuntu-advantage')
+    if cfgin is None:
+        LOG.debug(("Skipping module named %s,"
+                   " no 'ubuntu-advantage' key in configuration"), name)
+        return
+
+    validate_cloudconfig_schema(cfg, schema)
+    maybe_install_ua_tools(cloud)
+    run_commands(cfgin.get('commands', []))
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/config/tests/test_snap.py b/cloudinit/config/tests/test_snap.py
index cb1205e..c5b4a9d 100644
--- a/cloudinit/config/tests/test_snap.py
+++ b/cloudinit/config/tests/test_snap.py
@@ -4,11 +4,12 @@ import re
 from six import StringIO
 
 from cloudinit.config.cc_snap import (
-    ASSERTIONS_FILE, add_assertions, handle, prepend_snap_commands,
-    maybe_install_squashfuse, run_commands, schema)
+    ASSERTIONS_FILE, add_assertions, handle, maybe_install_squashfuse,
+    run_commands, schema)
 from cloudinit.config.schema import validate_cloudconfig_schema
 from cloudinit import util
-from cloudinit.tests.helpers import CiTestCase, mock, wrap_and_call
+from cloudinit.tests.helpers import (
+    CiTestCase, mock, wrap_and_call, skipUnlessJsonSchema)
 
 
 SYSTEM_USER_ASSERTION = """\
@@ -158,54 +159,6 @@ class TestAddAssertions(CiTestCase):
             util.load_file(compare_file), util.load_file(assert_file))
 
 
-class TestPrepentSnapCommands(CiTestCase):
-
-    with_logs = True
-
-    def test_prepend_snap_commands_errors_on_neither_string_nor_list(self):
-        """Raise an error for each command which is not a string or list."""
-        orig_commands = ['ls', 1, {'not': 'gonna work'}, ['snap', 'list']]
-        with self.assertRaises(TypeError) as context_manager:
-            prepend_snap_commands(orig_commands)
-        self.assertEqual(
-            "Invalid snap config. These commands are not a string or list:\n"
-            "1\n{'not': 'gonna work'}",
-            str(context_manager.exception))
-
-    def test_prepend_snap_commands_warns_on_non_snap_string_commands(self):
-        """Warn on each non-snap for commands of type string."""
-        orig_commands = ['ls', 'snap list', 'touch /blah', 'snap install x']
-        fixed_commands = prepend_snap_commands(orig_commands)
-        self.assertEqual(
-            'WARNING: Non-snap commands in snap config:\n'
-            'ls\ntouch /blah\n',
-            self.logs.getvalue())
-        self.assertEqual(orig_commands, fixed_commands)
-
-    def test_prepend_snap_commands_prepends_on_non_snap_list_commands(self):
-        """Prepend 'snap' for each non-snap command of type list."""
-        orig_commands = [['ls'], ['snap', 'list'], ['snapa', '/blah'],
-                         ['snap', 'install', 'x']]
-        expected = [['snap', 'ls'], ['snap', 'list'],
-                    ['snap', 'snapa', '/blah'],
-                    ['snap', 'install', 'x']]
-        fixed_commands = prepend_snap_commands(orig_commands)
-        self.assertEqual('', self.logs.getvalue())
-        self.assertEqual(expected, fixed_commands)
-
-    def test_prepend_snap_commands_removes_first_item_when_none(self):
-        """Remove the first element of a non-snap command when it is None."""
-        orig_commands = [[None, 'ls'], ['snap', 'list'],
-                         [None, 'touch', '/blah'],
-                         ['snap', 'install', 'x']]
-        expected = [['ls'], ['snap', 'list'],
-                    ['touch', '/blah'],
-                    ['snap', 'install', 'x']]
-        fixed_commands = prepend_snap_commands(orig_commands)
-        self.assertEqual('', self.logs.getvalue())
-        self.assertEqual(expected, fixed_commands)
-
-
 class TestRunCommands(CiTestCase):
 
     with_logs = True
@@ -291,6 +244,7 @@ class TestRunCommands(CiTestCase):
         self.assertEqual('MOM\nHI\n', util.load_file(outfile))
 
 
+@skipUnlessJsonSchema()
 class TestSchema(CiTestCase):
 
     with_logs = True
@@ -466,6 +420,7 @@ class TestHandle(CiTestCase):
             util.load_file(compare_file), util.load_file(assert_file))
 
     @mock.patch('cloudinit.config.cc_snap.util.subp')
+    @skipUnlessJsonSchema()
     def test_handle_validates_schema(self, m_subp):
         """Any provided configuration is runs validate_cloudconfig_schema."""
         assert_file = self.tmp_path('snapd.assertions', dir=self.tmp)
diff --git a/cloudinit/config/tests/test_ubuntu_advantage.py b/cloudinit/config/tests/test_ubuntu_advantage.py
new file mode 100644
index 0000000..f2a59fa
--- /dev/null
+++ b/cloudinit/config/tests/test_ubuntu_advantage.py
@@ -0,0 +1,269 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import re
+from six import StringIO
+
+from cloudinit.config.cc_ubuntu_advantage import (
+    handle, maybe_install_ua_tools, run_commands, schema)
+from cloudinit.config.schema import validate_cloudconfig_schema
+from cloudinit import util
+from cloudinit.tests.helpers import CiTestCase, mock, skipUnlessJsonSchema
+
+
+# Module path used in mocks
+MPATH = 'cloudinit.config.cc_ubuntu_advantage'
+
+
+class FakeCloud(object):
+    def __init__(self, distro):
+        self.distro = distro
+
+
+class TestRunCommands(CiTestCase):
+
+    with_logs = True
+
+    def setUp(self):
+        super(TestRunCommands, self).setUp()
+        self.tmp = self.tmp_dir()
+
+    @mock.patch('%s.util.subp' % MPATH)
+    def test_run_commands_on_empty_list(self, m_subp):
+        """When provided with an empty list, run_commands does nothing."""
+        run_commands([])
+        self.assertEqual('', self.logs.getvalue())
+        m_subp.assert_not_called()
+
+    def test_run_commands_on_non_list_or_dict(self):
+        """When provided an invalid type, run_commands raises an error."""
+        with self.assertRaises(TypeError) as context_manager:
+            run_commands(commands="I'm Not Valid")
+        self.assertEqual(
+            "commands parameter was not a list or dict: I'm Not Valid",
+            str(context_manager.exception))
+
+    def test_run_command_logs_commands_and_exit_codes_to_stderr(self):
+        """All exit codes are logged to stderr."""
+        outfile = self.tmp_path('output.log', dir=self.tmp)
+
+        cmd1 = 'echo "HI" >> %s' % outfile
+        cmd2 = 'bogus command'
+        cmd3 = 'echo "MOM" >> %s' % outfile
+        commands = [cmd1, cmd2, cmd3]
+
+        mock_path = '%s.sys.stderr' % MPATH
+        with mock.patch(mock_path, new_callable=StringIO) as m_stderr:
+            with self.assertRaises(RuntimeError) as context_manager:
+                run_commands(commands=commands)
+
+        self.assertIsNotNone(
+            re.search(r'bogus: (command )?not found',
+                      str(context_manager.exception)),
+            msg='Expected bogus command not found')
+        expected_stderr_log = '\n'.join([
+            'Begin run command: {cmd}'.format(cmd=cmd1),
+            'End run command: exit(0)',
+            'Begin run command: {cmd}'.format(cmd=cmd2),
+            'ERROR: End run command: exit(127)',
+            'Begin run command: {cmd}'.format(cmd=cmd3),
+            'End run command: exit(0)\n'])
+        self.assertEqual(expected_stderr_log, m_stderr.getvalue())
+
+    def test_run_command_as_lists(self):
+        """When commands are specified as a list, run them in order."""
+        outfile = self.tmp_path('output.log', dir=self.tmp)
+
+        cmd1 = 'echo "HI" >> %s' % outfile
+        cmd2 = 'echo "MOM" >> %s' % outfile
+        commands = [cmd1, cmd2]
+        with mock.patch('%s.sys.stderr' % MPATH, new_callable=StringIO):
+            run_commands(commands=commands)
+
+        self.assertIn(
+            'DEBUG: Running user-provided ubuntu-advantage commands',
+            self.logs.getvalue())
+        self.assertEqual('HI\nMOM\n', util.load_file(outfile))
+        self.assertIn(
+            'WARNING: Non-ubuntu-advantage commands in ubuntu-advantage'
+            ' config:',
+            self.logs.getvalue())
+
+    def test_run_command_dict_sorted_as_command_script(self):
+        """When commands are a dict, sort them and run."""
+        outfile = self.tmp_path('output.log', dir=self.tmp)
+        cmd1 = 'echo "HI" >> %s' % outfile
+        cmd2 = 'echo "MOM" >> %s' % outfile
+        commands = {'02': cmd1, '01': cmd2}
+        with mock.patch('%s.sys.stderr' % MPATH, new_callable=StringIO):
+            run_commands(commands=commands)
+
+        expected_messages = [
+            'DEBUG: Running user-provided ubuntu-advantage commands']
+        for message in expected_messages:
+            self.assertIn(message, self.logs.getvalue())
+        self.assertEqual('MOM\nHI\n', util.load_file(outfile))
+
+
+@skipUnlessJsonSchema()
+class TestSchema(CiTestCase):
+
+    with_logs = True
+
+    def test_schema_warns_on_ubuntu_advantage_not_as_dict(self):
+        """If ubuntu-advantage configuration is not a dict, emit a warning."""
+        validate_cloudconfig_schema({'ubuntu-advantage': 'wrong type'}, schema)
+        self.assertEqual(
+            "WARNING: Invalid config:\nubuntu-advantage: 'wrong type' is not"
+            " of type 'object'\n",
+            self.logs.getvalue())
+
+    @mock.patch('%s.run_commands' % MPATH)
+    def test_schema_disallows_unknown_keys(self, _):
+        """Unknown keys in ubuntu-advantage configuration emit warnings."""
+        validate_cloudconfig_schema(
+            {'ubuntu-advantage': {'commands': ['ls'], 'invalid-key': ''}},
+            schema)
+        self.assertIn(
+            'WARNING: Invalid config:\nubuntu-advantage: Additional properties'
+            " are not allowed ('invalid-key' was unexpected)",
+            self.logs.getvalue())
+
+    def test_warn_schema_requires_commands(self):
+        """Warn when ubuntu-advantage configuration lacks commands."""
+        validate_cloudconfig_schema(
+            {'ubuntu-advantage': {}}, schema)
+        self.assertEqual(
+            "WARNING: Invalid config:\nubuntu-advantage: 'commands' is a"
+            " required property\n",
+            self.logs.getvalue())
+
+    @mock.patch('%s.run_commands' % MPATH)
+    def test_warn_schema_commands_is_not_list_or_dict(self, _):
+        """Warn when ubuntu-advantage:commands config is not a list or dict."""
+        validate_cloudconfig_schema(
+            {'ubuntu-advantage': {'commands': 'broken'}}, schema)
+        self.assertEqual(
+            "WARNING: Invalid config:\nubuntu-advantage.commands: 'broken' is"
+            " not of type 'object', 'array'\n",
+            self.logs.getvalue())
+
+    @mock.patch('%s.run_commands' % MPATH)
+    def test_warn_schema_when_commands_is_empty(self, _):
+        """Emit warnings when ubuntu-advantage:commands is empty."""
+        validate_cloudconfig_schema(
+            {'ubuntu-advantage': {'commands': []}}, schema)
+        validate_cloudconfig_schema(
+            {'ubuntu-advantage': {'commands': {}}}, schema)
+        self.assertEqual(
+            "WARNING: Invalid config:\nubuntu-advantage.commands: [] is too"
+            " short\nWARNING: Invalid config:\nubuntu-advantage.commands: {}"
+            " does not have enough properties\n",
+            self.logs.getvalue())
+
+    @mock.patch('%s.run_commands' % MPATH)
+    def test_schema_when_commands_are_list_or_dict(self, _):
+        """No warnings when ubuntu-advantage:commands are a list or dict."""
+        validate_cloudconfig_schema(
+            {'ubuntu-advantage': {'commands': ['valid']}}, schema)
+        validate_cloudconfig_schema(
+            {'ubuntu-advantage': {'commands': {'01': 'also valid'}}}, schema)
+        self.assertEqual('', self.logs.getvalue())
+
+
+class TestHandle(CiTestCase):
+
+    with_logs = True
+
+    def setUp(self):
+        super(TestHandle, self).setUp()
+        self.tmp = self.tmp_dir()
+
+    @mock.patch('%s.run_commands' % MPATH)
+    @mock.patch('%s.validate_cloudconfig_schema' % MPATH)
+    def test_handle_no_config(self, m_schema, m_run):
+        """When no ua-related configuration is provided, nothing happens."""
+        cfg = {}
+        handle('ua-test', cfg=cfg, cloud=None, log=self.logger, args=None)
+        self.assertIn(
+            "DEBUG: Skipping module named ua-test, no 'ubuntu-advantage' key"
+            " in config",
+            self.logs.getvalue())
+        m_schema.assert_not_called()
+        m_run.assert_not_called()
+
+    @mock.patch('%s.maybe_install_ua_tools' % MPATH)
+    def test_handle_tries_to_install_ubuntu_advantage_tools(self, m_install):
+        """If ubuntu_advantage is provided, try installing ua-tools package."""
+        cfg = {'ubuntu-advantage': {}}
+        mycloud = FakeCloud(None)
+        handle('nomatter', cfg=cfg, cloud=mycloud, log=self.logger, args=None)
+        m_install.assert_called_once_with(mycloud)
+
+    @mock.patch('%s.maybe_install_ua_tools' % MPATH)
+    def test_handle_runs_commands_provided(self, m_install):
+        """When commands are specified as a list, run them."""
+        outfile = self.tmp_path('output.log', dir=self.tmp)
+
+        cfg = {
+            'ubuntu-advantage': {'commands': ['echo "HI" >> %s' % outfile,
+                                              'echo "MOM" >> %s' % outfile]}}
+        mock_path = '%s.sys.stderr' % MPATH
+        with mock.patch(mock_path, new_callable=StringIO):
+            handle('nomatter', cfg=cfg, cloud=None, log=self.logger, args=None)
+        self.assertEqual('HI\nMOM\n', util.load_file(outfile))
+
+
+class TestMaybeInstallUATools(CiTestCase):
+
+    with_logs = True
+
+    def setUp(self):
+        super(TestMaybeInstallUATools, self).setUp()
+        self.tmp = self.tmp_dir()
+
+    @mock.patch('%s.util.which' % MPATH)
+    def test_maybe_install_ua_tools_noop_when_ua_tools_present(self, m_which):
+        """Do nothing if ubuntu-advantage-tools already exists."""
+        m_which.return_value = '/usr/bin/ubuntu-advantage'  # already installed
+        distro = mock.MagicMock()
+        distro.update_package_sources.side_effect = RuntimeError(
+            'Some apt error')
+        maybe_install_ua_tools(cloud=FakeCloud(distro))  # No RuntimeError
+
+    @mock.patch('%s.util.which' % MPATH)
+    def test_maybe_install_ua_tools_raises_update_errors(self, m_which):
+        """maybe_install_ua_tools logs and raises apt update errors."""
+        m_which.return_value = None
+        distro = mock.MagicMock()
+        distro.update_package_sources.side_effect = RuntimeError(
+            'Some apt error')
+        with self.assertRaises(RuntimeError) as context_manager:
+            maybe_install_ua_tools(cloud=FakeCloud(distro))
+        self.assertEqual('Some apt error', str(context_manager.exception))
+        self.assertIn('Package update failed\nTraceback', self.logs.getvalue())
+
+    @mock.patch('%s.util.which' % MPATH)
+    def test_maybe_install_ua_raises_install_errors(self, m_which):
+        """maybe_install_ua_tools logs and raises package install errors."""
+        m_which.return_value = None
+        distro = mock.MagicMock()
+        distro.update_package_sources.return_value = None
+        distro.install_packages.side_effect = RuntimeError(
+            'Some install error')
+        with self.assertRaises(RuntimeError) as context_manager:
+            maybe_install_ua_tools(cloud=FakeCloud(distro))
+        self.assertEqual('Some install error', str(context_manager.exception))
+        self.assertIn(
+            'Failed to install ubuntu-advantage-tools\n', self.logs.getvalue())
+
+    @mock.patch('%s.util.which' % MPATH)
+    def test_maybe_install_ua_tools_happy_path(self, m_which):
+        """maybe_install_ua_tools installs ubuntu-advantage-tools."""
+        m_which.return_value = None
+        distro = mock.MagicMock()  # No errors raised
+        maybe_install_ua_tools(cloud=FakeCloud(distro))
+        distro.update_package_sources.assert_called_once_with()
+        distro.install_packages.assert_called_once_with(
+            ['ubuntu-advantage-tools'])
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py
index d6c61e4..dc3f0fc 100644
--- a/cloudinit/ec2_utils.py
+++ b/cloudinit/ec2_utils.py
@@ -135,10 +135,8 @@ class MetadataMaterializer(object):
 
 
 def _skip_retry_on_codes(status_codes, _request_args, cause):
-    """Returns if a request should retry based on a given set of codes that
-    case retrying to be stopped/skipped.
-    """
-    return cause.code in status_codes
+    """Returns False if cause.code is in status_codes."""
+    return cause.code not in status_codes
 
 
 def get_instance_userdata(api_version='latest',
diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py
index 6bee3d3..6344348 100644
--- a/cloudinit/net/netplan.py
+++ b/cloudinit/net/netplan.py
@@ -336,22 +336,15 @@ class Renderer(renderer.Renderer):
                 _extract_addresses(ifcfg, vlan)
                 vlans.update({ifname: vlan})
 
-        # inject global nameserver values under each physical interface
-        if nameservers:
-            for _eth, cfg in ethernets.items():
-                nscfg = cfg.get('nameservers', {})
-                addresses = nscfg.get('addresses', [])
-                addresses += nameservers
-                nscfg.update({'addresses': addresses})
-                cfg.update({'nameservers': nscfg})
-
-        if searchdomains:
-            for _eth, cfg in ethernets.items():
-                nscfg = cfg.get('nameservers', {})
-                search = nscfg.get('search', [])
-                search += searchdomains
-                nscfg.update({'search': search})
-                cfg.update({'nameservers': nscfg})
+        # inject global nameserver values under each all interface which
+        # has addresses and do not already have a DNS configuration
+        if nameservers or searchdomains:
+            nscfg = {'addresses': nameservers, 'search': searchdomains}
+            for section in [ethernets, wifis, bonds, bridges, vlans]:
+                for _name, cfg in section.items():
+                    if 'nameservers' in cfg or 'addresses' not in cfg:
+                        continue
+                    cfg.update({'nameservers': nscfg})
 
         # workaround yaml dictionary key sorting when dumping
         def _render_section(name, section):
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 0bb7fad..0ee622e 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -20,7 +20,7 @@ from cloudinit import net
 from cloudinit.net.dhcp import EphemeralDHCPv4
 from cloudinit import sources
 from cloudinit.sources.helpers.azure import get_metadata_from_fabric
-from cloudinit.url_helper import readurl, wait_for_url, UrlError
+from cloudinit.url_helper import readurl, UrlError
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -49,7 +49,6 @@ DEFAULT_FS = 'ext4'
 AZURE_CHASSIS_ASSET_TAG = '7783-7084-3265-9085-8269-3286-77'
 REPROVISION_MARKER_FILE = "/var/lib/cloud/data/poll_imds"
 IMDS_URL = "http://169.254.169.254/metadata/reprovisiondata"
-IMDS_RETRIES = 5
 
 
 def find_storvscid_from_sysctl_pnpinfo(sysctl_out, deviceid):
@@ -451,36 +450,24 @@ class DataSourceAzure(sources.DataSource):
         headers = {"Metadata": "true"}
         LOG.debug("Start polling IMDS")
 
-        def sleep_cb(response, loop_n):
-            return 1
-
-        def exception_cb(msg, exception):
+        def exc_cb(msg, exception):
             if isinstance(exception, UrlError) and exception.code == 404:
-                return
-            LOG.warning("Exception during polling. Will try DHCP.",
-                        exc_info=True)
-
+                return True
             # If we get an exception while trying to call IMDS, we
             # call DHCP and setup the ephemeral network to acquire the new IP.
-            raise exception
+            return False
 
         need_report = report_ready
-        for i in range(IMDS_RETRIES):
+        while True:
             try:
                 with EphemeralDHCPv4() as lease:
                     if need_report:
                         self._report_ready(lease=lease)
                         need_report = False
-                    wait_for_url([url], max_wait=None, timeout=60,
-                                 status_cb=LOG.info,
-                                 headers_cb=lambda url: headers, sleep_time=1,
-                                 exception_cb=exception_cb,
-                                 sleep_time_cb=sleep_cb)
-                    return str(readurl(url, headers=headers))
-            except Exception:
-                LOG.debug("Exception during polling-retrying dhcp" +
-                          " %d more time(s).", (IMDS_RETRIES - i),
-                          exc_info=True)
+                    return readurl(url, timeout=1, headers=headers,
+                                   exception_cb=exc_cb, infinite=True).contents
+            except UrlError:
+                pass
 
     def _report_ready(self, lease):
         """Tells the fabric provisioning has completed
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index b8db626..c7b5fe5 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -14,6 +14,7 @@ from cloudinit import util
 
 from cloudinit.net import eni
 
+from cloudinit.sources.DataSourceIBMCloud import get_ibm_platform
 from cloudinit.sources.helpers import openstack
 
 LOG = logging.getLogger(__name__)
@@ -255,6 +256,15 @@ def find_candidate_devs(probe_optical=True):
     # an unpartitioned block device (ex sda, not sda1)
     devices = [d for d in candidates
                if d in by_label or not util.is_partition(d)]
+
+    if devices:
+        # IBMCloud uses config-2 label, but limited to a single UUID.
+        ibm_platform, ibm_path = get_ibm_platform()
+        if ibm_path in devices:
+            devices.remove(ibm_path)
+            LOG.debug("IBMCloud device '%s' (%s) removed from candidate list",
+                      ibm_path, ibm_platform)
+
     return devices
 
 
diff --git a/cloudinit/sources/DataSourceIBMCloud.py b/cloudinit/sources/DataSourceIBMCloud.py
new file mode 100644
index 0000000..02b3d56
--- /dev/null
+++ b/cloudinit/sources/DataSourceIBMCloud.py
@@ -0,0 +1,325 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+"""Datasource for IBMCloud.
+
+IBMCloud is also know as SoftLayer or BlueMix.
+IBMCloud hypervisor is xen (2018-03-10).
+
+There are 2 different api exposed launch methods.
+ * template: This is the legacy method of launching instances.
+   When booting from an image template, the system boots first into
+   a "provisioning" mode.  There, host <-> guest mechanisms are utilized
+   to execute code in the guest and provision it.
+
+   Cloud-init will disable itself when it detects that it is in the
+   provisioning mode.  It detects this by the presence of
+   a file '/root/provisioningConfiguration.cfg'.
+
+   When provided with user-data, the "first boot" will contain a
+   ConfigDrive-like disk labeled with 'METADATA'.  If there is no user-data
+   provided, then there is no data-source.
+
+   Cloud-init never does any network configuration in this mode.
+
+ * os_code: Essentially "launch by OS Code" (Operating System Code).
+   This is a more modern approach.  There is no specific "provisioning" boot.
+   Instead, cloud-init does all the customization.  With or without
+   user-data provided, an OpenStack ConfigDrive like disk is attached.
+
+   Only disks with label 'config-2' and UUID '9796-932E' are considered.
+   This is to avoid this datasource claiming ConfigDrive.  This does
+   mean that 1 in 8^16 (~4 billion) Xen ConfigDrive systems will be
+   incorrectly identified as IBMCloud.
+
+TODO:
+ * is uuid (/sys/hypervisor/uuid) stable for life of an instance?
+   it seems it is not the same as data's uuid in the os_code case
+   but is in the template case.
+
+"""
+import base64
+import json
+import os
+
+from cloudinit import log as logging
+from cloudinit import sources
+from cloudinit.sources.helpers import openstack
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+IBM_CONFIG_UUID = "9796-932E"
+
+
+class Platforms(object):
+    TEMPLATE_LIVE_METADATA = "Template/Live/Metadata"
+    TEMPLATE_LIVE_NODATA = "UNABLE TO BE IDENTIFIED."
+    TEMPLATE_PROVISIONING_METADATA = "Template/Provisioning/Metadata"
+    TEMPLATE_PROVISIONING_NODATA = "Template/Provisioning/No-Metadata"
+    OS_CODE = "OS-Code/Live"
+
+
+PROVISIONING = (
+    Platforms.TEMPLATE_PROVISIONING_METADATA,
+    Platforms.TEMPLATE_PROVISIONING_NODATA)
+
+
+class DataSourceIBMCloud(sources.DataSource):
+
+    dsname = 'IBMCloud'
+    system_uuid = None
+
+    def __init__(self, sys_cfg, distro, paths):
+        super(DataSourceIBMCloud, self).__init__(sys_cfg, distro, paths)
+        self.source = None
+        self._network_config = None
+        self.network_json = None
+        self.platform = None
+
+    def __str__(self):
+        root = super(DataSourceIBMCloud, self).__str__()
+        mstr = "%s [%s %s]" % (root, self.platform, self.source)
+        return mstr
+
+    def _get_data(self):
+        results = read_md()
+        if results is None:
+            return False
+
+        self.source = results['source']
+        self.platform = results['platform']
+        self.metadata = results['metadata']
+        self.userdata_raw = results.get('userdata')
+        self.network_json = results.get('networkdata')
+        vd = results.get('vendordata')
+        self.vendordata_pure = vd
+        self.system_uuid = results['system-uuid']
+        try:
+            self.vendordata_raw = sources.convert_vendordata(vd)
+        except ValueError as e:
+            LOG.warning("Invalid content in vendor-data: %s", e)
+            self.vendordata_raw = None
+
+        return True
+
+    def check_instance_id(self, sys_cfg):
+        """quickly (local check only) if self.instance_id is still valid
+
+        in Template mode, the system uuid (/sys/hypervisor/uuid) is the
+        same as found in the METADATA disk.  But that is not true in OS_CODE
+        mode.  So we read the system_uuid and keep that for later compare."""
+        if self.system_uuid is None:
+            return False
+        return self.system_uuid == _read_system_uuid()
+
+    @property
+    def network_config(self):
+        if self.platform != Platforms.OS_CODE:
+            # If deployed from template, an agent in the provisioning
+            # environment handles networking configuration. Not cloud-init.
+            return {'config': 'disabled', 'version': 1}
+        if self._network_config is None:
+            if self.network_json is not None:
+                LOG.debug("network config provided via network_json")
+                self._network_config = openstack.convert_net_json(
+                    self.network_json, known_macs=None)
+            else:
+                LOG.debug("no network configuration available.")
+        return self._network_config
+
+
+def _read_system_uuid():
+    uuid_path = "/sys/hypervisor/uuid"
+    if not os.path.isfile(uuid_path):
+        return None
+    return util.load_file(uuid_path).strip().lower()
+
+
+def _is_xen():
+    return os.path.exists("/proc/xen")
+
+
+def _is_ibm_provisioning():
+    return os.path.exists("/root/provisioningConfiguration.cfg")
+
+
+def get_ibm_platform():
+    """Return a tuple (Platform, path)
+
+    If this is Not IBM cloud, then the return value is (None, None).
+    An instance in provisioning mode is considered running on IBM cloud."""
+    label_mdata = "METADATA"
+    label_cfg2 = "CONFIG-2"
+    not_found = (None, None)
+
+    if not _is_xen():
+        return not_found
+
+    # fslabels contains only the first entry with a given label.
+    fslabels = {}
+    try:
+        devs = util.blkid()
+    except util.ProcessExecutionError as e:
+        LOG.warning("Failed to run blkid: %s", e)
+        return (None, None)
+
+    for dev in sorted(devs.keys()):
+        data = devs[dev]
+        label = data.get("LABEL", "").upper()
+        uuid = data.get("UUID", "").upper()
+        if label not in (label_mdata, label_cfg2):
+            continue
+        if label in fslabels:
+            LOG.warning("Duplicate fslabel '%s'. existing=%s current=%s",
+                        label, fslabels[label], data)
+            continue
+        if label == label_cfg2 and uuid != IBM_CONFIG_UUID:
+            LOG.debug("Skipping %s with LABEL=%s due to uuid != %s: %s",
+                      dev, label, uuid, data)
+            continue
+        fslabels[label] = data
+
+    metadata_path = fslabels.get(label_mdata, {}).get('DEVNAME')
+    cfg2_path = fslabels.get(label_cfg2, {}).get('DEVNAME')
+
+    if cfg2_path:
+        return (Platforms.OS_CODE, cfg2_path)
+    elif metadata_path:
+        if _is_ibm_provisioning():
+            return (Platforms.TEMPLATE_PROVISIONING_METADATA, metadata_path)
+        else:
+            return (Platforms.TEMPLATE_LIVE_METADATA, metadata_path)
+    elif _is_ibm_provisioning():
+            return (Platforms.TEMPLATE_PROVISIONING_NODATA, None)
+    return not_found
+
+
+def read_md():
+    """Read data from IBM Cloud.
+
+    @return: None if not running on IBM Cloud.
+             dictionary with guaranteed fields: metadata, version
+             and optional fields: userdata, vendordata, networkdata.
+             Also includes the system uuid from /sys/hypervisor/uuid."""
+    platform, path = get_ibm_platform()
+    if platform is None:
+        LOG.debug("This is not an IBMCloud platform.")
+        return None
+    elif platform in PROVISIONING:
+        LOG.debug("Cloud-init is disabled during provisioning: %s.",
+                  platform)
+        return None
+
+    ret = {'platform': platform, 'source': path,
+           'system-uuid': _read_system_uuid()}
+
+    try:
+        if os.path.isdir(path):
+            results = metadata_from_dir(path)
+        else:
+            results = util.mount_cb(path, metadata_from_dir)
+    except BrokenMetadata as e:
+        raise RuntimeError(
+            "Failed reading IBM config disk (platform=%s path=%s): %s" %
+            (platform, path, e))
+
+    ret.update(results)
+    return ret
+
+
+class BrokenMetadata(IOError):
+    pass
+
+
+def metadata_from_dir(source_dir):
+    """Walk source_dir extracting standardized metadata.
+
+    Certain metadata keys are renamed to present a standardized set of metadata
+    keys.
+
+    This function has a lot in common with ConfigDriveReader.read_v2 but
+    there are a number of inconsistencies, such key renames and as only
+    presenting a 'latest' version which make it an unlikely candidate to share
+    code.
+
+    @return: Dict containing translated metadata, userdata, vendordata,
+        networkdata as present.
+    """
+
+    def opath(fname):
+        return os.path.join("openstack", "latest", fname)
+
+    def load_json_bytes(blob):
+        return json.loads(blob.decode('utf-8'))
+
+    files = [
+        # tuples of (results_name, path, translator)
+        ('metadata_raw', opath('meta_data.json'), load_json_bytes),
+        ('userdata', opath('user_data'), None),
+        ('vendordata', opath('vendor_data.json'), load_json_bytes),
+        ('networkdata', opath('network_data.json'), load_json_bytes),
+    ]
+
+    results = {}
+    for (name, path, transl) in files:
+        fpath = os.path.join(source_dir, path)
+        raw = None
+        try:
+            raw = util.load_file(fpath, decode=False)
+        except IOError as e:
+            LOG.debug("Failed reading path '%s': %s", fpath, e)
+
+        if raw is None or transl is None:
+            data = raw
+        else:
+            try:
+                data = transl(raw)
+            except Exception as e:
+                raise BrokenMetadata("Failed decoding %s: %s" % (path, e))
+
+        results[name] = data
+
+    if results.get('metadata_raw') is None:
+        raise BrokenMetadata(
+            "%s missing required file 'meta_data.json'" % source_dir)
+
+    results['metadata'] = {}
+
+    md_raw = results['metadata_raw']
+    md = results['metadata']
+    if 'random_seed' in md_raw:
+        try:
+            md['random_seed'] = base64.b64decode(md_raw['random_seed'])
+        except (ValueError, TypeError) as e:
+            raise BrokenMetadata(
+                "Badly formatted metadata random_seed entry: %s" % e)
+
+    renames = (
+        ('public_keys', 'public-keys'), ('hostname', 'local-hostname'),
+        ('uuid', 'instance-id'))
+    for mdname, newname in renames:
+        if mdname in md_raw:
+            md[newname] = md_raw[mdname]
+
+    return results
+
+
+# Used to match classes to dependencies
+datasources = [
+    (DataSourceIBMCloud, (sources.DEP_FILESYSTEM,)),
+]
+
+
+# Return a list of data sources that match this set of dependencies
+def get_datasource_list(depends):
+    return sources.list_from_depends(depends, datasources)
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(description='Query IBM Cloud Metadata')
+    args = parser.parse_args()
+    data = read_md()
+    print(util.json_dumps(data))
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/sources/DataSourceScaleway.py b/cloudinit/sources/DataSourceScaleway.py
index 9005624..e2502b0 100644
--- a/cloudinit/sources/DataSourceScaleway.py
+++ b/cloudinit/sources/DataSourceScaleway.py
@@ -113,9 +113,9 @@ def query_data_api_once(api_address, timeout, requests_session):
             retries=0,
             session=requests_session,
             # If the error is a HTTP/404 or a ConnectionError, go into raise
-            # block below.
-            exception_cb=lambda _, exc: exc.code == 404 or (
-                isinstance(exc.cause, requests.exceptions.ConnectionError)
+            # block below and don't bother retrying.
+            exception_cb=lambda _, exc: exc.code != 404 and (
+                not isinstance(exc.cause, requests.exceptions.ConnectionError)
             )
         )
         return util.decode_binary(resp.contents)
diff --git a/cloudinit/subp.py b/cloudinit/subp.py
new file mode 100644
index 0000000..0ad0930
--- /dev/null
+++ b/cloudinit/subp.py
@@ -0,0 +1,57 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+"""Common utility functions for interacting with subprocess."""
+
+# TODO move subp shellify and runparts related functions out of util.py
+
+import logging
+
+LOG = logging.getLogger(__name__)
+
+
+def prepend_base_command(base_command, commands):
+    """Ensure user-provided commands start with base_command; warn otherwise.
+
+    Each command is either a list or string. Perform the following:
+       - If the command is a list, pop the first element if it is None
+       - If the command is a list, insert base_command as the first element if
+         not present.
+       - When the command is a string not starting with 'base-command', warn.
+
+    Allow flexibility to provide non-base-command environment/config setup if
+    needed.
+
+    @commands: List of commands. Each command element is a list or string.
+
+    @return: List of 'fixed up' commands.
+    @raise: TypeError on invalid config item type.
+    """
+    warnings = []
+    errors = []
+    fixed_commands = []
+    for command in commands:
+        if isinstance(command, list):
+            if command[0] is None:  # Avoid warnings by specifying None
+                command = command[1:]
+            elif command[0] != base_command:  # Automatically prepend
+                command.insert(0, base_command)
+        elif isinstance(command, str):
+            if not command.startswith('%s ' % base_command):
+                warnings.append(command)
+        else:
+            errors.append(str(command))
+            continue
+        fixed_commands.append(command)
+
+    if warnings:
+        LOG.warning(
+            'Non-%s commands in %s config:\n%s',
+            base_command, base_command, '\n'.join(warnings))
+    if errors:
+        raise TypeError(
+            'Invalid {name} config.'
+            ' These commands are not a string or list:\n{errors}'.format(
+                name=base_command, errors='\n'.join(errors)))
+    return fixed_commands
+
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/tests/test_subp.py b/cloudinit/tests/test_subp.py
new file mode 100644
index 0000000..448097d
--- /dev/null
+++ b/cloudinit/tests/test_subp.py
@@ -0,0 +1,61 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Tests for cloudinit.subp utility functions"""
+
+from cloudinit import subp
+from cloudinit.tests.helpers import CiTestCase
+
+
+class TestPrependBaseCommands(CiTestCase):
+
+    with_logs = True
+
+    def test_prepend_base_command_errors_on_neither_string_nor_list(self):
+        """Raise an error for each command which is not a string or list."""
+        orig_commands = ['ls', 1, {'not': 'gonna work'}, ['basecmd', 'list']]
+        with self.assertRaises(TypeError) as context_manager:
+            subp.prepend_base_command(
+                base_command='basecmd', commands=orig_commands)
+        self.assertEqual(
+            "Invalid basecmd config. These commands are not a string or"
+            " list:\n1\n{'not': 'gonna work'}",
+            str(context_manager.exception))
+
+    def test_prepend_base_command_warns_on_non_base_string_commands(self):
+        """Warn on each non-base for commands of type string."""
+        orig_commands = [
+            'ls', 'basecmd list', 'touch /blah', 'basecmd install x']
+        fixed_commands = subp.prepend_base_command(
+            base_command='basecmd', commands=orig_commands)
+        self.assertEqual(
+            'WARNING: Non-basecmd commands in basecmd config:\n'
+            'ls\ntouch /blah\n',
+            self.logs.getvalue())
+        self.assertEqual(orig_commands, fixed_commands)
+
+    def test_prepend_base_command_prepends_on_non_base_list_commands(self):
+        """Prepend 'basecmd' for each non-basecmd command of type list."""
+        orig_commands = [['ls'], ['basecmd', 'list'], ['basecmda', '/blah'],
+                         ['basecmd', 'install', 'x']]
+        expected = [['basecmd', 'ls'], ['basecmd', 'list'],
+                    ['basecmd', 'basecmda', '/blah'],
+                    ['basecmd', 'install', 'x']]
+        fixed_commands = subp.prepend_base_command(
+            base_command='basecmd', commands=orig_commands)
+        self.assertEqual('', self.logs.getvalue())
+        self.assertEqual(expected, fixed_commands)
+
+    def test_prepend_base_command_removes_first_item_when_none(self):
+        """Remove the first element of a non-basecmd when it is None."""
+        orig_commands = [[None, 'ls'], ['basecmd', 'list'],
+                         [None, 'touch', '/blah'],
+                         ['basecmd', 'install', 'x']]
+        expected = [['ls'], ['basecmd', 'list'],
+                    ['touch', '/blah'],
+                    ['basecmd', 'install', 'x']]
+        fixed_commands = subp.prepend_base_command(
+            base_command='basecmd', commands=orig_commands)
+        self.assertEqual('', self.logs.getvalue())
+        self.assertEqual(expected, fixed_commands)
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py
index d30643d..3f37dbb 100644
--- a/cloudinit/tests/test_util.py
+++ b/cloudinit/tests/test_util.py
@@ -3,6 +3,7 @@
 """Tests for cloudinit.util"""
 
 import logging
+from textwrap import dedent
 
 import cloudinit.util as util
 
@@ -140,4 +141,75 @@ class TestGetHostnameFqdn(CiTestCase):
             [{'fqdn': True, 'metadata_only': True},
              {'metadata_only': True}], mycloud.calls)
 
+
+class TestBlkid(CiTestCase):
+    ids = {
+        "id01": "1111-1111",
+        "id02": "22222222-2222",
+        "id03": "33333333-3333",
+        "id04": "44444444-4444",
+        "id05": "55555555-5555-5555-5555-555555555555",
+        "id06": "66666666-6666-6666-6666-666666666666",
+        "id07": "52894610484658920398",
+        "id08": "86753098675309867530",
+        "id09": "99999999-9999-9999-9999-999999999999",
+    }
+
+    blkid_out = dedent("""\
+        /dev/loop0: TYPE="squashfs"
+        /dev/loop1: TYPE="squashfs"
+        /dev/loop2: TYPE="squashfs"
+        /dev/loop3: TYPE="squashfs"
+        /dev/sda1: UUID="{id01}" TYPE="vfat" PARTUUID="{id02}"
+        /dev/sda2: UUID="{id03}" TYPE="ext4" PARTUUID="{id04}"
+        /dev/sda3: UUID="{id05}" TYPE="ext4" PARTUUID="{id06}"
+        /dev/sda4: LABEL="default" UUID="{id07}" UUID_SUB="{id08}" """
+                       """TYPE="zfs_member" PARTUUID="{id09}"
+        /dev/loop4: TYPE="squashfs"
+      """)
+
+    maxDiff = None
+
+    def _get_expected(self):
+        return ({
+            "/dev/loop0": {"DEVNAME": "/dev/loop0", "TYPE": "squashfs"},
+            "/dev/loop1": {"DEVNAME": "/dev/loop1", "TYPE": "squashfs"},
+            "/dev/loop2": {"DEVNAME": "/dev/loop2", "TYPE": "squashfs"},
+            "/dev/loop3": {"DEVNAME": "/dev/loop3", "TYPE": "squashfs"},
+            "/dev/loop4": {"DEVNAME": "/dev/loop4", "TYPE": "squashfs"},
+            "/dev/sda1": {"DEVNAME": "/dev/sda1", "TYPE": "vfat",
+                          "UUID": self.ids["id01"],
+                          "PARTUUID": self.ids["id02"]},
+            "/dev/sda2": {"DEVNAME": "/dev/sda2", "TYPE": "ext4",
+                          "UUID": self.ids["id03"],
+                          "PARTUUID": self.ids["id04"]},
+            "/dev/sda3": {"DEVNAME": "/dev/sda3", "TYPE": "ext4",
+                          "UUID": self.ids["id05"],
+                          "PARTUUID": self.ids["id06"]},
+            "/dev/sda4": {"DEVNAME": "/dev/sda4", "TYPE": "zfs_member",
+                          "LABEL": "default",
+                          "UUID": self.ids["id07"],
+                          "UUID_SUB": self.ids["id08"],
+                          "PARTUUID": self.ids["id09"]},
+        })
+
+    @mock.patch("cloudinit.util.subp")
+    def test_functional_blkid(self, m_subp):
+        m_subp.return_value = (
+            self.blkid_out.format(**self.ids), "")
+        self.assertEqual(self._get_expected(), util.blkid())
+        m_subp.assert_called_with(["blkid", "-o", "full"], capture=True,
+                                  decode="replace")
+
+    @mock.patch("cloudinit.util.subp")
+    def test_blkid_no_cache_uses_no_cache(self, m_subp):
+        """blkid should turn off cache if disable_cache is true."""
+        m_subp.return_value = (
+            self.blkid_out.format(**self.ids), "")
+        self.assertEqual(self._get_expected(),
+                         util.blkid(disable_cache=True))
+        m_subp.assert_called_with(["blkid", "-o", "full", "-c", "/dev/null"],
+                                  capture=True, decode="replace")
+
+
 # vi: ts=4 expandtab
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index 4e814a5..03a573a 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -16,7 +16,7 @@ import time
 
 from email.utils import parsedate
 from functools import partial
-
+from itertools import count
 from requests import exceptions
 
 from six.moves.urllib.parse import (
@@ -172,7 +172,7 @@ def _get_ssl_args(url, ssl_details):
 def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
             headers=None, headers_cb=None, ssl_details=None,
             check_status=True, allow_redirects=True, exception_cb=None,
-            session=None):
+            session=None, infinite=False):
     url = _cleanurl(url)
     req_args = {
         'url': url,
@@ -220,7 +220,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
     excps = []
     # Handle retrying ourselves since the built-in support
     # doesn't handle sleeping between tries...
-    for i in range(0, manual_tries):
+    # Infinitely retry if infinite is True
+    for i in count() if infinite else range(0, manual_tries):
         req_args['headers'] = headers_cb(url)
         filtered_req_args = {}
         for (k, v) in req_args.items():
@@ -229,7 +230,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
             filtered_req_args[k] = v
         try:
             LOG.debug("[%s/%s] open '%s' with %s configuration", i,
-                      manual_tries, url, filtered_req_args)
+                      "infinite" if infinite else manual_tries, url,
+                      filtered_req_args)
 
             if session is None:
                 session = requests.Session()
@@ -258,11 +260,13 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
                     # ssl exceptions are not going to get fixed by waiting a
                     # few seconds
                     break
-            if exception_cb and exception_cb(req_args.copy(), excps[-1]):
-                # if an exception callback was given it should return None
-                # a true-ish value means to break and re-raise the exception
+            if exception_cb and not exception_cb(req_args.copy(), excps[-1]):
+                # if an exception callback was given, it should return True
+                # to continue retrying and False to break and re-raise the
+                # exception
                 break
-            if i + 1 < manual_tries and sec_between > 0:
+            if (infinite and sec_between > 0) or \
+               (i + 1 < manual_tries and sec_between > 0):
                 LOG.debug("Please wait %s seconds while we wait to try again",
                           sec_between)
                 time.sleep(sec_between)
diff --git a/cloudinit/util.py b/cloudinit/util.py
index cae8b19..fb4ee5f 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -1237,6 +1237,37 @@ def find_devs_with(criteria=None, oformat='device',
     return entries
 
 
+def blkid(devs=None, disable_cache=False):
+    """Get all device tags details from blkid.
+
+    @param devs: Optional list of device paths you wish to query.
+    @param disable_cache: Bool, set True to start with clean cache.
+
+    @return: Dict of key value pairs of info for the device.
+    """
+    if devs is None:
+        devs = []
+    else:
+        devs = list(devs)
+
+    cmd = ['blkid', '-o', 'full']
+    if disable_cache:
+        cmd.extend(['-c', '/dev/null'])
+    cmd.extend(devs)
+
+    # we have to decode with 'replace' as shelx.split (called by
+    # load_shell_content) can't take bytes.  So this is potentially
+    # lossy of non-utf-8 chars in blkid output.
+    out, _ = subp(cmd, capture=True, decode="replace")
+    ret = {}
+    for line in out.splitlines():
+        dev, _, data = line.partition(":")
+        ret[dev] = load_shell_content(data)
+        ret[dev]["DEVNAME"] = dev
+
+    return ret
+
+
 def peek_file(fname, max_bytes):
     LOG.debug("Peeking at %s (max_bytes=%s)", fname, max_bytes)
     with open(fname, 'rb') as ifh:
diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl
index 56a34fa..3129d4e 100644
--- a/config/cloud.cfg.tmpl
+++ b/config/cloud.cfg.tmpl
@@ -87,6 +87,9 @@ cloud_config_modules:
  - apt-pipelining
  - apt-configure
 {% endif %}
+{% if variant in ["ubuntu"] %}
+ - ubuntu-advantage
+{% endif %}
 {% if variant in ["suse"] %}
  - zypper-add-repo
 {% endif %}
diff --git a/debian/changelog b/debian/changelog
index c333306..9da09db 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,22 @@
+cloud-init (18.1-35-ge0f644b7-0ubuntu1) bionic; urgency=medium
+
+  * New upstream snapshot.
+    - IBMCloud: Initial IBM Cloud datasource.
+    - tests: remove jsonschema from xenial tox environment.
+    - tests: Fix newly added schema unit tests to skip if no jsonschema.
+    - ec2: Adjust ec2 datasource after exception_cb change.
+    - Reduce AzurePreprovisioning HTTP timeouts.
+      [Douglas Jordan] (LP: #1752977)
+    - Revert the logic of exception_cb in read_url.
+      [Kurt Garloff] (LP: #1702160, #1298921)
+    - ubuntu-advantage: Add new config module to support
+      ubuntu-advantage-tools
+    - Handle global dns entries in netplan (LP: #1750884)
+    - Identify OpenTelekomCloud Xen as OpenStack DS.
+      [Kurt Garloff] (LP: #1756471)
+
+ -- Chad Smith <[e-mail]>  Fri, 23 Mar 2018 17:21:37 -0600
+
 cloud-init (18.1-26-g685f9901-0ubuntu1) bionic; urgency=medium
 
   * debian/cloud-init.templates: Enable Hetzner Cloud datasource.
diff --git a/doc/rtd/topics/modules.rst b/doc/rtd/topics/modules.rst
index a0f6812..d9720f6 100644
--- a/doc/rtd/topics/modules.rst
+++ b/doc/rtd/topics/modules.rst
@@ -53,6 +53,7 @@ Modules
 .. automodule:: cloudinit.config.cc_ssh_authkey_fingerprints
 .. automodule:: cloudinit.config.cc_ssh_import_id
 .. automodule:: cloudinit.config.cc_timezone
+.. automodule:: cloudinit.config.cc_ubuntu_advantage
 .. automodule:: cloudinit.config.cc_update_etc_hosts
 .. automodule:: cloudinit.config.cc_update_hostname
 .. automodule:: cloudinit.config.cc_users_groups
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index da7da0c..3e8b791 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -1177,7 +1177,8 @@ class TestAzureDataSourcePreprovisioning(CiTestCase):
         url = 'http://{0}/metadata/reprovisiondata?api-version=2017-04-02'
         host = "169.254.169.254"
         full_url = url.format(host)
-        fake_resp.return_value = mock.MagicMock(status_code=200, text="ovf")
+        fake_resp.return_value = mock.MagicMock(status_code=200, text="ovf",
+                                                content="ovf")
         dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
         self.assertTrue(len(dsa._poll_imds()) > 0)
         self.assertEqual(fake_resp.call_args_list,
@@ -1185,13 +1186,8 @@ class TestAzureDataSourcePreprovisioning(CiTestCase):
                                     headers={'Metadata': 'true',
                                              'User-Agent':
                                              'Cloud-Init/%s' % vs()
-                                             }, method='GET', timeout=60.0,
-                                    url=full_url),
-                          mock.call(allow_redirects=True,
-                                    headers={'Metadata': 'true',
-                                             'User-Agent':
-                                             'Cloud-Init/%s' % vs()
-                                             }, method='GET', url=full_url)])
+                                             }, method='GET', timeout=1,
+                                    url=full_url)])
         self.assertEqual(m_dhcp.call_count, 1)
         m_net.assert_any_call(
             broadcast='192.168.2.255', interface='eth9', ip='192.168.2.9',
@@ -1217,7 +1213,8 @@ class TestAzureDataSourcePreprovisioning(CiTestCase):
         username = "myuser"
         odata = {'HostName': hostname, 'UserName': username}
         content = construct_valid_ovf_env(data=odata)
-        fake_resp.return_value = mock.MagicMock(status_code=200, text=content)
+        fake_resp.return_value = mock.MagicMock(status_code=200, text=content,
+                                                content=content)
         dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
         md, ud, cfg, d = dsa._reprovision()
         self.assertEqual(md['local-hostname'], hostname)
@@ -1227,12 +1224,7 @@ class TestAzureDataSourcePreprovisioning(CiTestCase):
                                     headers={'Metadata': 'true',
                                              'User-Agent':
                                              'Cloud-Init/%s' % vs()},
-                                    method='GET', timeout=60.0, url=full_url),
-                          mock.call(allow_redirects=True,
-                                    headers={'Metadata': 'true',
-                                             'User-Agent':
-                                             'Cloud-Init/%s' % vs()},
-                                    method='GET', url=full_url)])
+                                    method='GET', timeout=1, url=full_url)])
         self.assertEqual(m_dhcp.call_count, 1)
         m_net.assert_any_call(
             broadcast='192.168.2.255', interface='eth9', ip='192.168.2.9',
diff --git a/tests/unittests/test_datasource/test_ibmcloud.py b/tests/unittests/test_datasource/test_ibmcloud.py
new file mode 100644
index 0000000..621cfe4
--- /dev/null
+++ b/tests/unittests/test_datasource/test_ibmcloud.py
@@ -0,0 +1,262 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from cloudinit.sources import DataSourceIBMCloud as ibm
+from cloudinit.tests import helpers as test_helpers
+
+import base64
+import copy
+import json
+import mock
+from textwrap import dedent
+
+D_PATH = "cloudinit.sources.DataSourceIBMCloud."
+
+
+class TestIBMCloud(test_helpers.CiTestCase):
+    """Test the datasource."""
+    def setUp(self):
+        super(TestIBMCloud, self).setUp()
+        pass
+
+
[e-mail](D_PATH + "_is_xen", return_value=True)
[e-mail](D_PATH + "_is_ibm_provisioning")
[e-mail](D_PATH + "util.blkid")
+class TestGetIBMPlatform(test_helpers.CiTestCase):
+    """Test the get_ibm_platform helper."""
+
+    blkid_base = {
+        "/dev/xvda1": {
+            "DEVNAME": "/dev/xvda1", "LABEL": "cloudimg-bootfs",
+            "TYPE": "ext3"},
+        "/dev/xvda2": {
+            "DEVNAME": "/dev/xvda2", "LABEL": "cloudimg-rootfs",
+            "TYPE": "ext4"},
+    }
+
+    blkid_metadata_disk = {
+        "/dev/xvdh1": {
+            "DEVNAME": "/dev/xvdh1", "LABEL": "METADATA", "TYPE": "vfat",
+            "SEC_TYPE": "msdos", "UUID": "681B-8C5D",
+            "PARTUUID": "3d631e09-01"},
+    }
+
+    blkid_oscode_disk = {
+        "/dev/xvdh": {
+            "DEVNAME": "/dev/xvdh", "LABEL": "config-2", "TYPE": "vfat",
+            "SEC_TYPE": "msdos", "UUID": ibm.IBM_CONFIG_UUID}
+    }
+
+    def setUp(self):
+        self.blkid_metadata = copy.deepcopy(self.blkid_base)
+        self.blkid_metadata.update(copy.deepcopy(self.blkid_metadata_disk))
+
+        self.blkid_oscode = copy.deepcopy(self.blkid_base)
+        self.blkid_oscode.update(copy.deepcopy(self.blkid_oscode_disk))
+
+    def test_id_template_live_metadata(self, m_blkid, m_is_prov, _m_xen):
+        """identify TEMPLATE_LIVE_METADATA."""
+        m_blkid.return_value = self.blkid_metadata
+        m_is_prov.return_value = False
+        self.assertEqual(
+            (ibm.Platforms.TEMPLATE_LIVE_METADATA, "/dev/xvdh1"),
+            ibm.get_ibm_platform())
+
+    def test_id_template_prov_metadata(self, m_blkid, m_is_prov, _m_xen):
+        """identify TEMPLATE_PROVISIONING_METADATA."""
+        m_blkid.return_value = self.blkid_metadata
+        m_is_prov.return_value = True
+        self.assertEqual(
+            (ibm.Platforms.TEMPLATE_PROVISIONING_METADATA, "/dev/xvdh1"),
+            ibm.get_ibm_platform())
+
+    def test_id_template_prov_nodata(self, m_blkid, m_is_prov, _m_xen):
+        """identify TEMPLATE_PROVISIONING_NODATA."""
+        m_blkid.return_value = self.blkid_base
+        m_is_prov.return_value = True
+        self.assertEqual(
+            (ibm.Platforms.TEMPLATE_PROVISIONING_NODATA, None),
+            ibm.get_ibm_platform())
+
+    def test_id_os_code(self, m_blkid, m_is_prov, _m_xen):
+        """Identify OS_CODE."""
+        m_blkid.return_value = self.blkid_oscode
+        m_is_prov.return_value = False
+        self.assertEqual((ibm.Platforms.OS_CODE, "/dev/xvdh"),
+                         ibm.get_ibm_platform())
+
+    def test_id_os_code_must_match_uuid(self, m_blkid, m_is_prov, _m_xen):
+        """Test against false positive on openstack with non-ibm UUID."""
+        blkid = self.blkid_oscode
+        blkid["/dev/xvdh"]["UUID"] = "9999-9999"
+        m_blkid.return_value = blkid
+        m_is_prov.return_value = False
+        self.assertEqual((None, None), ibm.get_ibm_platform())
+
+
[e-mail](D_PATH + "_read_system_uuid", return_value=None)
[e-mail](D_PATH + "get_ibm_platform")
+class TestReadMD(test_helpers.CiTestCase):
+    """Test the read_datasource helper."""
+
+    template_md = {
+        "files": [],
+        "network_config": {"content_path": "/content/interfaces"},
+        "hostname": "ci-fond-ram",
+        "name": "ci-fond-ram",
+        "domain": "testing.ci.cloud-init.org",
+        "meta": {"dsmode": "net"},
+        "uuid": "8e636730-9f5d-c4a5-327c-d7123c46e82f",
+        "public_keys": {"1091307": "ssh-rsa AAAAB3NzaC1...Hw== ci-pubkey"},
+    }
+
+    oscode_md = {
+        "hostname": "ci-grand-gannet.testing.ci.cloud-init.org",
+        "name": "ci-grand-gannet",
+        "uuid": "2f266908-8e6c-4818-9b5c-42e9cc66a785",
+        "random_seed": "bm90LXJhbmRvbQo=",
+        "crypt_key": "ssh-rsa AAAAB3NzaC1yc2..n6z/",
+        "configuration_token": "eyJhbGciOi..M3ZA",
+        "public_keys": {"1091307": "ssh-rsa AAAAB3N..Hw== ci-pubkey"},
+    }
+
+    content_interfaces = dedent("""\
+        auto lo
+        iface lo inet loopback
+
+        auto eth0
+        allow-hotplug eth0
+        iface eth0 inet static
+        address 10.82.43.5
+        netmask 255.255.255.192
+        """)
+
+    userdata = b"#!/bin/sh\necho hi mom\n"
+    # meta.js file gets json encoded userdata as a list.
+    meta_js = '["#!/bin/sh\necho hi mom\n"]'
+    vendor_data = {
+        "cloud-init": "#!/bin/bash\necho 'root:$6$5ab01p1m1' | chpasswd -e"}
+
+    network_data = {
+        "links": [
+            {"id": "interface_29402281", "name": "eth0", "mtu": None,
+             "type": "phy", "ethernet_mac_address": "06:00:f1:bd:da:25"},
+            {"id": "interface_29402279", "name": "eth1", "mtu": None,
+             "type": "phy", "ethernet_mac_address": "06:98:5e:d0:7f:86"}
+        ],
+        "networks": [
+            {"id": "network_109887563", "link": "interface_29402281",
+             "type": "ipv4", "ip_address": "10.82.43.2",
+             "netmask": "255.255.255.192",
+             "routes": [
+                 {"network": "10.0.0.0", "netmask": "255.0.0.0",
+                  "gateway": "10.82.43.1"},
+                 {"network": "161.26.0.0", "netmask": "255.255.0.0",
+                  "gateway": "10.82.43.1"}]},
+            {"id": "network_109887551", "link": "interface_29402279",
+             "type": "ipv4", "ip_address": "108.168.194.252",
+             "netmask": "255.255.255.248",
+             "routes": [
+                 {"network": "0.0.0.0", "netmask": "0.0.0.0",
+                  "gateway": "108.168.194.249"}]}
+        ],
+        "services": [
+            {"type": "dns", "address": "10.0.80.11"},
+            {"type": "dns", "address": "10.0.80.12"}
+        ],
+    }
+
+    sysuuid = '7f79ebf5-d791-43c3-a723-854e8389d59f'
+
+    def _get_expected_metadata(self, os_md):
+        """return expected 'metadata' for data loaded from meta_data.json."""
+        os_md = copy.deepcopy(os_md)
+        renames = (
+            ('hostname', 'local-hostname'),
+            ('uuid', 'instance-id'),
+            ('public_keys', 'public-keys'))
+        ret = {}
+        for osname, mdname in renames:
+            if osname in os_md:
+                ret[mdname] = os_md[osname]
+        if 'random_seed' in os_md:
+            ret['random_seed'] = base64.b64decode(os_md['random_seed'])
+
+        return ret
+
+    def test_provisioning_md(self, m_platform, m_sysuuid):
+        """Provisioning env with a metadata disk should return None."""
+        m_platform.return_value = (
+            ibm.Platforms.TEMPLATE_PROVISIONING_METADATA, "/dev/xvdh")
+        self.assertIsNone(ibm.read_md())
+
+    def test_provisioning_no_metadata(self, m_platform, m_sysuuid):
+        """Provisioning env with no metadata disk should return None."""
+        m_platform.return_value = (
+            ibm.Platforms.TEMPLATE_PROVISIONING_NODATA, None)
+        self.assertIsNone(ibm.read_md())
+
+    def test_provisioning_not_ibm(self, m_platform, m_sysuuid):
+        """Provisioning env but not identified as IBM should return None."""
+        m_platform.return_value = (None, None)
+        self.assertIsNone(ibm.read_md())
+
+    def test_template_live(self, m_platform, m_sysuuid):
+        """Template live environment should be identified."""
+        tmpdir = self.tmp_dir()
+        m_platform.return_value = (
+            ibm.Platforms.TEMPLATE_LIVE_METADATA, tmpdir)
+        m_sysuuid.return_value = self.sysuuid
+
+        test_helpers.populate_dir(tmpdir, {
+            'openstack/latest/meta_data.json': json.dumps(self.template_md),
+            'openstack/latest/user_data': self.userdata,
+            'openstack/content/interfaces': self.content_interfaces,
+            'meta.js': self.meta_js})
+
+        ret = ibm.read_md()
+        self.assertEqual(ibm.Platforms.TEMPLATE_LIVE_METADATA,
+                         ret['platform'])
+        self.assertEqual(tmpdir, ret['source'])
+        self.assertEqual(self.userdata, ret['userdata'])
+        self.assertEqual(self._get_expected_metadata(self.template_md),
+                         ret['metadata'])
+        self.assertEqual(self.sysuuid, ret['system-uuid'])
+
+    def test_os_code_live(self, m_platform, m_sysuuid):
+        """Verify an os_code metadata path."""
+        tmpdir = self.tmp_dir()
+        m_platform.return_value = (ibm.Platforms.OS_CODE, tmpdir)
+        netdata = json.dumps(self.network_data)
+        test_helpers.populate_dir(tmpdir, {
+            'openstack/latest/meta_data.json': json.dumps(self.oscode_md),
+            'openstack/latest/user_data': self.userdata,
+            'openstack/latest/vendor_data.json': json.dumps(self.vendor_data),
+            'openstack/latest/network_data.json': netdata,
+        })
+
+        ret = ibm.read_md()
+        self.assertEqual(ibm.Platforms.OS_CODE, ret['platform'])
+        self.assertEqual(tmpdir, ret['source'])
+        self.assertEqual(self.userdata, ret['userdata'])
+        self.assertEqual(self._get_expected_metadata(self.oscode_md),
+                         ret['metadata'])
+
+    def test_os_code_live_no_userdata(self, m_platform, m_sysuuid):
+        """Verify os_code without user-data."""
+        tmpdir = self.tmp_dir()
+        m_platform.return_value = (ibm.Platforms.OS_CODE, tmpdir)
+        test_helpers.populate_dir(tmpdir, {
+            'openstack/latest/meta_data.json': json.dumps(self.oscode_md),
+            'openstack/latest/vendor_data.json': json.dumps(self.vendor_data),
+        })
+
+        ret = ibm.read_md()
+        self.assertEqual(ibm.Platforms.OS_CODE, ret['platform'])
+        self.assertEqual(tmpdir, ret['source'])
+        self.assertIsNone(ret['userdata'])
+        self.assertEqual(self._get_expected_metadata(self.oscode_md),
+                         ret['metadata'])
+
+
+# vi: ts=4 expandtab
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index 9c5628e..5364398 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -9,6 +9,8 @@ from cloudinit import util
 from cloudinit.tests.helpers import (
     CiTestCase, dir2dict, populate_dir)
 
+from cloudinit.sources import DataSourceIBMCloud as dsibm
+
 UNAME_MYSYS = ("Linux bart 4.4.0-62-generic #83-Ubuntu "
                "SMP Wed Jan 18 14:10:15 UTC 2017 x86_64 GNU/Linux")
 UNAME_PPC64EL = ("Linux diamond 4.4.0-83-generic #106-Ubuntu SMP "
@@ -37,8 +39,8 @@ BLKID_UEFI_UBUNTU = [
 
 POLICY_FOUND_ONLY = "search,found=all,maybe=none,notfound=disabled"
 POLICY_FOUND_OR_MAYBE = "search,found=all,maybe=all,notfound=disabled"
-DI_DEFAULT_POLICY = "search,found=all,maybe=all,notfound=enabled"
-DI_DEFAULT_POLICY_NO_DMI = "search,found=all,maybe=all,notfound=disabled"
+DI_DEFAULT_POLICY = "search,found=all,maybe=all,notfound=disabled"
+DI_DEFAULT_POLICY_NO_DMI = "search,found=all,maybe=all,notfound=enabled"
 DI_EC2_STRICT_ID_DEFAULT = "true"
 OVF_MATCH_STRING = 'http://schemas.dmtf.org/ovf/environment/1'
 
@@ -64,8 +66,12 @@ P_SYS_VENDOR = "sys/class/dmi/id/sys_vendor"
 P_SEED_DIR = "var/lib/cloud/seed"
 P_DSID_CFG = "etc/cloud/ds-identify.cfg"
 
+IBM_PROVISIONING_CHECK_PATH = "/root/provisioningConfiguration.cfg"
+IBM_CONFIG_UUID = "9796-932E"
+
 MOCK_VIRT_IS_KVM = {'name': 'detect_virt', 'RET': 'kvm', 'ret': 0}
 MOCK_VIRT_IS_VMWARE = {'name': 'detect_virt', 'RET': 'vmware', 'ret': 0}
+MOCK_VIRT_IS_XEN = {'name': 'detect_virt', 'RET': 'xen', 'ret': 0}
 MOCK_UNAME_IS_PPC64 = {'name': 'uname', 'out': UNAME_PPC64EL, 'ret': 0}
 
 
@@ -238,6 +244,57 @@ class TestDsIdentify(CiTestCase):
         self._test_ds_found('ConfigDriveUpper')
         return
 
+    def test_ibmcloud_template_userdata_in_provisioning(self):
+        """Template provisioned with user-data during provisioning stage.
+
+        Template provisioning with user-data has METADATA disk,
+        datasource should return not found."""
+        data = copy.deepcopy(VALID_CFG['IBMCloud-metadata'])
+        data['files'] = {IBM_PROVISIONING_CHECK_PATH: 'xxx'}
+        return self._check_via_dict(data, RC_NOT_FOUND)
+
+    def test_ibmcloud_template_userdata(self):
+        """Template provisioned with user-data first boot.
+
+        Template provisioning with user-data has METADATA disk.
+        datasource should return found."""
+        self._test_ds_found('IBMCloud-metadata')
+
+    def test_ibmcloud_template_no_userdata_in_provisioning(self):
+        """Template provisioned with no user-data during provisioning.
+
+        no disks attached.  Datasource should return not found."""
+        data = copy.deepcopy(VALID_CFG['IBMCloud-nodisks'])
+        data['files'] = {IBM_PROVISIONING_CHECK_PATH: 'xxx'}
+        return self._check_via_dict(data, RC_NOT_FOUND)
+
+    def test_ibmcloud_template_no_userdata(self):
+        """Template provisioned with no user-data first boot.
+
+        no disks attached.  Datasource should return found."""
+        self._check_via_dict(VALID_CFG['IBMCloud-nodisks'], RC_NOT_FOUND)
+
+    def test_ibmcloud_os_code(self):
+        """Launched by os code always has config-2 disk."""
+        self._test_ds_found('IBMCloud-config-2')
+
+    def test_ibmcloud_os_code_different_uuid(self):
+        """IBM cloud config-2 disks must be explicit match on UUID.
+
+        If the UUID is not 9796-932E then we actually expect ConfigDrive."""
+        data = copy.deepcopy(VALID_CFG['IBMCloud-config-2'])
+        offset = None
+        for m, d in enumerate(data['mocks']):
+            if d.get('name') == "blkid":
+                offset = m
+                break
+        if not offset:
+            raise ValueError("Expected to find 'blkid' mock, but did not.")
+        data['mocks'][offset]['out'] = d['out'].replace(dsibm.IBM_CONFIG_UUID,
+                                                        "DEAD-BEEF")
+        self._check_via_dict(
+            data, rc=RC_FOUND, dslist=['ConfigDrive', DS_NONE])
+
     def test_policy_disabled(self):
         """A Builtin policy of 'disabled' should return not found.
 
@@ -291,6 +348,10 @@ class TestDsIdentify(CiTestCase):
         """On Intel, openstack must be identified."""
         self._test_ds_found('OpenStack')
 
+    def test_openstack_open_telekom_cloud(self):
+        """Open Telecom identification."""
+        self._test_ds_found('OpenStack-OpenTelekom')
+
     def test_openstack_on_non_intel_is_maybe(self):
         """On non-Intel, openstack without dmi info is maybe.
 
@@ -447,7 +508,7 @@ VALID_CFG = {
     },
     'Ec2-xen': {
         'ds': 'Ec2',
-        'mocks': [{'name': 'detect_virt', 'RET': 'xen', 'ret': 0}],
+        'mocks': [MOCK_VIRT_IS_XEN],
         'files': {
             'sys/hypervisor/uuid': 'ec2c6e2f-5fac-4fc7-9c82-74127ec14bbb\n'
         },
@@ -502,6 +563,12 @@ VALID_CFG = {
         'policy_dmi': POLICY_FOUND_ONLY,
         'policy_no_dmi': POLICY_FOUND_ONLY,
     },
+    'OpenStack-OpenTelekom': {
+        # OTC gen1 (Xen) hosts use OpenStack datasource, LP: #1756471
+        'ds': 'OpenStack',
+        'files': {P_CHASSIS_ASSET_TAG: 'OpenTelekomCloud\n'},
+        'mocks': [MOCK_VIRT_IS_XEN],
+    },
     'OVF-seed': {
         'ds': 'OVF',
         'files': {
@@ -568,6 +635,48 @@ VALID_CFG = {
         'ds': 'Hetzner',
         'files': {P_SYS_VENDOR: 'Hetzner\n'},
     },
+    'IBMCloud-metadata': {
+        'ds': 'IBMCloud',
+        'mocks': [
+            MOCK_VIRT_IS_XEN,
+            {'name': 'blkid', 'ret': 0,
+             'out': blkid_out(
+                 [{'DEVNAME': 'xvda1', 'TYPE': 'vfat', 'PARTUUID': uuid4()},
+                  {'DEVNAME': 'xvda2', 'TYPE': 'ext4',
+                   'LABEL': 'cloudimg-rootfs', 'PARTUUID': uuid4()},
+                  {'DEVNAME': 'xvdb', 'TYPE': 'vfat', 'LABEL': 'METADATA'}]),
+             },
+        ],
+    },
+    'IBMCloud-config-2': {
+        'ds': 'IBMCloud',
+        'mocks': [
+            MOCK_VIRT_IS_XEN,
+            {'name': 'blkid', 'ret': 0,
+             'out': blkid_out(
+                 [{'DEVNAME': 'xvda1', 'TYPE': 'ext3', 'PARTUUID': uuid4(),
+                   'UUID': uuid4(), 'LABEL': 'cloudimg-bootfs'},
+                  {'DEVNAME': 'xvdb', 'TYPE': 'vfat', 'LABEL': 'config-2',
+                   'UUID': dsibm.IBM_CONFIG_UUID},
+                  {'DEVNAME': 'xvda2', 'TYPE': 'ext4',
+                   'LABEL': 'cloudimg-rootfs', 'PARTUUID': uuid4(),
+                   'UUID': uuid4()},
+                  ]),
+             },
+        ],
+    },
+    'IBMCloud-nodisks': {
+        'ds': 'IBMCloud',
+        'mocks': [
+            MOCK_VIRT_IS_XEN,
+            {'name': 'blkid', 'ret': 0,
+             'out': blkid_out(
+                 [{'DEVNAME': 'xvda1', 'TYPE': 'vfat', 'PARTUUID': uuid4()},
+                  {'DEVNAME': 'xvda2', 'TYPE': 'ext4',
+                   'LABEL': 'cloudimg-rootfs', 'PARTUUID': uuid4()}]),
+             },
+        ],
+    },
 }
 
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py
index 9b50ee7..ac41f12 100644
--- a/tests/unittests/test_handler/test_schema.py
+++ b/tests/unittests/test_handler/test_schema.py
@@ -27,6 +27,7 @@ class GetSchemaTest(CiTestCase):
                 'cc_resizefs',
                 'cc_runcmd',
                 'cc_snap',
+                'cc_ubuntu_advantage',
                 'cc_zypper_add_repo'
             ],
             [subschema['id'] for subschema in schema['allOf']])
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
index 84a0eab..c12a487 100644
--- a/tests/unittests/test_net.py
+++ b/tests/unittests/test_net.py
@@ -393,12 +393,6 @@ NETWORK_CONFIGS = {
                     eth1:
                         match:
                             macaddress: cf:d6:af:48:e8:80
-                        nameservers:
-                            addresses:
-                            - 1.2.3.4
-                            - 5.6.7.8
-                            search:
-                            - wark.maas
                         set-name: eth1
                     eth99:
                         addresses:
@@ -410,12 +404,9 @@ NETWORK_CONFIGS = {
                             addresses:
                             - 8.8.8.8
                             - 8.8.4.4
-                            - 1.2.3.4
-                            - 5.6.7.8
                             search:
                             - barley.maas
                             - sach.maas
-                            - wark.maas
                         routes:
                         -   to: 0.0.0.0/0
                             via: 65.61.151.37
@@ -654,81 +645,27 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true
                     eth0:
                         match:
                             macaddress: c0:d6:9f:2c:e8:80
-                        nameservers:
-                            addresses:
-                            - 8.8.8.8
-                            - 4.4.4.4
-                            - 8.8.4.4
-                            search:
-                            - barley.maas
-                            - wark.maas
-                            - foobar.maas
                         set-name: eth0
                     eth1:
                         match:
                             macaddress: aa:d6:9f:2c:e8:80
-                        nameservers:
-                            addresses:
-                            - 8.8.8.8
-                            - 4.4.4.4
-                            - 8.8.4.4
-                            search:
-                            - barley.maas
-                            - wark.maas
-                            - foobar.maas
                         set-name: eth1
                     eth2:
                         match:
                             macaddress: c0:bb:9f:2c:e8:80
-                        nameservers:
-                            addresses:
-                            - 8.8.8.8
-                            - 4.4.4.4
-                            - 8.8.4.4
-                            search:
-                            - barley.maas
-                            - wark.maas
-                            - foobar.maas
                         set-name: eth2
                     eth3:
                         match:
                             macaddress: 66:bb:9f:2c:e8:80
-                        nameservers:
-                            addresses:
-                            - 8.8.8.8
-                            - 4.4.4.4
-                            - 8.8.4.4
-                            search:
-                            - barley.maas
-                            - wark.maas
-                            - foobar.maas
                         set-name: eth3
                     eth4:
                         match:
                             macaddress: 98:bb:9f:2c:e8:80
-                        nameservers:
-                            addresses:
-                            - 8.8.8.8
-                            - 4.4.4.4
-                            - 8.8.4.4
-                            search:
-                            - barley.maas
-                            - wark.maas
-                            - foobar.maas
                         set-name: eth4
                     eth5:
                         dhcp4: true
                         match:
                             macaddress: 98:bb:9f:2c:e8:8a
-                        nameservers:
-                            addresses:
-                            - 8.8.8.8
-                            - 4.4.4.4
-                            - 8.8.4.4
-                            search:
-                            - barley.maas
-                            - wark.maas
-                            - foobar.maas
                         set-name: eth5
                 bonds:
                     bond0:
@@ -748,6 +685,15 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true
                         interfaces:
                         - eth3
                         - eth4
+                        nameservers:
+                            addresses:
+                            - 8.8.8.8
+                            - 4.4.4.4
+                            - 8.8.4.4
+                            search:
+                            - barley.maas
+                            - wark.maas
+                            - foobar.maas
                         parameters:
                             ageing-time: 250
                             forward-delay: 1
@@ -2334,6 +2280,9 @@ class TestNetplanRoundTrip(CiTestCase):
     def testsimple_render_all(self):
         entry = NETWORK_CONFIGS['all']
         files = self._render_and_read(network_config=yaml.load(entry['yaml']))
+        print(entry['expected_netplan'])
+        print('-- expected ^ | v rendered --')
+        print(files['/etc/netplan/50-cloud-init.yaml'])
         self.assertEqual(
             entry['expected_netplan'].splitlines(),
             files['/etc/netplan/50-cloud-init.yaml'].splitlines())
diff --git a/tools/ds-identify b/tools/ds-identify
index e3f93c9..9a2db5c 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -92,6 +92,7 @@ DI_DMI_SYS_VENDOR=""
 DI_DMI_PRODUCT_SERIAL=""
 DI_DMI_PRODUCT_UUID=""
 DI_FS_LABELS=""
+DI_FS_UUIDS=""
 DI_ISO9660_DEVS=""
 DI_KERNEL_CMDLINE=""
 DI_VIRT=""
@@ -114,7 +115,7 @@ DI_DSNAME=""
 # be searched if there is no setting found in config.
 DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \
 CloudSigma CloudStack DigitalOcean AliYun Ec2 GCE OpenNebula OpenStack \
-OVF SmartOS Scaleway Hetzner"
+OVF SmartOS Scaleway Hetzner IBMCloud"
 DI_DSLIST=""
 DI_MODE=""
 DI_ON_FOUND=""
@@ -123,6 +124,8 @@ DI_ON_NOTFOUND=""
 
 DI_EC2_STRICT_ID_DEFAULT="true"
 
+_IS_IBM_CLOUD=""
+
 error() {
     set -- "ERROR:" "$@";
     debug 0 "$@"
@@ -196,7 +199,7 @@ read_fs_info() {
         return
     fi
     local oifs="$IFS" line="" delim=","
-    local ret=0 out="" labels="" dev="" label="" ftype="" isodevs=""
+    local ret=0 out="" labels="" dev="" label="" ftype="" isodevs="" uuids=""
     out=$(blkid -c /dev/null -o export) || {
         ret=$?
         error "failed running [$ret]: blkid -c /dev/null -o export"
@@ -219,12 +222,14 @@ read_fs_info() {
             LABEL=*) label="${line#LABEL=}";
                      labels="${labels}${line#LABEL=}${delim}";;
             TYPE=*) ftype=${line#TYPE=};;
+            UUID=*) uuids="${uuids}${line#UUID=}$delim";;
         esac
     done
     [ -n "$dev" -a "$ftype" = "iso9660" ] &&
         isodevs="${isodevs} ${dev}=$label"
 
     DI_FS_LABELS="${labels%${delim}}"
+    DI_FS_UUIDS="${uuids%${delim}}"
     DI_ISO9660_DEVS="${isodevs# }"
 }
 
@@ -437,14 +442,25 @@ dmi_sys_vendor_is() {
     [ "${DI_DMI_SYS_VENDOR}" = "$1" ]
 }
 
-has_fs_with_label() {
-    local label="$1"
-    case ",${DI_FS_LABELS}," in
-        *,$label,*) return 0;;
+has_fs_with_uuid() {
+    case ",${DI_FS_UUIDS}," in
+        *,$1,*) return 0;;
     esac
     return 1
 }
 
+has_fs_with_label() {
+    # has_fs_with_label(label1[ ,label2 ..])
+    # return 0 if a there is a filesystem that matches any of the labels.
+    local label=""
+    for label in "$@"; do
+        case ",${DI_FS_LABELS}," in
+            *,$label,*) return 0;;
+        esac
+    done
+    return 1
+}
+
 nocase_equal() {
     # nocase_equal(a, b)
     # return 0 if case insenstive comparision a.lower() == b.lower()
@@ -583,6 +599,8 @@ dscheck_NoCloud() {
     case " ${DI_DMI_PRODUCT_SERIAL} " in
         *\ ds=nocloud*) return ${DS_FOUND};;
     esac
+
+    is_ibm_cloud && return ${DS_NOT_FOUND}
     for d in nocloud nocloud-net; do
         check_seed_dir "$d" meta-data user-data && return ${DS_FOUND}
         check_writable_seed_dir "$d" meta-data user-data && return ${DS_FOUND}
@@ -594,9 +612,8 @@ dscheck_NoCloud() {
 }
 
 check_configdrive_v2() {
-    if has_fs_with_label "config-2"; then
-        return ${DS_FOUND}
-    elif has_fs_with_label "CONFIG-2"; then
+    is_ibm_cloud && return ${DS_NOT_FOUND}
+    if has_fs_with_label CONFIG-2 config-2; then
         return ${DS_FOUND}
     fi
     # look in /config-drive <vlc>/seed/config_drive for a directory
@@ -894,6 +911,10 @@ dscheck_OpenStack() {
         return ${DS_FOUND}
     fi
 
+    if dmi_chassis_asset_tag_matches "OpenTelekomCloud"; then
+        return ${DS_FOUND}
+    fi
+
     # LP: #1715241 : arch other than intel are not identified properly.
     case "$DI_UNAME_MACHINE" in
         i?86|x86_64) :;;
@@ -984,6 +1005,36 @@ dscheck_Hetzner() {
     return ${DS_NOT_FOUND}
 }
 
+is_ibm_provisioning() {
+    [ -f "${PATH_ROOT}/root/provisioningConfiguration.cfg" ]
+}
+
+is_ibm_cloud() {
+    cached "${_IS_IBM_CLOUD}" && return ${_IS_IBM_CLOUD}
+    local ret=1
+    if [ "$DI_VIRT" = "xen" ]; then
+        if is_ibm_provisioning; then
+            ret=0
+        elif has_fs_with_label METADATA metadata; then
+            ret=0
+        elif has_fs_with_uuid 9796-932E &&
+            has_fs_with_label CONFIG-2 config-2; then
+            ret=0
+        fi
+    fi
+    _IS_IBM_CLOUD=$ret
+    return $ret
+}
+
+dscheck_IBMCloud() {
+    if is_ibm_provisioning; then
+        debug 1 "cloud-init disabled during provisioning on IBMCloud"
+        return ${DS_NOT_FOUND}
+    fi
+    is_ibm_cloud && return ${DS_FOUND}
+    return ${DS_NOT_FOUND}
+}
+
 collect_info() {
     read_virt
     read_pid1_product_name
diff --git a/tools/pipremove b/tools/pipremove
new file mode 100755
index 0000000..f8f4ff1
--- /dev/null
+++ b/tools/pipremove
@@ -0,0 +1,14 @@
+#!/usr/bin/python3
+import subprocess
+import sys
+
+for pkg in sys.argv[1:]:
+    try:
+        exec('import %s' % pkg)  # pylint: disable=W0122
+    except ImportError:
+        continue
+    sys.stderr.write("%s removing package %s\n" % (sys.argv[0], pkg))
+    ret = subprocess.Popen(['pip', 'uninstall', '--yes', pkg]).wait()
+    if ret != 0:
+        sys.stderr.write("Failed to uninstall %s (%d)\n" % (pkg, ret))
+        sys.exit(ret)
diff --git a/tox.ini b/tox.ini
index 1f990af..818ade3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -60,6 +60,9 @@ deps = sphinx
 commands = {envpython} -m sphinx {posargs:doc/rtd doc/rtd_html}
 
 [testenv:xenial]
+commands =
+  python ./tools/pipremove jsonschema
+  python -m nose {posargs:tests/unittests cloudinit}
 basepython = python3
 deps =
     # requirements