Merge ~chad.smith/cloud-init:ubuntu/devel into cloud-init:ubuntu/devel

[Chad Smith]

Sync tip of master for publish in Bionic.

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:a6262577f56d32fb6005a55f9022309c5dc7dce5
https://jenkins.ubuntu.com/server/job/cloud-init-ci/859/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Unit & Style Tests
    SUCCESS: Ubuntu LTS: Build
    SUCCESS: Ubuntu LTS: Integration
    SUCCESS: MAAS Compatability Testing
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/859/rebuild

review: Approve (continuous-integration)

diff --git a/.pylintrc b/.pylintrc
index 05a086d..0bdfa59 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -46,7 +46,17 @@ reports=no
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis. It
 # supports qualified module names, as well as Unix pattern matching.
-ignored-modules=six.moves,pkg_resources,httplib,http.client,paramiko,simplestreams
+ignored-modules=
+ http.client,
+ httplib,
+ pkg_resources,
+ six.moves,
+ # cloud_tests requirements.
+ boto3,
+ botocore,
+ paramiko,
+ pylxd,
+ simplestreams
 
 # List of class names for which member attributes should not be checked (useful
 # for classes with dynamically set attributes). This supports the use of
diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py
index ba61678..6d12c43 100644
--- a/cloudinit/cloud.py
+++ b/cloudinit/cloud.py
@@ -78,8 +78,9 @@ class Cloud(object):
     def get_locale(self):
         return self.datasource.get_locale()
 
-    def get_hostname(self, fqdn=False):
-        return self.datasource.get_hostname(fqdn=fqdn)
+    def get_hostname(self, fqdn=False, metadata_only=False):
+        return self.datasource.get_hostname(
+            fqdn=fqdn, metadata_only=metadata_only)
 
     def device_name_to_device(self, name):
         return self.datasource.device_name_to_device(name)
diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py
index d2f1b77..3f2dbb9 100644
--- a/cloudinit/cmd/main.py
+++ b/cloudinit/cmd/main.py
@@ -40,6 +40,7 @@ from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE,
 
 from cloudinit import atomic_helper
 
+from cloudinit.config import cc_set_hostname
 from cloudinit.dhclient_hook import LogDhclient
 
 
@@ -215,12 +216,10 @@ def main_init(name, args):
     if args.local:
         deps = [sources.DEP_FILESYSTEM]
 
-    early_logs = []
-    early_logs.append(
-        attempt_cmdline_url(
-            path=os.path.join("%s.d" % CLOUD_CONFIG,
-                              "91_kernel_cmdline_url.cfg"),
-            network=not args.local))
+    early_logs = [attempt_cmdline_url(
+        path=os.path.join("%s.d" % CLOUD_CONFIG,
+                          "91_kernel_cmdline_url.cfg"),
+        network=not args.local)]
 
     # Cloud-init 'init' stage is broken up into the following sub-stages
     # 1. Ensure that the init object fetches its config without errors
@@ -354,6 +353,11 @@ def main_init(name, args):
     LOG.debug("[%s] %s will now be targeting instance id: %s. new=%s",
               mode, name, iid, init.is_new_instance())
 
+    if mode == sources.DSMODE_LOCAL:
+        # Before network comes up, set any configured hostname to allow
+        # dhcp clients to advertize this hostname to any DDNS services
+        # LP: #1746455.
+        _maybe_set_hostname(init, stage='local', retry_stage='network')
     init.apply_network_config(bring_up=bool(mode != sources.DSMODE_LOCAL))
 
     if mode == sources.DSMODE_LOCAL:
@@ -370,6 +374,7 @@ def main_init(name, args):
     init.setup_datasource()
     # update fully realizes user-data (pulling in #include if necessary)
     init.update()
+    _maybe_set_hostname(init, stage='init-net', retry_stage='modules:config')
     # Stage 7
     try:
         # Attempt to consume the data per instance.
@@ -683,6 +688,24 @@ def status_wrapper(name, args, data_d=None, link_d=None):
     return len(v1[mode]['errors'])
 
 
+def _maybe_set_hostname(init, stage, retry_stage):
+    """Call set-hostname if metadata, vendordata or userdata provides it.
+
+    @param stage: String representing current stage in which we are running.
+    @param retry_stage: String represented logs upon error setting hostname.
+    """
+    cloud = init.cloudify()
+    (hostname, _fqdn) = util.get_hostname_fqdn(
+        init.cfg, cloud, metadata_only=True)
+    if hostname:  # meta-data or user-data hostname content
+        try:
+            cc_set_hostname.handle('set-hostname', init.cfg, cloud, LOG, None)
+        except cc_set_hostname.SetHostnameError as e:
+            LOG.debug(
+                'Failed setting hostname in %s stage. Will'
+                ' retry in %s stage. Error: %s.', stage, retry_stage, str(e))
+
+
 def main_features(name, args):
     sys.stdout.write('\n'.join(sorted(version.FEATURES)) + '\n')
 
diff --git a/cloudinit/cmd/tests/test_main.py b/cloudinit/cmd/tests/test_main.py
new file mode 100644
index 0000000..dbe421c
--- /dev/null
+++ b/cloudinit/cmd/tests/test_main.py
@@ -0,0 +1,161 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from collections import namedtuple
+import copy
+import os
+from six import StringIO
+
+from cloudinit.cmd import main
+from cloudinit.util import (
+    ensure_dir, load_file, write_file, yaml_dumps)
+from cloudinit.tests.helpers import (
+    FilesystemMockingTestCase, wrap_and_call)
+
+mypaths = namedtuple('MyPaths', 'run_dir')
+myargs = namedtuple('MyArgs', 'debug files force local reporter subcommand')
+
+
+class TestMain(FilesystemMockingTestCase):
+
+    with_logs = True
+
+    def setUp(self):
+        super(TestMain, self).setUp()
+        self.new_root = self.tmp_dir()
+        self.cloud_dir = self.tmp_path('var/lib/cloud/', dir=self.new_root)
+        os.makedirs(self.cloud_dir)
+        self.replicateTestRoot('simple_ubuntu', self.new_root)
+        self.cfg = {
+            'datasource_list': ['None'],
+            'runcmd': ['ls /etc'],  # test ALL_DISTROS
+            'system_info': {'paths': {'cloud_dir': self.cloud_dir,
+                                      'run_dir': self.new_root}},
+            'write_files': [
+                {
+                    'path': '/etc/blah.ini',
+                    'content': 'blah',
+                    'permissions': 0o755,
+                },
+            ],
+            'cloud_init_modules': ['write-files', 'runcmd'],
+        }
+        cloud_cfg = yaml_dumps(self.cfg)
+        ensure_dir(os.path.join(self.new_root, 'etc', 'cloud'))
+        self.cloud_cfg_file = os.path.join(
+            self.new_root, 'etc', 'cloud', 'cloud.cfg')
+        write_file(self.cloud_cfg_file, cloud_cfg)
+        self.patchOS(self.new_root)
+        self.patchUtils(self.new_root)
+        self.stderr = StringIO()
+        self.patchStdoutAndStderr(stderr=self.stderr)
+
+    def test_main_init_run_net_stops_on_file_no_net(self):
+        """When no-net file is present, main_init does not process modules."""
+        stop_file = os.path.join(self.cloud_dir, 'data', 'no-net')  # stop file
+        write_file(stop_file, '')
+        cmdargs = myargs(
+            debug=False, files=None, force=False, local=False, reporter=None,
+            subcommand='init')
+        (item1, item2) = wrap_and_call(
+            'cloudinit.cmd.main',
+            {'util.close_stdin': True,
+             'netinfo.debug_info': 'my net debug info',
+             'util.fixup_output': ('outfmt', 'errfmt')},
+            main.main_init, 'init', cmdargs)
+        # We should not run write_files module
+        self.assertFalse(
+            os.path.exists(os.path.join(self.new_root, 'etc/blah.ini')),
+            'Unexpected run of write_files module produced blah.ini')
+        self.assertEqual([], item2)
+        # Instancify is called
+        instance_id_path = 'var/lib/cloud/data/instance-id'
+        self.assertFalse(
+            os.path.exists(os.path.join(self.new_root, instance_id_path)),
+            'Unexpected call to datasource.instancify produced instance-id')
+        expected_logs = [
+            "Exiting. stop file ['{stop_file}'] existed\n".format(
+                stop_file=stop_file),
+            'my net debug info'  # netinfo.debug_info
+        ]
+        for log in expected_logs:
+            self.assertIn(log, self.stderr.getvalue())
+
+    def test_main_init_run_net_runs_modules(self):
+        """Modules like write_files are run in 'net' mode."""
+        cmdargs = myargs(
+            debug=False, files=None, force=False, local=False, reporter=None,
+            subcommand='init')
+        (item1, item2) = wrap_and_call(
+            'cloudinit.cmd.main',
+            {'util.close_stdin': True,
+             'netinfo.debug_info': 'my net debug info',
+             'util.fixup_output': ('outfmt', 'errfmt')},
+            main.main_init, 'init', cmdargs)
+        self.assertEqual([], item2)
+        # Instancify is called
+        instance_id_path = 'var/lib/cloud/data/instance-id'
+        self.assertEqual(
+            'iid-datasource-none\n',
+            os.path.join(load_file(
+                os.path.join(self.new_root, instance_id_path))))
+        # modules are run (including write_files)
+        self.assertEqual(
+            'blah', load_file(os.path.join(self.new_root, 'etc/blah.ini')))
+        expected_logs = [
+            'network config is disabled by fallback',  # apply_network_config
+            'my net debug info',  # netinfo.debug_info
+            'no previous run detected'
+        ]
+        for log in expected_logs:
+            self.assertIn(log, self.stderr.getvalue())
+
+    def test_main_init_run_net_calls_set_hostname_when_metadata_present(self):
+        """When local-hostname metadata is present, call cc_set_hostname."""
+        self.cfg['datasource'] = {
+            'None': {'metadata': {'local-hostname': 'md-hostname'}}}
+        cloud_cfg = yaml_dumps(self.cfg)
+        write_file(self.cloud_cfg_file, cloud_cfg)
+        cmdargs = myargs(
+            debug=False, files=None, force=False, local=False, reporter=None,
+            subcommand='init')
+
+        def set_hostname(name, cfg, cloud, log, args):
+            self.assertEqual('set-hostname', name)
+            updated_cfg = copy.deepcopy(self.cfg)
+            updated_cfg.update(
+                {'def_log_file': '/var/log/cloud-init.log',
+                 'log_cfgs': [],
+                 'syslog_fix_perms': ['syslog:adm', 'root:adm', 'root:wheel'],
+                 'vendor_data': {'enabled': True, 'prefix': []}})
+            updated_cfg.pop('system_info')
+
+            self.assertEqual(updated_cfg, cfg)
+            self.assertEqual(main.LOG, log)
+            self.assertIsNone(args)
+
+        (item1, item2) = wrap_and_call(
+            'cloudinit.cmd.main',
+            {'util.close_stdin': True,
+             'netinfo.debug_info': 'my net debug info',
+             'cc_set_hostname.handle': {'side_effect': set_hostname},
+             'util.fixup_output': ('outfmt', 'errfmt')},
+            main.main_init, 'init', cmdargs)
+        self.assertEqual([], item2)
+        # Instancify is called
+        instance_id_path = 'var/lib/cloud/data/instance-id'
+        self.assertEqual(
+            'iid-datasource-none\n',
+            os.path.join(load_file(
+                os.path.join(self.new_root, instance_id_path))))
+        # modules are run (including write_files)
+        self.assertEqual(
+            'blah', load_file(os.path.join(self.new_root, 'etc/blah.ini')))
+        expected_logs = [
+            'network config is disabled by fallback',  # apply_network_config
+            'my net debug info',  # netinfo.debug_info
+            'no previous run detected'
+        ]
+        for log in expected_logs:
+            self.assertIn(log, self.stderr.getvalue())
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py
index efedd4a..aff4010 100644
--- a/cloudinit/config/cc_keys_to_console.py
+++ b/cloudinit/config/cc_keys_to_console.py
@@ -63,9 +63,7 @@ def handle(name, cfg, cloud, log, _args):
                                              ["ssh-dss"])
 
     try:
-        cmd = [helper_path]
-        cmd.append(','.join(fp_blacklist))
-        cmd.append(','.join(key_blacklist))
+        cmd = [helper_path, ','.join(fp_blacklist), ','.join(key_blacklist)]
         (stdout, _stderr) = util.subp(cmd)
         util.multi_log("%s\n" % (stdout.strip()),
                        stderr=False, console=True)
diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py
index 449872f..539cbd5 100644
--- a/cloudinit/config/cc_runcmd.py
+++ b/cloudinit/config/cc_runcmd.py
@@ -39,8 +39,10 @@ schema = {
         using ``sh``.
 
         .. note::
-        all commands must be proper yaml, so you have to quote any characters
-        yaml would eat (':' can be problematic)"""),
+
+          all commands must be proper yaml, so you have to quote any characters
+          yaml would eat (':' can be problematic)
+    """),
     'distros': distros,
     'examples': [dedent("""\
         runcmd:
diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py
index 5112a34..d6a21d7 100644
--- a/cloudinit/config/cc_salt_minion.py
+++ b/cloudinit/config/cc_salt_minion.py
@@ -12,7 +12,9 @@ key is present in the config parts, then salt minion will be installed and
 started. Configuration for salt minion can be specified in the ``conf`` key
 under ``salt_minion``. Any conf values present there will be assigned in
 ``/etc/salt/minion``. The public and private keys to use for salt minion can be
-specified with ``public_key`` and ``private_key`` respectively.
+specified with ``public_key`` and ``private_key`` respectively. Optionally if
+you have a custom package name, service name or config directory you can
+specify them with ``pkg_name``, ``service_name`` and ``config_dir``.
 
 **Internal name:** ``cc_salt_minion``
 
@@ -23,6 +25,9 @@ specified with ``public_key`` and ``private_key`` respectively.
 **Config keys**::
 
     salt_minion:
+        pkg_name: 'salt-minion'
+        service_name: 'salt-minion'
+        config_dir: '/etc/salt'
         conf:
             master: salt.example.com
         grains:
@@ -42,7 +47,34 @@ import os
 
 from cloudinit import util
 
-# Note: see http://saltstack.org/topics/installation/
+# Note: see https://docs.saltstack.com/en/latest/topics/installation/
+# Note: see https://docs.saltstack.com/en/latest/ref/configuration/
+
+
+class SaltConstants(object):
+    """
+    defines default distribution specific salt variables
+    """
+    def __init__(self, cfg):
+
+        # constants tailored for FreeBSD
+        if util.is_FreeBSD():
+            self.pkg_name = 'py27-salt'
+            self.srv_name = 'salt_minion'
+            self.conf_dir = '/usr/local/etc/salt'
+        # constants for any other OS
+        else:
+            self.pkg_name = 'salt-minion'
+            self.srv_name = 'salt-minion'
+            self.conf_dir = '/etc/salt'
+
+        # if there are constants given in cloud config use those
+        self.pkg_name = util.get_cfg_option_str(cfg, 'pkg_name',
+                                                self.pkg_name)
+        self.conf_dir = util.get_cfg_option_str(cfg, 'config_dir',
+                                                self.conf_dir)
+        self.srv_name = util.get_cfg_option_str(cfg, 'service_name',
+                                                self.srv_name)
 
 
 def handle(name, cfg, cloud, log, _args):
@@ -52,45 +84,49 @@ def handle(name, cfg, cloud, log, _args):
                    " no 'salt_minion' key in configuration"), name)
         return
 
-    salt_cfg = cfg['salt_minion']
+    s_cfg = cfg['salt_minion']
+    const = SaltConstants(cfg=s_cfg)
 
     # Start by installing the salt package ...
-    cloud.distro.install_packages(('salt-minion',))
+    cloud.distro.install_packages(const.pkg_name)
 
     # Ensure we can configure files at the right dir
-    config_dir = salt_cfg.get("config_dir", '/etc/salt')
-    util.ensure_dir(config_dir)
+    util.ensure_dir(const.conf_dir)
 
     # ... and then update the salt configuration
-    if 'conf' in salt_cfg:
-        # Add all sections from the conf object to /etc/salt/minion
-        minion_config = os.path.join(config_dir, 'minion')
-        minion_data = util.yaml_dumps(salt_cfg.get('conf'))
+    if 'conf' in s_cfg:
+        # Add all sections from the conf object to minion config file
+        minion_config = os.path.join(const.conf_dir, 'minion')
+        minion_data = util.yaml_dumps(s_cfg.get('conf'))
         util.write_file(minion_config, minion_data)
 
-    if 'grains' in salt_cfg:
+    if 'grains' in s_cfg:
         # add grains to /etc/salt/grains
-        grains_config = os.path.join(config_dir, 'grains')
-        grains_data = util.yaml_dumps(salt_cfg.get('grains'))
+        grains_config = os.path.join(const.conf_dir, 'grains')
+        grains_data = util.yaml_dumps(s_cfg.get('grains'))
         util.write_file(grains_config, grains_data)
 
     # ... copy the key pair if specified
-    if 'public_key' in salt_cfg and 'private_key' in salt_cfg:
-        if os.path.isdir("/etc/salt/pki/minion"):
-            pki_dir_default = "/etc/salt/pki/minion"
-        else:
-            pki_dir_default = "/etc/salt/pki"
+    if 'public_key' in s_cfg and 'private_key' in s_cfg:
+        pki_dir_default = os.path.join(const.conf_dir, "pki/minion")
+        if not os.path.isdir(pki_dir_default):
+            pki_dir_default = os.path.join(const.conf_dir, "pki")
 
-        pki_dir = salt_cfg.get('pki_dir', pki_dir_default)
+        pki_dir = s_cfg.get('pki_dir', pki_dir_default)
         with util.umask(0o77):
             util.ensure_dir(pki_dir)
             pub_name = os.path.join(pki_dir, 'minion.pub')
             pem_name = os.path.join(pki_dir, 'minion.pem')
-            util.write_file(pub_name, salt_cfg['public_key'])
-            util.write_file(pem_name, salt_cfg['private_key'])
+            util.write_file(pub_name, s_cfg['public_key'])
+            util.write_file(pem_name, s_cfg['private_key'])
+
+    # we need to have the salt minion service enabled in rc in order to be
+    # able to start the service. this does only apply on FreeBSD servers.
+    if cloud.distro.osfamily == 'freebsd':
+        cloud.distro.updatercconf('salt_minion_enable', 'YES')
 
-    # restart salt-minion.  'service' will start even if not started.  if it
+    # restart salt-minion. 'service' will start even if not started. if it
     # was started, it needs to be restarted for config change.
-    util.subp(['service', 'salt-minion', 'restart'], capture=False)
+    util.subp(['service', const.srv_name, 'restart'], capture=False)
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py
index aa3dfe5..3d2b2da 100644
--- a/cloudinit/config/cc_set_hostname.py
+++ b/cloudinit/config/cc_set_hostname.py
@@ -32,22 +32,51 @@ will be used.
     hostname: <fqdn/hostname>
 """
 
+import os
+
+
+from cloudinit.atomic_helper import write_json
 from cloudinit import util
 
 
+class SetHostnameError(Exception):
+    """Raised when the distro runs into an exception when setting hostname.
+
+    This may happen if we attempt to set the hostname early in cloud-init's
+    init-local timeframe as certain services may not be running yet.
+    """
+    pass
+
+
 def handle(name, cfg, cloud, log, _args):
     if util.get_cfg_option_bool(cfg, "preserve_hostname", False):
         log.debug(("Configuration option 'preserve_hostname' is set,"
                    " not setting the hostname in module %s"), name)
         return
-
     (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud)
+    # Check for previous successful invocation of set-hostname
+
+    # set-hostname artifact file accounts for both hostname and fqdn
+    # deltas. As such, it's format is different than cc_update_hostname's
+    # previous-hostname file which only contains the base hostname.
+    # TODO consolidate previous-hostname and set-hostname artifact files and
+    # distro._read_hostname implementation so we only validate  one artifact.
+    prev_fn = os.path.join(cloud.get_cpath('data'), "set-hostname")
+    prev_hostname = {}
+    if os.path.exists(prev_fn):
+        prev_hostname = util.load_json(util.load_file(prev_fn))
+    hostname_changed = (hostname != prev_hostname.get('hostname') or
+                        fqdn != prev_hostname.get('fqdn'))
+    if not hostname_changed:
+        log.debug('No hostname changes. Skipping set-hostname')
+        return
+    log.debug("Setting the hostname to %s (%s)", fqdn, hostname)
     try:
-        log.debug("Setting the hostname to %s (%s)", fqdn, hostname)
         cloud.distro.set_hostname(hostname, fqdn)
-    except Exception:
-        util.logexc(log, "Failed to set the hostname to %s (%s)", fqdn,
-                    hostname)
-        raise
+    except Exception as e:
+        msg = "Failed to set the hostname to %s (%s)" % (fqdn, hostname)
+        util.logexc(log, msg)
+        raise SetHostnameError("%s: %s" % (msg, e))
+    write_json(prev_fn, {'hostname': hostname, 'fqdn': fqdn})
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py
index 35d8c57..98b0e66 100755
--- a/cloudinit/config/cc_ssh_authkey_fingerprints.py
+++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py
@@ -77,11 +77,10 @@ def _pprint_key_entries(user, key_fn, key_entries, hash_meth='md5',
     tbl = SimpleTable(tbl_fields)
     for entry in key_entries:
         if _is_printable_key(entry):
-            row = []
-            row.append(entry.keytype or '-')
-            row.append(_gen_fingerprint(entry.base64, hash_meth) or '-')
-            row.append(entry.options or '-')
-            row.append(entry.comment or '-')
+            row = [entry.keytype or '-',
+                   _gen_fingerprint(entry.base64, hash_meth) or '-',
+                   entry.options or '-',
+                   entry.comment or '-']
             tbl.add_row(row)
     authtbl_s = tbl.get_string()
     authtbl_lines = authtbl_s.splitlines()
diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py
index f87a343..b814c8b 100644
--- a/cloudinit/distros/arch.py
+++ b/cloudinit/distros/arch.py
@@ -129,11 +129,8 @@ class Distro(distros.Distro):
         if pkgs is None:
             pkgs = []
 
-        cmd = ['pacman']
+        cmd = ['pacman', "-Sy", "--quiet", "--noconfirm"]
         # Redirect output
-        cmd.append("-Sy")
-        cmd.append("--quiet")
-        cmd.append("--noconfirm")
 
         if args and isinstance(args, str):
             cmd.append(args)
diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py
index aa468bc..754d3df 100644
--- a/cloudinit/distros/freebsd.py
+++ b/cloudinit/distros/freebsd.py
@@ -132,6 +132,12 @@ class Distro(distros.Distro):
         LOG.debug("Using network interface %s", bsddev)
         return bsddev
 
+    def _select_hostname(self, hostname, fqdn):
+        # Should be FQDN if available. See rc.conf(5) in FreeBSD
+        if fqdn:
+            return fqdn
+        return hostname
+
     def _read_system_hostname(self):
         sys_hostname = self._read_hostname(filename=None)
         return ('rc.conf', sys_hostname)
diff --git a/cloudinit/distros/opensuse.py b/cloudinit/distros/opensuse.py
index a219e9f..162dfa0 100644
--- a/cloudinit/distros/opensuse.py
+++ b/cloudinit/distros/opensuse.py
@@ -67,11 +67,10 @@ class Distro(distros.Distro):
         if pkgs is None:
             pkgs = []
 
-        cmd = ['zypper']
         # No user interaction possible, enable non-interactive mode
-        cmd.append('--non-interactive')
+        cmd = ['zypper', '--non-interactive']
 
-        # Comand is the operation, such as install
+        # Command is the operation, such as install
         if command == 'upgrade':
             command = 'update'
         cmd.append(command)
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 4bcbf3a..0bb7fad 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -223,6 +223,8 @@ DEF_PASSWD_REDACTION = 'REDACTED'
 
 
 def get_hostname(hostname_command='hostname'):
+    if not isinstance(hostname_command, (list, tuple)):
+        hostname_command = (hostname_command,)
     return util.subp(hostname_command, capture=True)[0].strip()
 
 
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index ce47b6b..9450835 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -173,10 +173,7 @@ class OpenNebulaNetwork(object):
     def gen_conf(self):
         global_dns = self.context.get('DNS', "").split()
 
-        conf = []
-        conf.append('auto lo')
-        conf.append('iface lo inet loopback')
-        conf.append('')
+        conf = ['auto lo', 'iface lo inet loopback', '']
 
         for mac, dev in self.ifaces.items():
             mac = mac.lower()
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index a05ca2f..df0b374 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -276,21 +276,34 @@ class DataSource(object):
             return "iid-datasource"
         return str(self.metadata['instance-id'])
 
-    def get_hostname(self, fqdn=False, resolve_ip=False):
+    def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False):
+        """Get hostname or fqdn from the datasource. Look it up if desired.
+
+        @param fqdn: Boolean, set True to return hostname with domain.
+        @param resolve_ip: Boolean, set True to attempt to resolve an ipv4
+            address provided in local-hostname meta-data.
+        @param metadata_only: Boolean, set True to avoid looking up hostname
+            if meta-data doesn't have local-hostname present.
+
+        @return: hostname or qualified hostname. Optionally return None when
+            metadata_only is True and local-hostname data is not available.
+        """
         defdomain = "localdomain"
         defhost = "localhost"
         domain = defdomain
 
         if not self.metadata or 'local-hostname' not in self.metadata:
+            if metadata_only:
+                return None
             # this is somewhat questionable really.
             # the cloud datasource was asked for a hostname
             # and didn't have one. raising error might be more appropriate
             # but instead, basically look up the existing hostname
             toks = []
             hostname = util.get_hostname()
-            fqdn = util.get_fqdn_from_hosts(hostname)
-            if fqdn and fqdn.find(".") > 0:
-                toks = str(fqdn).split(".")
+            hosts_fqdn = util.get_fqdn_from_hosts(hostname)
+            if hosts_fqdn and hosts_fqdn.find(".") > 0:
+                toks = str(hosts_fqdn).split(".")
             elif hostname and hostname.find(".") > 0:
                 toks = str(hostname).split(".")
             elif hostname:
diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py
index af15115..5065083 100644
--- a/cloudinit/sources/tests/test_init.py
+++ b/cloudinit/sources/tests/test_init.py
@@ -7,7 +7,7 @@ import stat
 from cloudinit.helpers import Paths
 from cloudinit.sources import (
     INSTANCE_JSON_FILE, DataSource)
-from cloudinit.tests.helpers import CiTestCase, skipIf
+from cloudinit.tests.helpers import CiTestCase, skipIf, mock
 from cloudinit.user_data import UserDataProcessor
 from cloudinit import util
 
@@ -108,6 +108,74 @@ class TestDataSource(CiTestCase):
         self.assertEqual('userdata_raw', datasource.userdata_raw)
         self.assertEqual('vendordata_raw', datasource.vendordata_raw)
 
+    def test_get_hostname_strips_local_hostname_without_domain(self):
+        """Datasource.get_hostname strips metadata local-hostname of domain."""
+        tmp = self.tmp_dir()
+        datasource = DataSourceTestSubclassNet(
+            self.sys_cfg, self.distro, Paths({'run_dir': tmp}))
+        self.assertTrue(datasource.get_data())
+        self.assertEqual(
+            'test-subclass-hostname', datasource.metadata['local-hostname'])
+        self.assertEqual('test-subclass-hostname', datasource.get_hostname())
+        datasource.metadata['local-hostname'] = 'hostname.my.domain.com'
+        self.assertEqual('hostname', datasource.get_hostname())
+
+    def test_get_hostname_with_fqdn_returns_local_hostname_with_domain(self):
+        """Datasource.get_hostname with fqdn set gets qualified hostname."""
+        tmp = self.tmp_dir()
+        datasource = DataSourceTestSubclassNet(
+            self.sys_cfg, self.distro, Paths({'run_dir': tmp}))
+        self.assertTrue(datasource.get_data())
+        datasource.metadata['local-hostname'] = 'hostname.my.domain.com'
+        self.assertEqual(
+            'hostname.my.domain.com', datasource.get_hostname(fqdn=True))
+
+    def test_get_hostname_without_metadata_uses_system_hostname(self):
+        """Datasource.gethostname runs util.get_hostname when no metadata."""
+        tmp = self.tmp_dir()
+        datasource = DataSourceTestSubclassNet(
+            self.sys_cfg, self.distro, Paths({'run_dir': tmp}))
+        self.assertEqual({}, datasource.metadata)
+        mock_fqdn = 'cloudinit.sources.util.get_fqdn_from_hosts'
+        with mock.patch('cloudinit.sources.util.get_hostname') as m_gethost:
+            with mock.patch(mock_fqdn) as m_fqdn:
+                m_gethost.return_value = 'systemhostname.domain.com'
+                m_fqdn.return_value = None  # No maching fqdn in /etc/hosts
+                self.assertEqual('systemhostname', datasource.get_hostname())
+                self.assertEqual(
+                    'systemhostname.domain.com',
+                    datasource.get_hostname(fqdn=True))
+
+    def test_get_hostname_without_metadata_returns_none(self):
+        """Datasource.gethostname returns None when metadata_only and no MD."""
+        tmp = self.tmp_dir()
+        datasource = DataSourceTestSubclassNet(
+            self.sys_cfg, self.distro, Paths({'run_dir': tmp}))
+        self.assertEqual({}, datasource.metadata)
+        mock_fqdn = 'cloudinit.sources.util.get_fqdn_from_hosts'
+        with mock.patch('cloudinit.sources.util.get_hostname') as m_gethost:
+            with mock.patch(mock_fqdn) as m_fqdn:
+                self.assertIsNone(datasource.get_hostname(metadata_only=True))
+                self.assertIsNone(
+                    datasource.get_hostname(fqdn=True, metadata_only=True))
+        self.assertEqual([], m_gethost.call_args_list)
+        self.assertEqual([], m_fqdn.call_args_list)
+
+    def test_get_hostname_without_metadata_prefers_etc_hosts(self):
+        """Datasource.gethostname prefers /etc/hosts to util.get_hostname."""
+        tmp = self.tmp_dir()
+        datasource = DataSourceTestSubclassNet(
+            self.sys_cfg, self.distro, Paths({'run_dir': tmp}))
+        self.assertEqual({}, datasource.metadata)
+        mock_fqdn = 'cloudinit.sources.util.get_fqdn_from_hosts'
+        with mock.patch('cloudinit.sources.util.get_hostname') as m_gethost:
+            with mock.patch(mock_fqdn) as m_fqdn:
+                m_gethost.return_value = 'systemhostname.domain.com'
+                m_fqdn.return_value = 'fqdnhostname.domain.com'
+                self.assertEqual('fqdnhostname', datasource.get_hostname())
+                self.assertEqual('fqdnhostname.domain.com',
+                                 datasource.get_hostname(fqdn=True))
+
     def test_get_data_write_json_instance_data(self):
         """get_data writes INSTANCE_JSON_FILE to run_dir as readonly root."""
         tmp = self.tmp_dir()
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index d045268..bc4ebc8 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -132,8 +132,7 @@ class Init(object):
         return initial_dirs
 
     def purge_cache(self, rm_instance_lnk=False):
-        rm_list = []
-        rm_list.append(self.paths.boot_finished)
+        rm_list = [self.paths.boot_finished]
         if rm_instance_lnk:
             rm_list.append(self.paths.instance_link)
         for f in rm_list:
diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py
index 41d9a8e..14c0b0b 100644
--- a/cloudinit/tests/helpers.py
+++ b/cloudinit/tests/helpers.py
@@ -409,6 +409,19 @@ except AttributeError:
         return decorator
 
 
+try:
+    import jsonschema
+    assert jsonschema  # avoid pyflakes error F401: import unused
+    _missing_jsonschema_dep = False
+except ImportError:
+    _missing_jsonschema_dep = True
+
+
+def skipUnlessJsonSchema():
+    return skipIf(
+        _missing_jsonschema_dep, "No python-jsonschema dependency present.")
+
+
 # older versions of mock do not have the useful 'assert_not_called'
 if not hasattr(mock.Mock, 'assert_not_called'):
     def __mock_assert_not_called(mmock):
diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py
index ba6bf69..d30643d 100644
--- a/cloudinit/tests/test_util.py
+++ b/cloudinit/tests/test_util.py
@@ -16,6 +16,25 @@ MOUNT_INFO = [
 ]
 
 
+class FakeCloud(object):
+
+    def __init__(self, hostname, fqdn):
+        self.hostname = hostname
+        self.fqdn = fqdn
+        self.calls = []
+
+    def get_hostname(self, fqdn=None, metadata_only=None):
+        myargs = {}
+        if fqdn is not None:
+            myargs['fqdn'] = fqdn
+        if metadata_only is not None:
+            myargs['metadata_only'] = metadata_only
+        self.calls.append(myargs)
+        if fqdn:
+            return self.fqdn
+        return self.hostname
+
+
 class TestUtil(CiTestCase):
 
     def test_parse_mount_info_no_opts_no_arg(self):
@@ -44,3 +63,81 @@ class TestUtil(CiTestCase):
         m_mount_info.return_value = ('/dev/sda1', 'btrfs', '/', 'ro,relatime')
         is_rw = util.mount_is_read_write('/')
         self.assertEqual(is_rw, False)
+
+
+class TestShellify(CiTestCase):
+
+    def test_input_dict_raises_type_error(self):
+        self.assertRaisesRegex(
+            TypeError, 'Input.*was.*dict.*xpected',
+            util.shellify, {'mykey': 'myval'})
+
+    def test_input_str_raises_type_error(self):
+        self.assertRaisesRegex(
+            TypeError, 'Input.*was.*str.*xpected', util.shellify, "foobar")
+
+    def test_value_with_int_raises_type_error(self):
+        self.assertRaisesRegex(
+            TypeError, 'shellify.*int', util.shellify, ["foo", 1])
+
+    def test_supports_strings_and_lists(self):
+        self.assertEqual(
+            '\n'.join(["#!/bin/sh", "echo hi mom", "'echo' 'hi dad'",
+                       "'echo' 'hi' 'sis'", ""]),
+            util.shellify(["echo hi mom", ["echo", "hi dad"],
+                           ('echo', 'hi', 'sis')]))
+
+
+class TestGetHostnameFqdn(CiTestCase):
+
+    def test_get_hostname_fqdn_from_only_cfg_fqdn(self):
+        """When cfg only has the fqdn key, derive hostname and fqdn from it."""
+        hostname, fqdn = util.get_hostname_fqdn(
+            cfg={'fqdn': 'myhost.domain.com'}, cloud=None)
+        self.assertEqual('myhost', hostname)
+        self.assertEqual('myhost.domain.com', fqdn)
+
+    def test_get_hostname_fqdn_from_cfg_fqdn_and_hostname(self):
+        """When cfg has both fqdn and hostname keys, return them."""
+        hostname, fqdn = util.get_hostname_fqdn(
+            cfg={'fqdn': 'myhost.domain.com', 'hostname': 'other'}, cloud=None)
+        self.assertEqual('other', hostname)
+        self.assertEqual('myhost.domain.com', fqdn)
+
+    def test_get_hostname_fqdn_from_cfg_hostname_with_domain(self):
+        """When cfg has only hostname key which represents a fqdn, use that."""
+        hostname, fqdn = util.get_hostname_fqdn(
+            cfg={'hostname': 'myhost.domain.com'}, cloud=None)
+        self.assertEqual('myhost', hostname)
+        self.assertEqual('myhost.domain.com', fqdn)
+
+    def test_get_hostname_fqdn_from_cfg_hostname_without_domain(self):
+        """When cfg has a hostname without a '.' query cloud.get_hostname."""
+        mycloud = FakeCloud('cloudhost', 'cloudhost.mycloud.com')
+        hostname, fqdn = util.get_hostname_fqdn(
+            cfg={'hostname': 'myhost'}, cloud=mycloud)
+        self.assertEqual('myhost', hostname)
+        self.assertEqual('cloudhost.mycloud.com', fqdn)
+        self.assertEqual(
+            [{'fqdn': True, 'metadata_only': False}], mycloud.calls)
+
+    def test_get_hostname_fqdn_from_without_fqdn_or_hostname(self):
+        """When cfg has neither hostname nor fqdn cloud.get_hostname."""
+        mycloud = FakeCloud('cloudhost', 'cloudhost.mycloud.com')
+        hostname, fqdn = util.get_hostname_fqdn(cfg={}, cloud=mycloud)
+        self.assertEqual('cloudhost', hostname)
+        self.assertEqual('cloudhost.mycloud.com', fqdn)
+        self.assertEqual(
+            [{'fqdn': True, 'metadata_only': False},
+             {'metadata_only': False}], mycloud.calls)
+
+    def test_get_hostname_fqdn_from_passes_metadata_only_to_cloud(self):
+        """Calls to cloud.get_hostname pass the metadata_only parameter."""
+        mycloud = FakeCloud('cloudhost', 'cloudhost.mycloud.com')
+        hostname, fqdn = util.get_hostname_fqdn(
+            cfg={}, cloud=mycloud, metadata_only=True)
+        self.assertEqual(
+            [{'fqdn': True, 'metadata_only': True},
+             {'metadata_only': True}], mycloud.calls)
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index 0a5be0b..4e814a5 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -47,7 +47,7 @@ try:
     _REQ_VER = LooseVersion(_REQ.version)  # pylint: disable=no-member
     if _REQ_VER >= LooseVersion('0.8.8'):
         SSL_ENABLED = True
-    if _REQ_VER >= LooseVersion('0.7.0') and _REQ_VER < LooseVersion('1.0.0'):
+    if LooseVersion('0.7.0') <= _REQ_VER < LooseVersion('1.0.0'):
         CONFIG_ENABLED = True
 except ImportError:
     pass
@@ -121,7 +121,7 @@ class UrlResponse(object):
         upper = 300
         if redirects_ok:
             upper = 400
-        if self.code >= 200 and self.code < upper:
+        if 200 <= self.code < upper:
             return True
         else:
             return False
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 02dc2ce..823d80b 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -546,7 +546,7 @@ def is_ipv4(instr):
         return False
 
     try:
-        toks = [x for x in toks if int(x) < 256 and int(x) >= 0]
+        toks = [x for x in toks if 0 <= int(x) < 256]
     except Exception:
         return False
 
@@ -716,8 +716,7 @@ def redirect_output(outfmt, errfmt, o_out=None, o_err=None):
 def make_url(scheme, host, port=None,
              path='', params='', query='', fragment=''):
 
-    pieces = []
-    pieces.append(scheme or '')
+    pieces = [scheme or '']
 
     netloc = ''
     if host:
@@ -1026,9 +1025,16 @@ def dos2unix(contents):
     return contents.replace('\r\n', '\n')
 
 
-def get_hostname_fqdn(cfg, cloud):
-    # return the hostname and fqdn from 'cfg'.  If not found in cfg,
-    # then fall back to data from cloud
+def get_hostname_fqdn(cfg, cloud, metadata_only=False):
+    """Get hostname and fqdn from config if present and fallback to cloud.
+
+    @param cfg: Dictionary of merged user-data configuration (from init.cfg).
+    @param cloud: Cloud instance from init.cloudify().
+    @param metadata_only: Boolean, set True to only query cloud meta-data,
+        returning None if not present in meta-data.
+    @return: a Tuple of strings <hostname>, <fqdn>. Values can be none when
+        metadata_only is True and no cfg or metadata provides hostname info.
+    """
     if "fqdn" in cfg:
         # user specified a fqdn.  Default hostname then is based off that
         fqdn = cfg['fqdn']
@@ -1042,11 +1048,11 @@ def get_hostname_fqdn(cfg, cloud):
         else:
             # no fqdn set, get fqdn from cloud.
             # get hostname from cfg if available otherwise cloud
-            fqdn = cloud.get_hostname(fqdn=True)
+            fqdn = cloud.get_hostname(fqdn=True, metadata_only=metadata_only)
             if "hostname" in cfg:
                 hostname = cfg['hostname']
             else:
-                hostname = cloud.get_hostname()
+                hostname = cloud.get_hostname(metadata_only=metadata_only)
     return (hostname, fqdn)
 
 
@@ -1868,8 +1874,14 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False,
     # Popen converts entries in the arguments array from non-bytes to bytes.
     # When locale is unset it may use ascii for that encoding which can
     # cause UnicodeDecodeErrors. (LP: #1751051)
-    bytes_args = [x if isinstance(x, six.binary_type) else x.encode("utf-8")
-                  for x in args]
+    if isinstance(args, six.binary_type):
+        bytes_args = args
+    elif isinstance(args, six.string_types):
+        bytes_args = args.encode("utf-8")
+    else:
+        bytes_args = [
+            x if isinstance(x, six.binary_type) else x.encode("utf-8")
+            for x in args]
     try:
         sp = subprocess.Popen(bytes_args, stdout=stdout,
                               stderr=stderr, stdin=stdin,
@@ -1923,6 +1935,11 @@ def abs_join(*paths):
 #    if it is an array, shell protect it (with single ticks)
 #    if it is a string, do nothing
 def shellify(cmdlist, add_header=True):
+    if not isinstance(cmdlist, (tuple, list)):
+        raise TypeError(
+            "Input to shellify was type '%s'. Expected list or tuple." %
+            (type_utils.obj_name(cmdlist)))
+
     content = ''
     if add_header:
         content += "#!/bin/sh\n"
@@ -1931,7 +1948,7 @@ def shellify(cmdlist, add_header=True):
     for args in cmdlist:
         # If the item is a list, wrap all items in single tick.
         # If its not, then just write it directly.
-        if isinstance(args, list):
+        if isinstance(args, (list, tuple)):
             fixed = []
             for f in args:
                 fixed.append("'%s'" % (six.text_type(f).replace("'", escaped)))
@@ -1941,9 +1958,10 @@ def shellify(cmdlist, add_header=True):
             content = "%s%s\n" % (content, args)
             cmds_made += 1
         else:
-            raise RuntimeError(("Unable to shellify type %s"
-                                " which is not a list or string")
-                               % (type_utils.obj_name(args)))
+            raise TypeError(
+                "Unable to shellify type '%s'. Expected list, string, tuple. "
+                "Got: %s" % (type_utils.obj_name(args), args))
+
     LOG.debug("Shellified %s commands.", cmds_made)
     return content
 
diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl
index fad1184..cf2e240 100644
--- a/config/cloud.cfg.tmpl
+++ b/config/cloud.cfg.tmpl
@@ -113,9 +113,9 @@ cloud_final_modules:
 {% if variant not in ["freebsd"] %}
  - puppet
  - chef
- - salt-minion
  - mcollective
 {% endif %}
+ - salt-minion
  - rightscale_userdata
  - scripts-vendor
  - scripts-per-once
diff --git a/debian/changelog b/debian/changelog
index 27dba2c..f1ba6ef 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,25 @@
+cloud-init (18.1-17-g97012fbb-0ubuntu1) bionic; urgency=medium
+
+  * New upstream snapshot.
+    - util: Fix subp regression. Allow specifying subp command as a string.
+      (LP: #1755965)
+    - doc: fix all warnings issued by 'tox -e doc'
+    - FreeBSD: Set hostname to FQDN. [Dominic Schlegel] (LP: #1753499)
+    - tests: fix run_tree and bddeb
+    - tests: Fix some warnings in tests that popped up with newer python.
+    - set_hostname: When present in metadata, set it before network bringup.
+      (LP: #1746455)
+    - tests: Centralize and re-use skipTest based on json schema presense.
+    - This commit fixes get_hostname on the AzureDataSource.
+      [Douglas Jordan] (LP: #1754495)
+    - shellify: raise TypeError on bad input.
+    - Make salt minion module work on FreeBSD.
+      [Dominic Schlegel] (LP: #1721503)
+    - Simplify some comparisions. [Rémy Léone]
+    - Change some list creation and population to literal. [Rémy Léone]
+
+ -- Chad Smith <[e-mail]>  Thu, 15 Mar 2018 14:48:29 -0600
+
 cloud-init (18.1-5-g40e77380-0ubuntu1) bionic; urgency=medium
 
   * New upstream snapshot.
diff --git a/doc/rtd/topics/capabilities.rst b/doc/rtd/topics/capabilities.rst
index ae3a0c7..3e2c9e3 100644
--- a/doc/rtd/topics/capabilities.rst
+++ b/doc/rtd/topics/capabilities.rst
@@ -44,13 +44,14 @@ Currently defined feature names include:
 CLI Interface
 =============
 
-   The command line documentation is accessible on any cloud-init
-installed system:
+The command line documentation is accessible on any cloud-init installed
+system:
 
-.. code-block:: bash
+.. code-block:: shell-session
 
   % cloud-init --help
   usage: cloud-init [-h] [--version] [--file FILES]
+
                     [--debug] [--force]
                     {init,modules,single,dhclient-hook,features,analyze,devel,collect-logs,clean,status}
                     ...
@@ -88,7 +89,7 @@ Print out each feature supported.  If cloud-init does not have the
 features subcommand, it also does not support any features described in
 this document.
 
-.. code-block:: bash
+.. code-block:: shell-session
 
   % cloud-init features
   NETWORK_CONFIG_V1
@@ -100,10 +101,11 @@ cloud-init status
 -----------------
 Report whether cloud-init is running, done, disabled or errored. Exits
 non-zero if an error is detected in cloud-init.
+
  * **--long**: Detailed status information.
  * **--wait**: Block until cloud-init completes.
 
-.. code-block:: bash
+.. code-block:: shell-session
 
   % cloud-init status --long
   status: done
@@ -214,7 +216,7 @@ of once-per-instance:
  * **--frequency**: Optionally override the declared module frequency
    with one of (always|once-per-instance|once)
 
-.. code-block:: bash
+.. code-block:: shell-session
 
   % cloud-init single --name set_hostname --frequency always
 
diff --git a/doc/rtd/topics/debugging.rst b/doc/rtd/topics/debugging.rst
index c2b47ed..cacc8a2 100644
--- a/doc/rtd/topics/debugging.rst
+++ b/doc/rtd/topics/debugging.rst
@@ -1,6 +1,6 @@
-**********************
+********************************
 Testing and debugging cloud-init
-**********************
+********************************
 
 Overview
 ========
@@ -10,7 +10,7 @@ deployed instances.
 .. _boot_time_analysis:
 
 Boot Time Analysis - cloud-init analyze
-======================================
+=======================================
 Occasionally instances don't appear as performant as we would like and
 cloud-init packages a simple facility to inspect what operations took
 cloud-init the longest during boot and setup.
@@ -22,9 +22,9 @@ determine the long-pole in cloud-init configuration and setup. These
 subcommands default to reading /var/log/cloud-init.log.
 
 * ``analyze show`` Parse and organize cloud-init.log events by stage and
-include each sub-stage granularity with time delta reports.
+  include each sub-stage granularity with time delta reports.
 
-.. code-block:: bash
+.. code-block:: shell-session
 
     $ cloud-init analyze show -i my-cloud-init.log
     -- Boot Record 01 --
@@ -41,9 +41,9 @@ include each sub-stage granularity with time delta reports.
 
 
 * ``analyze dump`` Parse cloud-init.log into event records and return a list of
-dictionaries that can be consumed for other reporting needs.
+  dictionaries that can be consumed for other reporting needs.
 
-.. code-block:: bash
+.. code-block:: shell-session
 
     $ cloud-init analyze blame -i my-cloud-init.log
     [
@@ -56,10 +56,10 @@ dictionaries that can be consumed for other reporting needs.
      },...
 
 * ``analyze blame`` Parse cloud-init.log into event records and sort them based
-on highest time cost for quick assessment of areas of cloud-init that may need
-improvement.
+  on highest time cost for quick assessment of areas of cloud-init that may
+  need improvement.
 
-.. code-block:: bash
+.. code-block:: shell-session
 
     $ cloud-init analyze blame -i my-cloud-init.log
     -- Boot Record 11 --
@@ -73,31 +73,36 @@ Analyze quickstart - LXC
 ---------------------------
 To quickly obtain a cloud-init log try using lxc on any ubuntu system:
 
-.. code-block:: bash
+.. code-block:: shell-session
+
+    $ lxc init ubuntu-daily:xenial x1
+    $ lxc start x1
+    $ # Take lxc's cloud-init.log and pipe it to the analyzer
+    $ lxc file pull x1/var/log/cloud-init.log - | cloud-init analyze dump -i -
+    $ lxc file pull x1/var/log/cloud-init.log - | \
+      python3 -m cloudinit.analyze dump -i -
 
-  $ lxc init ubuntu-daily:xenial x1
-  $ lxc start x1
-  # Take lxc's cloud-init.log and pipe it to the analyzer
-  $ lxc file pull x1/var/log/cloud-init.log - | cloud-init analyze dump -i -
-  $ lxc file pull x1/var/log/cloud-init.log - | \
-  python3 -m cloudinit.analyze dump -i -
 
 Analyze quickstart - KVM
 ---------------------------
 To quickly analyze a KVM a cloud-init log:
 
 1. Download the current cloud image
-  wget https://cloud-images.ubuntu.com/daily/server/xenial/current/xenial-server-cloudimg-amd64.img
+
+.. code-block:: shell-session
+
+    $ wget https://cloud-images.ubuntu.com/daily/server/xenial/current/xenial-server-cloudimg-amd64.img
+
 2. Create a snapshot image to preserve the original cloud-image
 
-.. code-block:: bash
+.. code-block:: shell-session
 
     $ qemu-img create -b xenial-server-cloudimg-amd64.img -f qcow2 \
     test-cloudinit.qcow2
 
 3. Create a seed image with metadata using `cloud-localds`
 
-.. code-block:: bash
+.. code-block:: shell-session
 
     $ cat > user-data <<EOF
       #cloud-config
@@ -108,18 +113,18 @@ To quickly analyze a KVM a cloud-init log:
 
 4. Launch your modified VM
 
-.. code-block:: bash
+.. code-block:: shell-session
 
     $  kvm -m 512 -net nic -net user -redir tcp:2222::22 \
-   -drive file=test-cloudinit.qcow2,if=virtio,format=qcow2 \
-   -drive file=my-seed.img,if=virtio,format=raw
+        -drive file=test-cloudinit.qcow2,if=virtio,format=qcow2 \
+        -drive file=my-seed.img,if=virtio,format=raw
 
 5. Analyze the boot (blame, dump, show)
 
-.. code-block:: bash
+.. code-block:: shell-session
 
     $ ssh -p 2222 ubuntu@localhost 'cat /var/log/cloud-init.log' | \
-   cloud-init analyze blame -i -
+       cloud-init analyze blame -i -
 
 
 Running single cloud config modules
@@ -136,7 +141,7 @@ prevents a module from running again if it has already been run. To ensure that
 a module is run again, the desired frequency can be overridden on the
 commandline:
 
-.. code-block:: bash
+.. code-block:: shell-session
 
   $ sudo cloud-init single --name cc_ssh --frequency always
   ...
diff --git a/doc/rtd/topics/network-config.rst b/doc/rtd/topics/network-config.rst
index 96c1cf5..1e99455 100644
--- a/doc/rtd/topics/network-config.rst
+++ b/doc/rtd/topics/network-config.rst
@@ -202,7 +202,7 @@ is helpful for examining expected output for a given input format.
 
 CLI Interface :
 
-.. code-block:: bash
+.. code-block:: shell-session
 
   % tools/net-convert.py --help
   usage: net-convert.py [-h] --network-data PATH --kind
@@ -222,7 +222,7 @@ CLI Interface :
 
 Example output converting V2 to sysconfig:
 
-.. code-block:: bash
+.. code-block:: shell-session
 
   % tools/net-convert.py --network-data v2.yaml --kind yaml \
       --output-kind sysconfig -d target
diff --git a/doc/rtd/topics/tests.rst b/doc/rtd/topics/tests.rst
index bf04bb3..cac4a6e 100644
--- a/doc/rtd/topics/tests.rst
+++ b/doc/rtd/topics/tests.rst
@@ -21,7 +21,7 @@ Overview
 In order to avoid the need for dependencies and ease the setup and
 configuration users can run the integration tests via tox:
 
-.. code-block:: bash
+.. code-block:: shell-session
 
     $ git clone https://git.launchpad.net/cloud-init
     $ cd cloud-init
@@ -51,7 +51,7 @@ The first example will provide a complete end-to-end run of data
 collection and verification. There are additional examples below
 explaining how to run one or the other independently.
 
-.. code-block:: bash
+.. code-block:: shell-session
 
     $ git clone https://git.launchpad.net/cloud-init
     $ cd cloud-init
@@ -93,7 +93,7 @@ If developing tests it may be necessary to see if cloud-config works as
 expected and the correct files are pulled down. In this case only a
 collect can be ran by running:
 
-.. code-block:: bash
+.. code-block:: shell-session
 
     $ tox -e citest -- collect -n xenial --data-dir /tmp/collection
 
@@ -106,7 +106,7 @@ Verify
 When developing tests it is much easier to simply rerun the verify scripts
 without the more lengthy collect process. This can be done by running:
 
-.. code-block:: bash
+.. code-block:: shell-session
 
     $ tox -e citest -- verify --data-dir /tmp/collection
 
@@ -133,7 +133,7 @@ cloud-init deb from or use the ``tree_run`` command using a copy of
 cloud-init located in a different directory, use the option ``--cloud-init
 /path/to/cloud-init``.
 
-.. code-block:: bash
+.. code-block:: shell-session
 
     $ tox -e citest -- tree_run --verbose \
         --os-name xenial --os-name stretch \
@@ -331,7 +331,7 @@ Integration tests are located under the `tests/cloud_tests` directory.
 Test configurations are placed under `configs` and the test verification
 scripts under `testcases`:
 
-.. code-block:: bash
+.. code-block:: shell-session
 
     cloud-init$ tree -d tests/cloud_tests/
     tests/cloud_tests/
@@ -362,7 +362,7 @@ The following would create a test case named ``example`` under the
 ``modules`` category with the given description, and cloud config data read
 in from ``/tmp/user_data``.
 
-.. code-block:: bash
+.. code-block:: shell-session
 
     $ tox -e citest -- create modules/example \
         -d "a simple example test case" -c "$(< /tmp/user_data)"
@@ -385,7 +385,7 @@ Development Checklist
     * Placed in the appropriate sub-folder in the test cases directory
 * Tested by running the test:
 
-   .. code-block:: bash
+   .. code-block:: shell-session
 
        $ tox -e citest -- run -verbose \
            --os-name <release target> \
@@ -404,14 +404,14 @@ These configuration files are the standard that the AWS cli and other AWS
 tools utilize for interacting directly with AWS itself and are normally
 generated when running ``aws configure``:
 
-.. code-block:: bash
+.. code-block:: shell-session
 
     $ cat $HOME/.aws/credentials
     [default]
     aws_access_key_id = <KEY HERE>
     aws_secret_access_key = <KEY HERE>
 
-.. code-block:: bash
+.. code-block:: shell-session
 
     $ cat $HOME/.aws/config
     [default]
diff --git a/tests/cloud_tests/bddeb.py b/tests/cloud_tests/bddeb.py
index a6d5069..b9cfcfa 100644
--- a/tests/cloud_tests/bddeb.py
+++ b/tests/cloud_tests/bddeb.py
@@ -16,7 +16,7 @@ pre_reqs = ['devscripts', 'equivs', 'git', 'tar']
 
 def _out(cmd_res):
     """Get clean output from cmd result."""
-    return cmd_res[0].strip()
+    return cmd_res[0].decode("utf-8").strip()
 
 
 def build_deb(args, instance):
diff --git a/tests/cloud_tests/platforms/ec2/__init__.py b/tests/cloud_tests/platforms/ec2/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests/cloud_tests/platforms/ec2/__init__.py
diff --git a/tests/cloud_tests/platforms/lxd/__init__.py b/tests/cloud_tests/platforms/lxd/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests/cloud_tests/platforms/lxd/__init__.py
diff --git a/tests/cloud_tests/platforms/lxd/platform.py b/tests/cloud_tests/platforms/lxd/platform.py
index 6a01692..f7251a0 100644
--- a/tests/cloud_tests/platforms/lxd/platform.py
+++ b/tests/cloud_tests/platforms/lxd/platform.py
@@ -101,8 +101,4 @@ class LXDPlatform(Platform):
         """
         return self.client.images.get_by_alias(alias)
 
-    def destroy(self):
-        """Clean up platform data."""
-        super(LXDPlatform, self).destroy()
-
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/platforms/nocloudkvm/__init__.py b/tests/cloud_tests/platforms/nocloudkvm/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests/cloud_tests/platforms/nocloudkvm/__init__.py
diff --git a/tests/cloud_tests/platforms/nocloudkvm/instance.py b/tests/cloud_tests/platforms/nocloudkvm/instance.py
index 932dc0f..33ff3f2 100644
--- a/tests/cloud_tests/platforms/nocloudkvm/instance.py
+++ b/tests/cloud_tests/platforms/nocloudkvm/instance.py
@@ -109,7 +109,7 @@ class NoCloudKVMInstance(Instance):
         if self.pid:
             try:
                 c_util.subp(['kill', '-9', self.pid])
-            except util.ProcessExectuionError:
+            except c_util.ProcessExecutionError:
                 pass
 
         if self.pid_file:
diff --git a/tests/cloud_tests/platforms/nocloudkvm/platform.py b/tests/cloud_tests/platforms/nocloudkvm/platform.py
index a7e6f5d..8593346 100644
--- a/tests/cloud_tests/platforms/nocloudkvm/platform.py
+++ b/tests/cloud_tests/platforms/nocloudkvm/platform.py
@@ -21,10 +21,6 @@ class NoCloudKVMPlatform(Platform):
 
     platform_name = 'nocloud-kvm'
 
-    def __init__(self, config):
-        """Set up platform."""
-        super(NoCloudKVMPlatform, self).__init__(config)
-
     def get_image(self, img_conf):
         """Get image using specified image configuration.
 
diff --git a/tests/cloud_tests/platforms/platforms.py b/tests/cloud_tests/platforms/platforms.py
index 1542b3b..abbfebb 100644
--- a/tests/cloud_tests/platforms/platforms.py
+++ b/tests/cloud_tests/platforms/platforms.py
@@ -2,12 +2,15 @@
 
 """Base platform class."""
 import os
+import shutil
 
 from simplestreams import filters, mirrors
 from simplestreams import util as s_util
 
 from cloudinit import util as c_util
 
+from tests.cloud_tests import util
+
 
 class Platform(object):
     """Base class for platforms."""
@@ -17,7 +20,14 @@ class Platform(object):
     def __init__(self, config):
         """Set up platform."""
         self.config = config
-        self._generate_ssh_keys(config['data_dir'])
+        self.tmpdir = util.mkdtemp()
+        if 'data_dir' in config:
+            self.data_dir = config['data_dir']
+        else:
+            self.data_dir = os.path.join(self.tmpdir, "data_dir")
+            os.mkdir(self.data_dir)
+
+        self._generate_ssh_keys(self.data_dir)
 
     def get_image(self, img_conf):
         """Get image using specified image configuration.
@@ -29,7 +39,7 @@ class Platform(object):
 
     def destroy(self):
         """Clean up platform data."""
-        pass
+        shutil.rmtree(self.tmpdir)
 
     def _generate_ssh_keys(self, data_dir):
         """Generate SSH keys to be used with image."""
diff --git a/tests/cloud_tests/testcases/modules/salt_minion.py b/tests/cloud_tests/testcases/modules/salt_minion.py
index f13b48a..70917a4 100644
--- a/tests/cloud_tests/testcases/modules/salt_minion.py
+++ b/tests/cloud_tests/testcases/modules/salt_minion.py
@@ -31,4 +31,9 @@ class Test(base.CloudTestCase):
         out = self.get_data_file('grains')
         self.assertIn('role: web', out)
 
+    def test_minion_installed(self):
+        """Test if the salt-minion package is installed"""
+        out = self.get_data_file('minion_installed')
+        self.assertEqual(1, int(out))
+
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/testcases/modules/salt_minion.yaml b/tests/cloud_tests/testcases/modules/salt_minion.yaml
index ab0e05b..f20b976 100644
--- a/tests/cloud_tests/testcases/modules/salt_minion.yaml
+++ b/tests/cloud_tests/testcases/modules/salt_minion.yaml
@@ -3,7 +3,7 @@
 #
 # 2016-11-17: Currently takes >60 seconds results in test failure
 #
-enabled: False
+enabled: True
 cloud_config: |
   #cloud-config
   salt_minion:
@@ -35,5 +35,8 @@ collect_scripts:
   grains: |
     #!/bin/bash
     cat /etc/salt/grains
+  minion_installed: |
+    #!/bin/bash
+    dpkg -l | grep salt-minion | grep ii | wc -l
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/util.py b/tests/cloud_tests/util.py
index 6ff285e..3dd4996 100644
--- a/tests/cloud_tests/util.py
+++ b/tests/cloud_tests/util.py
@@ -460,6 +460,10 @@ class PlatformError(IOError):
         IOError.__init__(self, message)
 
 
+def mkdtemp(prefix='cloud_test_data'):
+    return tempfile.mkdtemp(prefix=prefix)
+
+
 class TempDir(object):
     """Configurable temporary directory like tempfile.TemporaryDirectory."""
 
@@ -480,7 +484,7 @@ class TempDir(object):
         @return_value: tempdir path
         """
         if not self.tmpdir:
-            self.tmpdir = tempfile.mkdtemp(prefix=self.prefix)
+            self.tmpdir = mkdtemp(prefix=self.prefix)
         LOG.debug('using tmpdir: %s', self.tmpdir)
         return self.tmpdir
 
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index 254e987..da7da0c 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -643,6 +643,21 @@ fdescfs            /dev/fd          fdescfs rw              0 0
         expected_config['config'].append(blacklist_config)
         self.assertEqual(netconfig, expected_config)
 
+    @mock.patch("cloudinit.sources.DataSourceAzure.util.subp")
+    def test_get_hostname_with_no_args(self, subp):
+        dsaz.get_hostname()
+        subp.assert_called_once_with(("hostname",), capture=True)
+
+    @mock.patch("cloudinit.sources.DataSourceAzure.util.subp")
+    def test_get_hostname_with_string_arg(self, subp):
+        dsaz.get_hostname(hostname_command="hostname")
+        subp.assert_called_once_with(("hostname",), capture=True)
+
+    @mock.patch("cloudinit.sources.DataSourceAzure.util.subp")
+    def test_get_hostname_with_iterable_arg(self, subp):
+        dsaz.get_hostname(hostname_command=("hostname",))
+        subp.assert_called_once_with(("hostname",), capture=True)
+
 
 class TestAzureBounce(CiTestCase):
 
diff --git a/tests/unittests/test_handler/test_handler_bootcmd.py b/tests/unittests/test_handler/test_handler_bootcmd.py
index dbf43e0..29fc25e 100644
--- a/tests/unittests/test_handler/test_handler_bootcmd.py
+++ b/tests/unittests/test_handler/test_handler_bootcmd.py
@@ -3,17 +3,11 @@
 from cloudinit.config import cc_bootcmd
 from cloudinit.sources import DataSourceNone
 from cloudinit import (distros, helpers, cloud, util)
-from cloudinit.tests.helpers import CiTestCase, mock, skipIf
+from cloudinit.tests.helpers import CiTestCase, mock, skipUnlessJsonSchema
 
 import logging
 import tempfile
 
-try:
-    import jsonschema
-    assert jsonschema  # avoid pyflakes error F401: import unused
-    _missing_jsonschema_dep = False
-except ImportError:
-    _missing_jsonschema_dep = True
 
 LOG = logging.getLogger(__name__)
 
@@ -69,10 +63,10 @@ class TestBootcmd(CiTestCase):
             cc_bootcmd.handle('cc_bootcmd', invalid_config, cc, LOG, [])
         self.assertIn('Failed to shellify bootcmd', self.logs.getvalue())
         self.assertEqual(
-            "'int' object is not iterable",
+            "Input to shellify was type 'int'. Expected list or tuple.",
             str(context_manager.exception))
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_handler_schema_validation_warns_non_array_type(self):
         """Schema validation warns of non-array type for bootcmd key.
 
@@ -88,7 +82,7 @@ class TestBootcmd(CiTestCase):
             self.logs.getvalue())
         self.assertIn('Failed to shellify', self.logs.getvalue())
 
-    @skipIf(_missing_jsonschema_dep, 'No python-jsonschema dependency')
+    @skipUnlessJsonSchema()
     def test_handler_schema_validation_warns_non_array_item_type(self):
         """Schema validation warns of non-array or string bootcmd items.
 
@@ -98,7 +92,7 @@ class TestBootcmd(CiTestCase):
         invalid_config = {
             'bootcmd': ['ls /', 20, ['wget', 'http://stuff/blah'], {'a': 'n'}]}
         cc = self._get_cloud('ubuntu')
-        with self.assertRaises(RuntimeError) as context_manager:
+        with self.assertRaises(TypeError) as context_manager:
             cc_bootcmd.handle('cc_bootcmd', invalid_config, cc, LOG, [])
         expected_warnings = [
             'bootcmd.1: 20 is not valid under any of the given schemas',
@@ -110,7 +104,8 @@ class TestBootcmd(CiTestCase):
             self.assertIn(warning, logs)
         self.assertIn('Failed to shellify', logs)
         self.assertEqual(
-            'Unable to shellify type int which is not a list or string',
+            ("Unable to shellify type 'int'. Expected list, string, tuple. "
+             "Got: 20"),
             str(context_manager.exception))
 
     def test_handler_creates_and_runs_bootcmd_script_with_instance_id(self):
diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py
index 28a8455..695897c 100644
--- a/tests/unittests/test_handler/test_handler_ntp.py
+++ b/tests/unittests/test_handler/test_handler_ntp.py
@@ -3,7 +3,8 @@
 from cloudinit.config import cc_ntp
 from cloudinit.sources import DataSourceNone
 from cloudinit import (distros, helpers, cloud, util)
-from cloudinit.tests.helpers import FilesystemMockingTestCase, mock, skipIf
+from cloudinit.tests.helpers import (
+    FilesystemMockingTestCase, mock, skipUnlessJsonSchema)
 
 
 import os
@@ -24,13 +25,6 @@ NTP={% for host in servers|list + pools|list %}{{ host }} {% endfor -%}
 {% endif -%}
 """
 
-try:
-    import jsonschema
-    assert jsonschema  # avoid pyflakes error F401: import unused
-    _missing_jsonschema_dep = False
-except ImportError:
-    _missing_jsonschema_dep = True
-
 
 class TestNtp(FilesystemMockingTestCase):
 
@@ -312,7 +306,7 @@ class TestNtp(FilesystemMockingTestCase):
                 content)
         self.assertNotIn('Invalid config:', self.logs.getvalue())
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_ntp_handler_schema_validation_warns_non_string_item_type(self):
         """Ntp schema validation warns of non-strings in pools or servers.
 
@@ -333,7 +327,7 @@ class TestNtp(FilesystemMockingTestCase):
             content = stream.read()
         self.assertEqual("servers ['valid', None]\npools [123]\n", content)
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_ntp_handler_schema_validation_warns_of_non_array_type(self):
         """Ntp schema validation warns of non-array pools or servers types.
 
@@ -354,7 +348,7 @@ class TestNtp(FilesystemMockingTestCase):
             content = stream.read()
         self.assertEqual("servers non-array\npools 123\n", content)
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_ntp_handler_schema_validation_warns_invalid_key_present(self):
         """Ntp schema validation warns of invalid keys present in ntp config.
 
@@ -378,7 +372,7 @@ class TestNtp(FilesystemMockingTestCase):
             "servers []\npools ['0.mycompany.pool.ntp.org']\n",
             content)
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_ntp_handler_schema_validation_warns_of_duplicates(self):
         """Ntp schema validation warns of duplicates in servers or pools.
 
diff --git a/tests/unittests/test_handler/test_handler_resizefs.py b/tests/unittests/test_handler/test_handler_resizefs.py
index 5aa3c49..c2a7f9f 100644
--- a/tests/unittests/test_handler/test_handler_resizefs.py
+++ b/tests/unittests/test_handler/test_handler_resizefs.py
@@ -7,21 +7,13 @@ from collections import namedtuple
 import logging
 import textwrap
 
-from cloudinit.tests.helpers import (CiTestCase, mock, skipIf, util,
-                                     wrap_and_call)
+from cloudinit.tests.helpers import (
+    CiTestCase, mock, skipUnlessJsonSchema, util, wrap_and_call)
 
 
 LOG = logging.getLogger(__name__)
 
 
-try:
-    import jsonschema
-    assert jsonschema  # avoid pyflakes error F401: import unused
-    _missing_jsonschema_dep = False
-except ImportError:
-    _missing_jsonschema_dep = True
-
-
 class TestResizefs(CiTestCase):
     with_logs = True
 
@@ -76,7 +68,7 @@ class TestResizefs(CiTestCase):
             'DEBUG: Skipping module named cc_resizefs, resizing disabled\n',
             self.logs.getvalue())
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_handle_schema_validation_logs_invalid_resize_rootfs_value(self):
         """The handle reports json schema violations as a warning.
 
diff --git a/tests/unittests/test_handler/test_handler_runcmd.py b/tests/unittests/test_handler/test_handler_runcmd.py
index 374c1d3..dbbb271 100644
--- a/tests/unittests/test_handler/test_handler_runcmd.py
+++ b/tests/unittests/test_handler/test_handler_runcmd.py
@@ -3,19 +3,13 @@
 from cloudinit.config import cc_runcmd
 from cloudinit.sources import DataSourceNone
 from cloudinit import (distros, helpers, cloud, util)
-from cloudinit.tests.helpers import FilesystemMockingTestCase, skipIf
+from cloudinit.tests.helpers import (
+    FilesystemMockingTestCase, skipUnlessJsonSchema)
 
 import logging
 import os
 import stat
 
-try:
-    import jsonschema
-    assert jsonschema  # avoid pyflakes error F401: import unused
-    _missing_jsonschema_dep = False
-except ImportError:
-    _missing_jsonschema_dep = True
-
 LOG = logging.getLogger(__name__)
 
 
@@ -56,7 +50,7 @@ class TestRuncmd(FilesystemMockingTestCase):
             ' /var/lib/cloud/instances/iid-datasource-none/scripts/runcmd',
             self.logs.getvalue())
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_handler_schema_validation_warns_non_array_type(self):
         """Schema validation warns of non-array type for runcmd key.
 
@@ -71,7 +65,7 @@ class TestRuncmd(FilesystemMockingTestCase):
             self.logs.getvalue())
         self.assertIn('Failed to shellify', self.logs.getvalue())
 
-    @skipIf(_missing_jsonschema_dep, 'No python-jsonschema dependency')
+    @skipUnlessJsonSchema()
     def test_handler_schema_validation_warns_non_array_item_type(self):
         """Schema validation warns of non-array or string runcmd items.
 
diff --git a/tests/unittests/test_handler/test_handler_set_hostname.py b/tests/unittests/test_handler/test_handler_set_hostname.py
index abdc17e..d09ec23 100644
--- a/tests/unittests/test_handler/test_handler_set_hostname.py
+++ b/tests/unittests/test_handler/test_handler_set_hostname.py
@@ -11,6 +11,7 @@ from cloudinit.tests import helpers as t_help
 
 from configobj import ConfigObj
 import logging
+import os
 import shutil
 from six import BytesIO
 import tempfile
@@ -19,14 +20,18 @@ LOG = logging.getLogger(__name__)
 
 
 class TestHostname(t_help.FilesystemMockingTestCase):
+
+    with_logs = True
+
     def setUp(self):
         super(TestHostname, self).setUp()
         self.tmp = tempfile.mkdtemp()
+        util.ensure_dir(os.path.join(self.tmp, 'data'))
         self.addCleanup(shutil.rmtree, self.tmp)
 
     def _fetch_distro(self, kind):
         cls = distros.fetch(kind)
-        paths = helpers.Paths({})
+        paths = helpers.Paths({'cloud_dir': self.tmp})
         return cls(kind, {}, paths)
 
     def test_write_hostname_rhel(self):
@@ -34,7 +39,7 @@ class TestHostname(t_help.FilesystemMockingTestCase):
             'hostname': 'blah.blah.blah.yahoo.com',
         }
         distro = self._fetch_distro('rhel')
-        paths = helpers.Paths({})
+        paths = helpers.Paths({'cloud_dir': self.tmp})
         ds = None
         cc = cloud.Cloud(ds, paths, {}, distro, None)
         self.patchUtils(self.tmp)
@@ -51,7 +56,7 @@ class TestHostname(t_help.FilesystemMockingTestCase):
             'hostname': 'blah.blah.blah.yahoo.com',
         }
         distro = self._fetch_distro('debian')
-        paths = helpers.Paths({})
+        paths = helpers.Paths({'cloud_dir': self.tmp})
         ds = None
         cc = cloud.Cloud(ds, paths, {}, distro, None)
         self.patchUtils(self.tmp)
@@ -65,7 +70,7 @@ class TestHostname(t_help.FilesystemMockingTestCase):
             'hostname': 'blah.blah.blah.suse.com',
         }
         distro = self._fetch_distro('sles')
-        paths = helpers.Paths({})
+        paths = helpers.Paths({'cloud_dir': self.tmp})
         ds = None
         cc = cloud.Cloud(ds, paths, {}, distro, None)
         self.patchUtils(self.tmp)
@@ -74,4 +79,48 @@ class TestHostname(t_help.FilesystemMockingTestCase):
             contents = util.load_file(distro.hostname_conf_fn)
             self.assertEqual('blah', contents.strip())
 
+    def test_multiple_calls_skips_unchanged_hostname(self):
+        """Only new hostname or fqdn values will generate a hostname call."""
+        distro = self._fetch_distro('debian')
+        paths = helpers.Paths({'cloud_dir': self.tmp})
+        ds = None
+        cc = cloud.Cloud(ds, paths, {}, distro, None)
+        self.patchUtils(self.tmp)
+        cc_set_hostname.handle(
+            'cc_set_hostname', {'hostname': 'hostname1.me.com'}, cc, LOG, [])
+        contents = util.load_file("/etc/hostname")
+        self.assertEqual('hostname1', contents.strip())
+        cc_set_hostname.handle(
+            'cc_set_hostname', {'hostname': 'hostname1.me.com'}, cc, LOG, [])
+        self.assertIn(
+            'DEBUG: No hostname changes. Skipping set-hostname\n',
+            self.logs.getvalue())
+        cc_set_hostname.handle(
+            'cc_set_hostname', {'hostname': 'hostname2.me.com'}, cc, LOG, [])
+        contents = util.load_file("/etc/hostname")
+        self.assertEqual('hostname2', contents.strip())
+        self.assertIn(
+            'Non-persistently setting the system hostname to hostname2',
+            self.logs.getvalue())
+
+    def test_error_on_distro_set_hostname_errors(self):
+        """Raise SetHostnameError on exceptions from distro.set_hostname."""
+        distro = self._fetch_distro('debian')
+
+        def set_hostname_error(hostname, fqdn):
+            raise Exception("OOPS on: %s" % fqdn)
+
+        distro.set_hostname = set_hostname_error
+        paths = helpers.Paths({'cloud_dir': self.tmp})
+        ds = None
+        cc = cloud.Cloud(ds, paths, {}, distro, None)
+        self.patchUtils(self.tmp)
+        with self.assertRaises(cc_set_hostname.SetHostnameError) as ctx_mgr:
+            cc_set_hostname.handle(
+                'somename', {'hostname': 'hostname1.me.com'}, cc, LOG, [])
+        self.assertEqual(
+            'Failed to set the hostname to hostname1.me.com (hostname1):'
+            ' OOPS on: hostname1.me.com',
+            str(ctx_mgr.exception))
+
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py
index df67a0e..1ecb6c6 100644
--- a/tests/unittests/test_handler/test_schema.py
+++ b/tests/unittests/test_handler/test_schema.py
@@ -6,7 +6,7 @@ from cloudinit.config.schema import (
     validate_cloudconfig_schema, main)
 from cloudinit.util import subp, write_file
 
-from cloudinit.tests.helpers import CiTestCase, mock, skipIf
+from cloudinit.tests.helpers import CiTestCase, mock, skipUnlessJsonSchema
 
 from copy import copy
 import os
@@ -14,13 +14,6 @@ from six import StringIO
 from textwrap import dedent
 from yaml import safe_load
 
-try:
-    import jsonschema
-    assert jsonschema  # avoid pyflakes error F401: import unused
-    _missing_jsonschema_dep = False
-except ImportError:
-    _missing_jsonschema_dep = True
-
 
 class GetSchemaTest(CiTestCase):
 
@@ -73,7 +66,7 @@ class ValidateCloudConfigSchemaTest(CiTestCase):
 
     with_logs = True
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_validateconfig_schema_non_strict_emits_warnings(self):
         """When strict is False validate_cloudconfig_schema emits warnings."""
         schema = {'properties': {'p1': {'type': 'string'}}}
@@ -82,7 +75,7 @@ class ValidateCloudConfigSchemaTest(CiTestCase):
             "Invalid config:\np1: -1 is not of type 'string'\n",
             self.logs.getvalue())
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_validateconfig_schema_emits_warning_on_missing_jsonschema(self):
         """Warning from validate_cloudconfig_schema when missing jsonschema."""
         schema = {'properties': {'p1': {'type': 'string'}}}
@@ -92,7 +85,7 @@ class ValidateCloudConfigSchemaTest(CiTestCase):
             'Ignoring schema validation. python-jsonschema is not present',
             self.logs.getvalue())
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_validateconfig_schema_strict_raises_errors(self):
         """When strict is True validate_cloudconfig_schema raises errors."""
         schema = {'properties': {'p1': {'type': 'string'}}}
@@ -102,7 +95,7 @@ class ValidateCloudConfigSchemaTest(CiTestCase):
             "Cloud config schema errors: p1: -1 is not of type 'string'",
             str(context_mgr.exception))
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_validateconfig_schema_honors_formats(self):
         """With strict True, validate_cloudconfig_schema errors on format."""
         schema = {
@@ -153,7 +146,7 @@ class ValidateCloudConfigFileTest(CiTestCase):
                 self.config_file),
             str(context_mgr.exception))
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_validateconfig_file_sctricty_validates_schema(self):
         """validate_cloudconfig_file raises errors on invalid schema."""
         schema = {
@@ -376,7 +369,7 @@ class CloudTestsIntegrationTest(CiTestCase):
     raises Warnings or errors on invalid cloud-config schema.
     """
 
-    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    @skipUnlessJsonSchema()
     def test_all_integration_test_cloud_config_schema(self):
         """Validate schema of cloud_tests yaml files looking for warnings."""
         schema = get_schema()
diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
index 89ae40f..499e7c9 100644
--- a/tests/unittests/test_util.py
+++ b/tests/unittests/test_util.py
@@ -632,6 +632,24 @@ class TestSubp(helpers.CiTestCase):
         # but by using bash, we remove dependency on another program.
         return([BASH, '-c', 'printf "$@"', 'printf'] + list(args))
 
+    def test_subp_handles_bytestrings(self):
+        """subp can run a bytestring command if shell is True."""
+        tmp_file = self.tmp_path('test.out')
+        cmd = 'echo HI MOM >> {tmp_file}'.format(tmp_file=tmp_file)
+        (out, _err) = util.subp(cmd.encode('utf-8'), shell=True)
+        self.assertEqual(u'', out)
+        self.assertEqual(u'', _err)
+        self.assertEqual('HI MOM\n', util.load_file(tmp_file))
+
+    def test_subp_handles_strings(self):
+        """subp can run a string command if shell is True."""
+        tmp_file = self.tmp_path('test.out')
+        cmd = 'echo HI MOM >> {tmp_file}'.format(tmp_file=tmp_file)
+        (out, _err) = util.subp(cmd, shell=True)
+        self.assertEqual(u'', out)
+        self.assertEqual(u'', _err)
+        self.assertEqual('HI MOM\n', util.load_file(tmp_file))
+
     def test_subp_handles_utf8(self):
         # The given bytes contain utf-8 accented characters as seen in e.g.
         # the "deja dup" package in Ubuntu.