Merge lp:~chad.smith/curtin/xenial-sru-1721808 into lp:~curtin-dev/curtin/xenial

[Chad Smith]

Upstream snapshot from Xenial for SRU.

=== modified file 'curtin/__init__.py'
--- curtin/__init__.py	2017-06-12 20:39:06 +0000
+++ curtin/__init__.py	2017-10-06 16:35:22 +0000
@@ -23,6 +23,8 @@
 # can determine which features are supported.  Each entry should have
 # a consistent meaning.
 FEATURES = [
+    # curtin can apply centos networking via centos_apply_network_config
+    'CENTOS_APPLY_NETWORK_CONFIG',
     # install supports the 'network' config version 1
     'NETWORK_CONFIG_V1',
     # reporter supports 'webhook' type
 
=== modified file 'curtin/block/__init__.py'
--- curtin/block/__init__.py	2017-06-12 20:39:06 +0000
+++ curtin/block/__init__.py	2017-10-06 16:35:22 +0000
@@ -19,7 +19,6 @@
 import errno
 import itertools
 import os
-import shlex
 import stat
 import sys
 import tempfile
@@ -204,30 +203,13 @@
         return [path_to_kname(device)]
 
 
-def _shlex_split(str_in):
-    # shlex.split takes a string
-    # but in python2 if input here is a unicode, encode it to a string.
-    # http://stackoverflow.com/questions/2365411/
-    #     python-convert-unicode-to-ascii-without-errors
-    if sys.version_info.major == 2:
-        try:
-            if isinstance(str_in, unicode):
-                str_in = str_in.encode('utf-8')
-        except NameError:
-            pass
-
-        return shlex.split(str_in)
-    else:
-        return shlex.split(str_in)
-
-
 def _lsblock_pairs_to_dict(lines):
     """
     parse lsblock output and convert to dict
     """
     ret = {}
     for line in lines.splitlines():
-        toks = _shlex_split(line)
+        toks = util.shlex_split(line)
         cur = {}
         for tok in toks:
             k, v = tok.split("=", 1)
@@ -468,7 +450,7 @@
     for line in out.splitlines():
         curdev, curdata = line.split(":", 1)
         data[curdev] = dict(tok.split('=', 1)
-                            for tok in _shlex_split(curdata))
+                            for tok in util.shlex_split(curdata))
     return data
 
 
@@ -978,4 +960,71 @@
     else:
         raise ValueError("wipe mode %s not supported" % mode)
 
+
+def storage_config_required_packages(storage_config, mapping):
+    """Read storage configuration dictionary and determine
+       which packages are required for the supplied configuration
+       to function.  Return a list of packaged to install.
+    """
+
+    if not storage_config or not isinstance(storage_config, dict):
+        raise ValueError('Invalid storage configuration.  '
+                         'Must be a dict:\n %s' % storage_config)
+
+    if not mapping or not isinstance(mapping, dict):
+        raise ValueError('Invalid storage mapping.  Must be a dict')
+
+    if 'storage' in storage_config:
+        storage_config = storage_config.get('storage')
+
+    needed_packages = []
+
+    # get reqs by device operation type
+    dev_configs = set(operation['type']
+                      for operation in storage_config['config'])
+
+    for dev_type in dev_configs:
+        if dev_type in mapping:
+            needed_packages.extend(mapping[dev_type])
+
+    # for any format operations, check the fstype and
+    # determine if we need any mkfs tools as well.
+    format_configs = set([operation['fstype']
+                         for operation in storage_config['config']
+                         if operation['type'] == 'format'])
+    for format_type in format_configs:
+        if format_type in mapping:
+            needed_packages.extend(mapping[format_type])
+
+    return needed_packages
+
+
+def detect_required_packages_mapping():
+    """Return a dictionary providing a versioned configuration which maps
+       storage configuration elements to the packages which are required
+       for functionality.
+
+       The mapping key is either a config type value, or an fstype value.
+
+    """
+    version = 1
+    mapping = {
+        version: {
+            'handler': storage_config_required_packages,
+            'mapping': {
+                'bcache': ['bcache-tools'],
+                'btrfs': ['btrfs-tools'],
+                'ext2': ['e2fsprogs'],
+                'ext3': ['e2fsprogs'],
+                'ext4': ['e2fsprogs'],
+                'lvm_partition': ['lvm2'],
+                'lvm_volgroup': ['lvm2'],
+                'raid': ['mdadm'],
+                'xfs': ['xfsprogs']
+            },
+        },
+    }
+    return mapping
+
+
 # vi: ts=4 expandtab syntax=python
 
=== modified file 'curtin/block/iscsi.py'
--- curtin/block/iscsi.py	2017-06-12 20:39:06 +0000
+++ curtin/block/iscsi.py	2017-10-06 16:35:22 +0000
@@ -195,6 +195,15 @@
     return target_nodes_location
 
 
+def restart_iscsi_service():
+    LOG.info('restarting iscsi service')
+    if util.uses_systemd():
+        cmd = ['systemctl', 'reload-or-restart', 'open-iscsi']
+    else:
+        cmd = ['service', 'open-iscsi', 'restart']
+    util.subp(cmd, capture=True)
+
+
 def save_iscsi_config(iscsi_disk):
     state = util.load_command_environment()
     # A nodes directory will be created in the same directory as the
@@ -238,11 +247,35 @@
     return _ISCSI_DISKS
 
 
+def get_iscsi_disks_from_config(cfg):
+    """Parse a curtin storage config and return a list
+       of iscsi disk objects for each configuration present
+    """
+    if not cfg:
+        cfg = {}
+
+    sconfig = cfg.get('storage', {}).get('config', {})
+    if not sconfig:
+        LOG.warning('Configuration dictionary did not contain'
+                    ' a storage configuration')
+        return []
+
+    # Construct IscsiDisk objects for each iscsi volume present
+    iscsi_disks = [IscsiDisk(disk['path']) for disk in sconfig
+                   if disk['type'] == 'disk' and
+                   disk.get('path', "").startswith('iscsi:')]
+    LOG.debug('Found %s iscsi disks in storage config', len(iscsi_disks))
+    return iscsi_disks
+
+
 def disconnect_target_disks(target_root_path=None):
     target_nodes_path = util.target_path(target_root_path, '/etc/iscsi/nodes')
     fails = []
     if os.path.isdir(target_nodes_path):
         for target in os.listdir(target_nodes_path):
+            if target not in iscsiadm_sessions():
+                LOG.debug('iscsi target %s not active, skipping', target)
+                continue
             # conn is "host,port,lun"
             for conn in os.listdir(
                             os.path.sep.join([target_nodes_path, target])):
@@ -254,7 +287,9 @@
                     fails.append(target)
                     LOG.warn("Unable to logout of iSCSI target %s: %s",
                              target, e)
-
+    else:
+        LOG.warning('Skipping disconnect: failed to find iscsi nodes path: %s',
+                    target_nodes_path)
     if fails:
         raise RuntimeError(
             "Unable to logout of iSCSI targets: %s" % ', '.join(fails))
@@ -414,9 +449,15 @@
 
     def disconnect(self):
         if self.target not in iscsiadm_sessions():
+            LOG.warning('Iscsi target %s not in active iscsi sessions',
+                        self.target)
             return
 
-        util.subp(['sync'])
-        iscsiadm_logout(self.target, self.portal)
+        try:
+            util.subp(['sync'])
+            iscsiadm_logout(self.target, self.portal)
+        except util.ProcessExecutionError as e:
+            LOG.warn("Unable to logout of iSCSI target %s from portal %s: %s",
+                     self.target, self.portal, e)
 
 # vi: ts=4 expandtab syntax=python
 
=== modified file 'curtin/block/mdadm.py'
--- curtin/block/mdadm.py	2017-06-12 20:39:06 +0000
+++ curtin/block/mdadm.py	2017-10-06 16:35:22 +0000
@@ -273,7 +273,11 @@
             LOG.debug('%s/sync_max = %s', sync_action, val)
             if val != "idle":
                 LOG.debug("mdadm: setting array sync_action=idle")
-                util.write_file(sync_action, content="idle")
+                try:
+                    util.write_file(sync_action, content="idle")
+                except (IOError, OSError) as e:
+                    LOG.debug("mdadm: (non-fatal) write to %s failed %s",
+                              sync_action, e)
 
             # Setting the sync_{max,min} may can help prevent the array from
             # changing back to 'resync' which may prevent the array from being
@@ -283,11 +287,11 @@
             if val != "0":
                 LOG.debug("mdadm: setting array sync_{min,max}=0")
                 try:
-                    util.write_file(sync_max, content="0")
-                    util.write_file(sync_min, content="0")
-                except IOError:
-                    LOG.warning('mdadm: failed to set sync_{max,min} values')
-                    pass
+                    for sync_file in [sync_max, sync_min]:
+                        util.write_file(sync_file, content="0")
+                except (IOError, OSError) as e:
+                    LOG.debug('mdadm: (non-fatal) write to %s failed %s',
+                              sync_file, e)
 
             # one wonders why this command doesn't do any of the above itself?
             out, err = util.subp(["mdadm", "--manage", "--stop", devpath],
 
=== modified file 'curtin/commands/apply_net.py'
--- curtin/commands/apply_net.py	2017-02-08 22:22:44 +0000
+++ curtin/commands/apply_net.py	2017-10-06 16:35:22 +0000
@@ -21,6 +21,7 @@
 from .. import log
 import curtin.net as net
 import curtin.util as util
+from curtin import config
 from . import populate_one_subcmd
 
 
@@ -89,15 +90,38 @@
         sys.stderr.write(msg + "\n")
         raise Exception(msg)
 
+    passthrough = False
     if network_state:
+        # NB: we cannot support passthrough until curtin can convert from
+        # network_state to network-config yaml
         ns = net.network_state.from_state_file(network_state)
+        raise ValueError('Not Supported; curtin lacks a network_state to '
+                         'network_config converter.')
     elif network_config:
-        ns = net.parse_net_config(network_config)
-
-    net.render_network_state(target=target, network_state=ns)
+        netcfg = config.load_config(network_config)
+
+        # curtin will pass-through the netconfig into the target
+        # for rendering at runtime unless the target OS does not
+        # support NETWORK_CONFIG_V2 feature.
+        LOG.info('Checking cloud-init in target [%s] for network '
+                 'configuration passthrough support.', target)
+        try:
+            passthrough = net.netconfig_passthrough_available(target)
+        except util.ProcessExecutionError:
+            LOG.warning('Failed to determine if passthrough is available')
+
+        if passthrough:
+            LOG.info('Passing network configuration through to target: %s',
+                     target)
+            net.render_netconfig_passthrough(target, netconfig=netcfg)
+        else:
+            ns = net.parse_net_config_data(netcfg.get('network', {}))
+
+    if not passthrough:
+        LOG.info('Rendering network configuration in target')
+        net.render_network_state(target=target, network_state=ns)
 
     _maybe_remove_legacy_eth0(target)
-    LOG.info('Attempting to remove ipv6 privacy extensions')
     _disable_ipv6_privacy_extensions(target)
     _patch_ifupdown_ipv6_mtu_hook(target)
 
@@ -130,6 +154,7 @@
        by default; this races with the cloud-image desire to disable them.
        Resolve this by allowing the cloud-image setting to win. """
 
+    LOG.debug('Attempting to remove ipv6 privacy extensions')
     cfg = util.target_path(target, path=path)
     if not os.path.exists(cfg):
         LOG.warn('Failed to find ipv6 privacy conf file %s', cfg)
@@ -143,7 +168,7 @@
         lines = [f.strip() for f in contents.splitlines()
                  if not f.startswith("#")]
         if lines == known_contents:
-            LOG.info('deleting file: %s', cfg)
+            LOG.info('Removing ipv6 privacy extension config file: %s', cfg)
             util.del_file(cfg)
             msg = "removed %s with known contents" % cfg
             curtin_contents = '\n'.join(
@@ -153,9 +178,10 @@
                  "# net.ipv6.conf.default.use_tempaddr = 2"])
             util.write_file(cfg, curtin_contents)
         else:
-            LOG.info('skipping, content didnt match')
-            LOG.debug("found content:\n%s", lines)
-            LOG.debug("expected contents:\n%s", known_contents)
+            LOG.debug('skipping removal of %s, expected content not found',
+                      cfg)
+            LOG.debug("Found content in file %s:\n%s", cfg, lines)
+            LOG.debug("Expected contents in file %s:\n%s", cfg, known_contents)
             msg = (bmsg + " '%s' exists with user configured content." % cfg)
     except Exception as e:
         msg = bmsg + " %s exists, but could not be read. %s" % (cfg, e)
 
=== modified file 'curtin/commands/apt_config.py'
--- curtin/commands/apt_config.py	2017-03-01 16:13:56 +0000
+++ curtin/commands/apt_config.py	2017-10-06 16:35:22 +0000
@@ -24,7 +24,6 @@
 import os
 import re
 import sys
-import time
 import yaml
 
 from curtin.log import LOG
@@ -406,20 +405,12 @@
         if aa_repo_match(source):
             with util.ChrootableTarget(
                     target, sys_resolvconf=True) as in_chroot:
-                time_entered = time.time()
                 try:
                     in_chroot.subp(["add-apt-repository", source],
                                    retries=(1, 2, 5, 10))
                 except util.ProcessExecutionError:
                     LOG.exception("add-apt-repository failed.")
                     raise
-                finally:
-                    # workaround to gnupg >=2.x spawning daemons (LP: #1645680)
-                    seconds_since = time.time() - time_entered + 1
-                    in_chroot.subp(['killall', '--wait', '--quiet',
-                                    '--younger-than', '%ds' % seconds_since,
-                                    '--regexp', '(dirmngr|gpg-agent)'],
-                                   rcs=[0, 1])
             continue
 
         sourcefn = util.target_path(target, ent['filename'])
 
=== modified file 'curtin/commands/curthooks.py'
--- curtin/commands/curthooks.py	2017-06-12 20:39:06 +0000
+++ curtin/commands/curthooks.py	2017-10-06 16:35:22 +0000
@@ -16,6 +16,7 @@
 #   along with Curtin.  If not, see <http://www.gnu.org/licenses/>.
 
 import copy
+import glob
 import os
 import platform
 import re
@@ -25,6 +26,7 @@
 
 from curtin import config
 from curtin import block
+from curtin import net
 from curtin import futil
 from curtin.log import LOG
 from curtin import swap
@@ -65,28 +67,18 @@
     }
 }
 
-
-def write_files(cfg, target):
-    # this takes 'write_files' entry in config and writes files in the target
-    # config entry example:
-    # f1:
-    #  path: /file1
-    #  content: !!binary |
-    #    f0VMRgIBAQAAAAAAAAAAAAIAPgABAAAAwARAAAAAAABAAAAAAAAAAJAVAAAAAAA
-    # f2: {path: /file2, content: "foobar", permissions: '0666'}
-    if 'write_files' not in cfg:
-        return
-
-    for (key, info) in cfg.get('write_files').items():
-        if not info.get('path'):
-            LOG.warn("Warning, write_files[%s] had no 'path' entry", key)
-            continue
-
-        futil.write_finfo(path=target + os.path.sep + info['path'],
-                          content=info.get('content', ''),
-                          owner=info.get('owner', "-1:-1"),
-                          perms=info.get('permissions',
-                                         info.get('perms', "0644")))
+CLOUD_INIT_YUM_REPO_TEMPLATE = """
+[group_cloud-init-el-stable]
+name=Copr repo for el-stable owned by @cloud-init
+baseurl=https://copr-be.cloud.fedoraproject.org/results/@cloud-init/el-stable/epel-%s-$basearch/
+type=rpm-md
+skip_if_unavailable=True
+gpgcheck=1
+gpgkey=https://copr-be.cloud.fedoraproject.org/results/@cloud-init/el-stable/pubkey.gpg
+repo_gpgcheck=0
+enabled=1
+enabled_metadata=1
+"""
 
 
 def do_apt_config(cfg, target):
@@ -142,15 +134,9 @@
 parameters = root=%s
 
 """ % root_arg
-    zipl_cfg = {
-        "write_files": {
-            "zipl_cfg": {
-                "path": "/etc/zipl.conf",
-                "content": zipl_conf,
-            }
-        }
-    }
-    write_files(zipl_cfg, target)
+    futil.write_files(
+        files={"zipl_conf": {"path": "/etc/zipl.conf", "content": zipl_conf}},
+        base_dir=target)
 
 
 def run_zipl(cfg, target):
@@ -648,6 +634,40 @@
     update_initramfs(target, all_kernels=True)
 
 
+def detect_required_packages(cfg):
+    """
+    detect packages that will be required in-target by custom config items
+    """
+
+    mapping = {
+        'storage': block.detect_required_packages_mapping(),
+        'network': net.detect_required_packages_mapping(),
+    }
+
+    needed_packages = []
+    for cfg_type, cfg_map in mapping.items():
+
+        # skip missing or invalid config items, configs may
+        # only have network or storage, not always both
+        if not isinstance(cfg.get(cfg_type), dict):
+            continue
+
+        cfg_version = cfg[cfg_type].get('version')
+        if not isinstance(cfg_version, int) or cfg_version not in cfg_map:
+            msg = ('Supplied configuration version "%s", for config type'
+                   '"%s" is not present in the known mapping.' % (cfg_version,
+                                                                  cfg_type))
+            raise ValueError(msg)
+
+        mapped_config = cfg_map[cfg_version]
+        found_reqs = mapped_config['handler'](cfg, mapped_config['mapping'])
+        needed_packages.extend(found_reqs)
+
+    LOG.debug('Curtin config dependencies requires additional packages: %s',
+              needed_packages)
+    return needed_packages
+
+
 def install_missing_packages(cfg, target):
     ''' describe which operation types will require specific packages
 
@@ -655,46 +675,10 @@
          'pkg1': ['op_name_1', 'op_name_2', ...]
      }
     '''
-    custom_configs = {
-        'storage': {
-            'lvm2': ['lvm_volgroup', 'lvm_partition'],
-            'mdadm': ['raid'],
-            'bcache-tools': ['bcache']},
-        'network': {
-            'vlan': ['vlan'],
-            'ifenslave': ['bond'],
-            'bridge-utils': ['bridge']},
-    }
-
-    format_configs = {
-        'xfsprogs': ['xfs'],
-        'e2fsprogs': ['ext2', 'ext3', 'ext4'],
-        'btrfs-tools': ['btrfs'],
-    }
-
-    needed_packages = []
+
     installed_packages = util.get_installed_packages(target)
-    for cust_cfg, pkg_reqs in custom_configs.items():
-        if cust_cfg not in cfg:
-            continue
-
-        all_types = set(
-            operation['type']
-            for operation in cfg[cust_cfg]['config']
-            )
-        for pkg, types in pkg_reqs.items():
-            if set(types).intersection(all_types) and \
-               pkg not in installed_packages:
-                needed_packages.append(pkg)
-
-        format_types = set(
-            [operation['fstype']
-             for operation in cfg[cust_cfg]['config']
-             if operation['type'] == 'format'])
-        for pkg, fstypes in format_configs.items():
-            if set(fstypes).intersection(format_types) and \
-               pkg not in installed_packages:
-                needed_packages.append(pkg)
+    needed_packages = set([pkg for pkg in detect_required_packages(cfg)
+                           if pkg not in installed_packages])
 
     arch_packages = {
         's390x': [('s390-tools', 'zipl')],
@@ -703,16 +687,28 @@
     for pkg, cmd in arch_packages.get(platform.machine(), []):
         if not util.which(cmd, target=target):
             if pkg not in needed_packages:
-                needed_packages.append(pkg)
+                needed_packages.add(pkg)
+
+    # FIXME: This needs cleaning up.
+    # do not install certain packages on artful as they are no longer needed.
+    # ifenslave specifically causes issuse due to dependency on ifupdown.
+    codename = util.lsb_release(target=target).get('codename')
+    if codename == 'artful':
+        drops = set(['bridge-utils', 'ifenslave', 'vlan'])
+        if needed_packages.union(drops):
+            LOG.debug("Skipping install of %s.  Not needed on artful.",
+                      needed_packages.union(drops))
+            needed_packages = needed_packages.difference(drops)
 
     if needed_packages:
+        to_add = list(sorted(needed_packages))
         state = util.load_command_environment()
         with events.ReportEventStack(
                 name=state.get('report_stack_prefix'),
                 reporting_enabled=True, level="INFO",
                 description="Installing packages on target system: " +
-                str(needed_packages)):
-            util.install_packages(needed_packages, target=target)
+                str(to_add)):
+            util.install_packages(to_add, target=target)
 
 
 def system_upgrade(cfg, target):
@@ -737,8 +733,8 @@
     util.system_upgrade(target=target)
 
 
-def handle_cloudconfig(cfg, target=None):
-    """write cloud-init configuration files into target
+def handle_cloudconfig(cfg, base_dir=None):
+    """write cloud-init configuration files into base_dir.
 
     cloudconfig format is a dictionary of keys and values of content
 
@@ -773,9 +769,9 @@
         cfgvalue['path'] = cfgpath
 
     # re-use write_files format and adjust target to prepend
-    LOG.debug('Calling write_files with cloudconfig @ %s', target)
+    LOG.debug('Calling write_files with cloudconfig @ %s', base_dir)
     LOG.debug('Injecting cloud-config:\n%s', cfg)
-    write_files({'write_files': cfg}, target)
+    futil.write_files(cfg, base_dir)
 
 
 def ubuntu_core_curthooks(cfg, target=None):
@@ -795,17 +791,98 @@
         if os.path.exists(cloudinit_disable):
             util.del_file(cloudinit_disable)
 
-        handle_cloudconfig(cloudconfig, target=cc_target)
+        handle_cloudconfig(cloudconfig, base_dir=cc_target)
 
     netconfig = cfg.get('network', None)
     if netconfig:
         LOG.info('Writing network configuration')
         ubuntu_core_netconfig = os.path.join(cc_target,
-                                             "50-network-config.cfg")
+                                             "50-curtin-networking.cfg")
         util.write_file(ubuntu_core_netconfig,
                         content=config.dump_config({'network': netconfig}))
 
 
+def rpm_get_dist_id(target):
+    """Use rpm command to extract the '%rhel' distro macro which returns
+       the major os version id (6, 7, 8).  This works for centos or rhel
+    """
+    with util.ChrootableTarget(target) as in_chroot:
+        dist, _ = in_chroot.subp(['rpm', '-E', '%rhel'], capture=True)
+    return dist.rstrip()
+
+
+def centos_apply_network_config(netcfg, target=None):
+    """ CentOS images execute built-in curthooks which only supports
+        simple networking configuration.  This hook enables advanced
+        network configuration via config passthrough to the target.
+    """
+
+    def cloud_init_repo(version):
+        if not version:
+            raise ValueError('Missing required version parameter')
+
+        return CLOUD_INIT_YUM_REPO_TEMPLATE % version
+
+    if netcfg:
+        LOG.info('Removing embedded network configuration (if present)')
+        ifcfgs = glob.glob(util.target_path(target,
+                                            'etc/sysconfig/network-scripts') +
+                           '/ifcfg-*')
+        # remove ifcfg-* (except ifcfg-lo)
+        for ifcfg in ifcfgs:
+            if os.path.basename(ifcfg) != "ifcfg-lo":
+                util.del_file(ifcfg)
+
+        LOG.info('Checking cloud-init in target [%s] for network '
+                 'configuration passthrough support.', target)
+        passthrough = net.netconfig_passthrough_available(target)
+        LOG.debug('passthrough available via in-target: %s', passthrough)
+
+        # if in-target cloud-init is not updated, upgrade via cloud-init repo
+        if not passthrough:
+            cloud_init_yum_repo = (
+                util.target_path(target,
+                                 'etc/yum.repos.d/curtin-cloud-init.repo'))
+            # Inject cloud-init daily yum repo
+            util.write_file(cloud_init_yum_repo,
+                            content=cloud_init_repo(rpm_get_dist_id(target)))
+
+            # we separate the installation of repository packages (epel,
+            # cloud-init-el-release) as we need a new invocation of yum
+            # to read the newly installed repo files.
+            YUM_CMD = ['yum', '-y', '--noplugins', 'install']
+            retries = [1] * 30
+            with util.ChrootableTarget(target) as in_chroot:
+                # ensure up-to-date ca-certificates to handle https mirror
+                # connections
+                in_chroot.subp(YUM_CMD + ['ca-certificates'], capture=True,
+                               log_captured=True, retries=retries)
+                in_chroot.subp(YUM_CMD + ['epel-release'], capture=True,
+                               log_captured=True, retries=retries)
+                in_chroot.subp(YUM_CMD + ['cloud-init-el-release'],
+                               log_captured=True, capture=True,
+                               retries=retries)
+                in_chroot.subp(YUM_CMD + ['cloud-init'], capture=True,
+                               log_captured=True, retries=retries)
+
+            # remove cloud-init el-stable bootstrap repo config as the
+            # cloud-init-el-release package points to the correct repo
+            util.del_file(cloud_init_yum_repo)
+
+            # install bridge-utils if needed
+            with util.ChrootableTarget(target) as in_chroot:
+                try:
+                    in_chroot.subp(['rpm', '-q', 'bridge-utils'],
+                                   capture=False, rcs=[0])
+                except util.ProcessExecutionError:
+                    LOG.debug('Image missing bridge-utils package, installing')
+                    in_chroot.subp(YUM_CMD + ['bridge-utils'], capture=True,
+                                   log_captured=True, retries=retries)
+
+    LOG.info('Passing network configuration through to target')
+    net.render_netconfig_passthrough(target, netconfig={'network': netcfg})
+
+
 def target_is_ubuntu_core(target):
     """Check if Ubuntu-Core specific directory is present at target"""
     if target:
@@ -814,6 +891,22 @@
     return False
 
 
+def target_is_centos(target):
+    """Check if CentOS specific file is present at target"""
+    if target:
+        return os.path.exists(util.target_path(target, 'etc/centos-release'))
+
+    return False
+
+
+def target_is_rhel(target):
+    """Check if RHEL specific file is present at target"""
+    if target:
+        return os.path.exists(util.target_path(target, 'etc/redhat-release'))
+
+    return False
+
+
 def curthooks(args):
     state = util.load_command_environment()
 
@@ -827,14 +920,28 @@
                          "Use --target or set TARGET_MOUNT_POINT\n")
         sys.exit(2)
 
-    # if network-config hook exists in target,
-    # we do not run the builtin
-    if util.run_hook_if_exists(target, 'curtin-hooks'):
-        sys.exit(0)
-
     cfg = config.load_command_config(args, state)
     stack_prefix = state.get('report_stack_prefix', '')
 
+    # if curtin-hooks hook exists in target we can defer to the in-target hooks
+    if util.run_hook_if_exists(target, 'curtin-hooks'):
+        # For vmtests to force execute centos_apply_network_config, uncomment
+        # the value in examples/tests/centos_defaults.yaml
+        if cfg.get('_ammend_centos_curthooks'):
+            if cfg.get('cloudconfig'):
+                handle_cloudconfig(
+                    cfg['cloudconfig'],
+                    base_dir=util.target_path(target, 'etc/cloud/cloud.cfg.d'))
+
+            if target_is_centos(target) or target_is_rhel(target):
+                LOG.info('Detected RHEL/CentOS image, running extra hooks')
+                with events.ReportEventStack(
+                        name=stack_prefix, reporting_enabled=True,
+                        level="INFO",
+                        description="Configuring CentOS for first boot"):
+                    centos_apply_network_config(cfg.get('network', {}), target)
+        sys.exit(0)
+
     if target_is_ubuntu_core(target):
         LOG.info('Detected Ubuntu-Core image, running hooks')
         with events.ReportEventStack(
@@ -846,13 +953,16 @@
     with events.ReportEventStack(
             name=stack_prefix + '/writing-config',
             reporting_enabled=True, level="INFO",
-            description="writing config files and configuring apt"):
-        write_files(cfg, target)
+            description="configuring apt configuring apt"):
         do_apt_config(cfg, target)
         disable_overlayroot(cfg, target)
 
     # packages may be needed prior to installing kernel
-    install_missing_packages(cfg, target)
+    with events.ReportEventStack(
+            name=stack_prefix + '/installing-missing-packages',
+            reporting_enabled=True, level="INFO",
+            description="installing missing packages"):
+        install_missing_packages(cfg, target)
 
     # If a /etc/iscsi/nodes/... file was created by block_meta then it
     # needs to be copied onto the target system
@@ -880,7 +990,6 @@
         setup_zipl(cfg, target)
         install_kernel(cfg, target)
         run_zipl(cfg, target)
-
         restore_dist_interfaces(cfg, target)
 
     with events.ReportEventStack(
@@ -908,12 +1017,6 @@
         detect_and_handle_multipath(cfg, target)
 
     with events.ReportEventStack(
-            name=stack_prefix + '/installing-missing-packages',
-            reporting_enabled=True, level="INFO",
-            description="installing missing packages"):
-        install_missing_packages(cfg, target)
-
-    with events.ReportEventStack(
             name=stack_prefix + '/system-upgrade',
             reporting_enabled=True, level="INFO",
             description="updating packages on target system"):
 
=== modified file 'curtin/commands/extract.py'
--- curtin/commands/extract.py	2016-05-10 16:13:29 +0000
+++ curtin/commands/extract.py	2017-10-06 16:35:22 +0000
@@ -21,6 +21,7 @@
 import curtin.config
 from curtin.log import LOG
 import curtin.util
+from curtin.futil import write_files
 from curtin.reporter import events
 
 from . import populate_one_subcmd
@@ -122,6 +123,11 @@
                     "do not know how to extract '%s'" %
                     source['uri'])
 
+    if cfg.get('write_files'):
+        LOG.info("Applying write_files from config.")
+        write_files(cfg['write_files'], target)
+    else:
+        LOG.info("No write_files in config.")
     sys.exit(0)
 
 
 
=== modified file 'curtin/commands/install.py'
--- curtin/commands/install.py	2017-06-12 20:39:06 +0000
+++ curtin/commands/install.py	2017-10-06 16:35:22 +0000
@@ -366,6 +366,27 @@
         return True
 
 
+def migrate_proxy_settings(cfg):
+    """Move the legacy proxy setting 'http_proxy' into cfg['proxy']."""
+    proxy = cfg.get('proxy', {})
+    if not isinstance(proxy, dict):
+        raise ValueError("'proxy' in config is not a dictionary: %s" % proxy)
+
+    if 'http_proxy' in cfg:
+        hp = cfg['http_proxy']
+        if hp:
+            if proxy.get('http_proxy', hp) != hp:
+                LOG.warn("legacy http_proxy setting (%s) differs from "
+                         "proxy/http_proxy (%s), using %s",
+                         hp, proxy['http_proxy'], proxy['http_proxy'])
+            else:
+                LOG.debug("legacy 'http_proxy' migrated to proxy/http_proxy")
+                proxy['http_proxy'] = hp
+        del cfg['http_proxy']
+
+    cfg['proxy'] = proxy
+
+
 def cmd_install(args):
     cfg = CONFIG_BUILTIN.copy()
     config.merge_config(cfg, args.config)
@@ -384,8 +405,10 @@
         # we default to tgz for old style sources config
         cfg['sources'][i] = util.sanitize_source(cfg['sources'][i])
 
-    if cfg.get('http_proxy'):
-        os.environ['http_proxy'] = cfg['http_proxy']
+    migrate_proxy_settings(cfg)
+    for k in ('http_proxy', 'https_proxy', 'no_proxy'):
+        if k in cfg['proxy']:
+            os.environ[k] = cfg['proxy'][k]
 
     instcfg = cfg.get('install', {})
     logfile = instcfg.get('log_file')
@@ -454,9 +477,26 @@
                                       '/root/curtin-install.log')
         if log_target_path:
             copy_install_log(logfile, workingd.target, log_target_path)
+        # unmount everything (including iscsi disks)
         util.do_umount(workingd.target, recursive=True)
-        # need to do some processing on iscsi disks to disconnect?
-        iscsi.disconnect_target_disks(workingd.target)
+
+        # The open-iscsi service in the ephemeral environment handles
+        # disconnecting active sessions.  On Artful release the systemd
+        # unit file has conditionals that are not met at boot time and
+        # results in open-iscsi service not being started; This breaks
+        # shutdown on Artful releases.
+        # Additionally, in release < Artful, if the storage configuration
+        # is layered, like RAID over iscsi volumes, then disconnecting iscsi
+        # sessions before stopping the raid device hangs.
+        # As it turns out, letting the open-iscsi service take down the
+        # session last is the cleanest way to handle all releases regardless
+        # of what may be layered on top of the iscsi disks.
+        #
+        # Check if storage configuration has iscsi volumes and if so ensure
+        # iscsi service is active before exiting install
+        if iscsi.get_iscsi_disks_from_config(cfg):
+            iscsi.restart_iscsi_service()
+
         shutil.rmtree(workingd.top)
 
     apply_power_state(cfg.get('power_state'))
 
=== modified file 'curtin/futil.py'
--- curtin/futil.py	2014-03-26 17:34:57 +0000
+++ curtin/futil.py	2017-10-06 16:35:22 +0000
@@ -19,7 +19,8 @@
 import pwd
 import os
 
-from .util import write_file
+from .util import write_file, target_path
+from .log import LOG
 
 
 def chownbyid(fname, uid=None, gid=None):
@@ -78,3 +79,25 @@
         omode = "wb"
     write_file(path, content, mode=decode_perms(perms), omode=omode)
     chownbyname(path, u, g)
+
+
+def write_files(files, base_dir=None):
+    """Write files described in the dictionary 'files'
+
+    paths are assumed under 'base_dir', which will default to '/'.
+    A trailing '/' will be applied if not present.
+
+    files is a dictionary where each entry has:
+       path: /file1
+       content: (bytes or string)
+       permissions: (optional, default=0644)
+       owner: (optional, default -1:-1): string of 'uid:gid'."""
+    for (key, info) in files.items():
+        if not info.get('path'):
+            LOG.warn("Warning, write_files[%s] had no 'path' entry", key)
+            continue
+
+        write_finfo(path=target_path(base_dir, info['path']),
+                    content=info.get('content', ''),
+                    owner=info.get('owner', "-1:-1"),
+                    perms=info.get('permissions', info.get('perms', "0644")))
 
=== modified file 'curtin/net/__init__.py'
--- curtin/net/__init__.py	2017-03-01 16:13:56 +0000
+++ curtin/net/__init__.py	2017-10-06 16:35:22 +0000
@@ -520,7 +520,52 @@
     return content
 
 
+def netconfig_passthrough_available(target, feature='NETWORK_CONFIG_V2'):
+    """
+    Determine if curtin can pass v2 network config to in target cloud-init
+    """
+    LOG.debug('Checking in-target cloud-init for feature: %s', feature)
+    with util.ChrootableTarget(target) as in_chroot:
+
+        cloudinit = util.which('cloud-init', target=target)
+        if not cloudinit:
+            LOG.warning('Target does not have cloud-init installed')
+            return False
+
+        available = False
+        try:
+            out, _ = in_chroot.subp([cloudinit, 'features'], capture=True)
+            available = feature in out.splitlines()
+        except util.ProcessExecutionError:
+            # we explicitly don't dump the exception as this triggers
+            # vmtest failures when parsing the installation log file
+            LOG.warning("Failed to probe cloudinit features")
+            return False
+
+        LOG.debug('cloud-init feature %s available? %s', feature, available)
+        return available
+
+
+def render_netconfig_passthrough(target, netconfig=None):
+    """
+    Extract original network config and pass it
+    through to cloud-init in target
+    """
+    cc = 'etc/cloud/cloud.cfg.d/50-curtin-networking.cfg'
+    if not isinstance(netconfig, dict):
+        raise ValueError('Network config must be a dictionary')
+
+    if 'network' not in netconfig:
+        raise ValueError("Network config must contain the key 'network'")
+
+    content = config.dump_config(netconfig)
+    cc_passthrough = os.path.sep.join((target, cc,))
+    LOG.info('Writing network config to %s: %s', cc, cc_passthrough)
+    util.write_file(cc_passthrough, content=content)
+
+
 def render_network_state(target, network_state):
+    LOG.debug("rendering eni from netconfig")
     eni = 'etc/network/interfaces'
     netrules = 'etc/udev/rules.d/70-persistent-net.rules'
     cc = 'etc/cloud/cloud.cfg.d/curtin-disable-cloudinit-networking.cfg'
@@ -542,4 +587,65 @@
     """Returns the string value of an interface's MAC Address"""
     return read_sys_net(ifname, "address", enoent=False)
 
+
+def network_config_required_packages(network_config, mapping=None):
+
+    if network_config is None:
+        network_config = {}
+
+    if not isinstance(network_config, dict):
+        raise ValueError('Invalid network configuration.  Must be a dict')
+
+    if mapping is None:
+        mapping = {}
+
+    if not isinstance(mapping, dict):
+        raise ValueError('Invalid network mapping.  Must be a dict')
+
+    # allow top-level 'network' key
+    if 'network' in network_config:
+        network_config = network_config.get('network')
+
+    # v1 has 'config' key and uses type: devtype elements
+    if 'config' in network_config:
+        dev_configs = set(device['type']
+                          for device in network_config['config'])
+    else:
+        # v2 has no config key
+        dev_configs = set(cfgtype for (cfgtype, cfg) in
+                          network_config.items() if cfgtype not in ['version'])
+
+    needed_packages = []
+    for dev_type in dev_configs:
+        if dev_type in mapping:
+            needed_packages.extend(mapping[dev_type])
+
+    return needed_packages
+
+
+def detect_required_packages_mapping():
+    """Return a dictionary providing a versioned configuration which maps
+       network configuration elements to the packages which are required
+       for functionality.
+    """
+    mapping = {
+        1: {
+            'handler': network_config_required_packages,
+            'mapping': {
+                'bond': ['ifenslave'],
+                'bridge': ['bridge-utils'],
+                'vlan': ['vlan']},
+        },
+        2: {
+            'handler': network_config_required_packages,
+            'mapping': {
+                'bonds': ['ifenslave'],
+                'bridges': ['bridge-utils'],
+                'vlans': ['vlan']}
+        },
+    }
+
+    return mapping
+
+
 # vi: ts=4 expandtab syntax=python
 
=== modified file 'curtin/reporter/handlers.py'
--- curtin/reporter/handlers.py	2017-02-08 22:22:44 +0000
+++ curtin/reporter/handlers.py	2017-10-06 16:35:22 +0000
@@ -80,7 +80,49 @@
             LOG.warn("failed posting event: %s [%s]" % (event.as_string(), e))
 
 
+class JournaldHandler(ReportingHandler):
+
+    def __init__(self, level="DEBUG", identifier="curtin_event"):
+        super(JournaldHandler, self).__init__()
+        if isinstance(level, int):
+            pass
+        else:
+            input_level = level
+            try:
+                level = getattr(logging, level.upper())
+            except Exception:
+                LOG.warn("invalid level '%s', using WARN", input_level)
+                level = logging.WARN
+        self.level = level
+        self.identifier = identifier
+
+    def publish_event(self, event):
+        # Ubuntu older than precise will not have python-systemd installed.
+        try:
+            from systemd import journal
+        except ImportError:
+            raise
+        level = str(getattr(journal, "LOG_" + event.level, journal.LOG_DEBUG))
+        extra = {}
+        if hasattr(event, 'result'):
+            extra['CURTIN_RESULT'] = event.result
+        journal.send(
+            event.as_string(),
+            PRIORITY=level,
+            SYSLOG_IDENTIFIER=self.identifier,
+            CURTIN_EVENT_TYPE=event.event_type,
+            CURTIN_MESSAGE=event.description,
+            CURTIN_NAME=event.name,
+            **extra
+            )
+
+
 available_handlers = DictRegistry()
 available_handlers.register_item('log', LogHandler)
 available_handlers.register_item('print', PrintHandler)
 available_handlers.register_item('webhook', WebHookHandler)
+# only add journald handler on systemd systems
+try:
+    available_handlers.register_item('journald', JournaldHandler)
+except ImportError:
+    print('journald report handler not supported; no systemd module')
 
=== modified file 'curtin/util.py'
--- curtin/util.py	2017-06-12 20:39:06 +0000
+++ curtin/util.py	2017-10-06 16:35:22 +0000
@@ -23,6 +23,7 @@
 import os
 import platform
 import re
+import shlex
 import shutil
 import socket
 import subprocess
@@ -57,6 +58,8 @@
 _INSTALLED_MAIN = '/usr/bin/curtin'
 
 _LSB_RELEASE = {}
+_USES_SYSTEMD = None
+_HAS_UNSHARE_PID = None
 
 _DNS_REDIRECT_IP = None
 
@@ -66,21 +69,31 @@
 
 def _subp(args, data=None, rcs=None, env=None, capture=False,
           shell=False, logstring=False, decode="replace",
-          target=None, cwd=None, log_captured=False):
+          target=None, cwd=None, log_captured=False, unshare_pid=None):
     if rcs is None:
         rcs = [0]
-
     devnull_fp = None
-    try:
-        if target_path(target) != "/":
-            args = ['chroot', target] + list(args)
-
-        if not logstring:
-            LOG.debug(("Running command %s with allowed return codes %s"
-                       " (shell=%s, capture=%s)"), args, rcs, shell, capture)
-        else:
-            LOG.debug(("Running hidden command to protect sensitive "
-                       "input/output logstring: %s"), logstring)
+
+    tpath = target_path(target)
+    chroot_args = [] if tpath == "/" else ['chroot', target]
+    sh_args = ['sh', '-c'] if shell else []
+    if isinstance(args, string_types):
+        args = [args]
+
+    try:
+        unshare_args = _get_unshare_pid_args(unshare_pid, tpath)
+    except RuntimeError as e:
+        raise RuntimeError("Unable to unshare pid (cmd=%s): %s" % (args, e))
+
+    args = unshare_args + chroot_args + sh_args + list(args)
+
+    if not logstring:
+        LOG.debug(("Running command %s with allowed return codes %s"
+                   " (capture=%s)"), args, rcs, capture)
+    else:
+        LOG.debug(("Running hidden command to protect sensitive "
+                   "input/output logstring: %s"), logstring)
+    try:
         stdin = None
         stdout = None
         stderr = None
@@ -94,7 +107,7 @@
             stdin = subprocess.PIPE
         sp = subprocess.Popen(args, stdout=stdout,
                               stderr=stderr, stdin=stdin,
-                              env=env, shell=shell, cwd=cwd)
+                              env=env, shell=False, cwd=cwd)
         # communicate in python2 returns str, python3 returns bytes
         (out, err) = sp.communicate(data)
 
@@ -128,6 +141,63 @@
     return (out, err)
 
 
+def _has_unshare_pid():
+    global _HAS_UNSHARE_PID
+    if _HAS_UNSHARE_PID is not None:
+        return _HAS_UNSHARE_PID
+
+    if not which('unshare'):
+        _HAS_UNSHARE_PID = False
+        return False
+    out, err = subp(["unshare", "--help"], capture=True, decode=False,
+                    unshare_pid=False)
+    joined = b'\n'.join([out, err])
+    _HAS_UNSHARE_PID = b'--fork' in joined and b'--pid' in joined
+    return _HAS_UNSHARE_PID
+
+
+def _get_unshare_pid_args(unshare_pid=None, target=None, euid=None):
+    """Get args for calling unshare for a pid.
+
+    If unshare_pid is False, return empty list.
+    If unshare_pid is True, check if it is usable.  If not, raise exception.
+    if unshare_pid is None, then unshare if
+       * euid is 0
+       * 'unshare' with '--fork' and '--pid' is available.
+       * target != /
+    """
+    if unshare_pid is not None and not unshare_pid:
+        # given a false-ish other than None means no.
+        return []
+
+    if euid is None:
+        euid = os.geteuid()
+
+    tpath = target_path(target)
+
+    unshare_pid_in = unshare_pid
+    if unshare_pid is None:
+        unshare_pid = False
+        if tpath != "/" and euid == 0:
+            if _has_unshare_pid():
+                unshare_pid = True
+
+    if not unshare_pid:
+        return []
+
+    # either unshare was passed in as True, or None and turned to True.
+    if euid != 0:
+        raise RuntimeError(
+            "given unshare_pid=%s but euid (%s) != 0." %
+            (unshare_pid_in, euid))
+
+    if not _has_unshare_pid():
+        raise RuntimeError(
+            "given unshare_pid=%s but no unshare command." % unshare_pid_in)
+
+    return ['unshare', '--fork', '--pid', '--']
+
+
 def subp(*args, **kwargs):
     """Run a subprocess.
 
@@ -160,6 +230,10 @@
         means to run, sleep 1, run, sleep 3, run and then return exit code.
     :param target:
         run the command as 'chroot target <args>'
+    :param unshare_pid:
+        unshare the pid namespace.
+        default value (None) is to unshare pid namespace if possible
+        and target != /
 
     :return
         if not capturing, return is (None, None)
@@ -1275,6 +1349,9 @@
     if not path:
         return target
 
+    if not isinstance(path, string_types):
+        raise ValueError("Unexpected input for path: %s" % path)
+
     # os.path.join("/etc", "/foo") returns "/foo". Chomp all leading /.
     while len(path) and path[0] == "/":
         path = path[1:]
@@ -1290,4 +1367,51 @@
     __call__ = ChrootableTarget.subp
 
 
+def shlex_split(str_in):
+    # shlex.split takes a string
+    # but in python2 if input here is a unicode, encode it to a string.
+    # http://stackoverflow.com/questions/2365411/
+    #     python-convert-unicode-to-ascii-without-errors
+    if sys.version_info.major == 2:
+        try:
+            if isinstance(str_in, unicode):
+                str_in = str_in.encode('utf-8')
+        except NameError:
+            pass
+
+        return shlex.split(str_in)
+    else:
+        return shlex.split(str_in)
+
+
+def load_shell_content(content, add_empty=False, empty_val=None):
+    """Given shell like syntax (key=value\nkey2=value2\n) in content
+       return the data in dictionary form.  If 'add_empty' is True
+       then add entries in to the returned dictionary for 'VAR='
+       variables.  Set their value to empty_val."""
+
+    data = {}
+    for line in shlex_split(content):
+        key, value = line.split("=", 1)
+        if not value:
+            value = empty_val
+        if add_empty or value:
+            data[key] = value
+
+    return data
+
+
+def uses_systemd():
+    """ Check if current enviroment uses systemd by testing if
+        /run/systemd/system is a directory; only present if
+        systemd is available on running system.
+    """
+
+    global _USES_SYSTEMD
+    if _USES_SYSTEMD is None:
+        _USES_SYSTEMD = os.path.isdir('/run/systemd/system')
+
+    return _USES_SYSTEMD
+
+
 # vi: ts=4 expandtab syntax=python
 
=== modified file 'debian/changelog'
--- debian/changelog	2017-06-12 20:52:38 +0000
+++ debian/changelog	2017-10-06 16:35:22 +0000
@@ -1,3 +1,36 @@
+curtin (0.1.0~bzr532-0ubuntu1~16.04.1) xenial; urgency=medium
+
+  * New upstream snapshot. (LP: #1721808)
+    - vmtest: fix artful networking
+    - docs: Trivial doc fix for enabling proposed.
+    - setup.py: fix to allow installation into a virtualenv
+    - doc: update documentation on curtin-hooks and non-ubuntu installation.
+    - reporter: Add journald reporter to send events to journald
+    - vmtests: add option to tar disk images after test run
+    - install: ensure iscsi service is running to handle shutdown properly
+    - mdadm: handle write failures to sysfs entries when stopping mdadm
+    - vmtest: catch exceptions in curtin-log-print
+    - iscsi: use curtin storage config to disconnect iscsi targets
+    - vmtests: bump skip_by_date values out to give cloud-init SRU more time
+    - vmtest: get info about collected symlinks and then delete them.
+    - Update network cloud-init related skiptest dates, SRU still pending
+    - tests: Add CiTestCase common parent for all curtin tests.
+    - vmtests: Remove force flag for centos curthooks
+    - tools/jenkins-runner: improve tgtd cleanup logic
+    - tests: Drop EOL Wily Vivid and Yakkety tests.
+    - Disable yum plugins when installing packages, update ca-certs for https
+    - Rename centos_network_curthooks -> centos_apply_network_config.
+    - tests: in centos_defaults use write_files for grub serial.
+    - write_files: write files after extract, change write_files signature.
+    - pass network configuration through to target for ubuntu and centos
+    - tests: disable yakkety tests.
+    - tools/launch: automatically pass on proxy settings to curtin
+    - Add top level 'proxy' to config, deprecate top level http_proxy.
+    - tools/curtainer: fix to enable deb-src for -proposed.
+    - Use unshare to put chroot commands in own pid namespace.
+
+ -- Chad Smith <[e-mail]>  Fri, 06 Oct 2017 10:07:36 -0600
+
 curtin (0.1.0~bzr505-0ubuntu1~16.04.1) xenial-proposed; urgency=medium
 
   * debian/new-upstream-snapshot: create tarball in .. otherwise it
 
=== modified file 'doc/index.rst'
--- doc/index.rst	2016-10-03 18:42:29 +0000
+++ doc/index.rst	2017-10-06 16:35:22 +0000
@@ -17,6 +17,7 @@
    topics/apt_source
    topics/networking
    topics/storage
+   topics/curthooks
    topics/reporting
    topics/development
    topics/integration-testing
 
=== modified file 'doc/topics/apt_source.rst'
--- doc/topics/apt_source.rst	2016-10-03 18:42:29 +0000
+++ doc/topics/apt_source.rst	2017-10-06 16:35:22 +0000
@@ -135,7 +135,9 @@
 
  apt:
    sources:
-     proposed.list: deb $MIRROR $RELEASE-proposed main restricted universe multiverse
+     proposed.list:
+       source: |
+         deb $MIRROR $RELEASE-proposed main restricted universe multiverse
 
 * Make debug symbols available
 
@@ -143,11 +145,12 @@
 
  apt:
    sources:
-     ddebs.list: |
-       deb http://ddebs.ubuntu.com $RELEASE main restricted universe multiverse
-       deb http://ddebs.ubuntu.com $RELEASE-updates main restricted universe multiverse
-       deb http://ddebs.ubuntu.com $RELEASE-security main restricted universe multiverse
-       deb http://ddebs.ubuntu.com $RELEASE-proposed main restricted universe multiverse
+     ddebs.list:
+       source: |
+         deb http://ddebs.ubuntu.com $RELEASE main restricted universe multiverse
+         deb http://ddebs.ubuntu.com $RELEASE-updates main restricted universe multiverse
+         deb http://ddebs.ubuntu.com $RELEASE-security main restricted universe multiverse
+         deb http://ddebs.ubuntu.com $RELEASE-proposed main restricted universe multiverse
 
 Timing
 ~~~~~~
 
=== modified file 'doc/topics/config.rst'
--- doc/topics/config.rst	2016-10-03 18:42:29 +0000
+++ doc/topics/config.rst	2017-10-06 16:35:22 +0000
@@ -24,6 +24,7 @@
 - multipath (``multipath``)
 - network (``network``)
 - power_state (``power_state``)
+- proxy (``proxy``)
 - reporting (``reporting``)
 - restore_dist_interfaces: (``restore_dist_interfaces``)
 - sources (``sources``)
@@ -177,6 +178,7 @@
 http_proxy
 ~~~~~~~~~~
 Curtin will export ``http_proxy`` value into the installer environment.
+**Deprecated**: This setting is deprecated in favor of ``proxy`` below.
 
 **http_proxy**: *<HTTP Proxy URL>*
 
@@ -348,6 +350,22 @@
     message: Bye Bye
 
 
+proxy
+~~~~~
+Curtin will put ``http_proxy``, ``https_proxy`` and ``no_proxy``
+into its install environment.  This is in affect for curtin's process
+and subprocesses.
+
+**proxy**: A dictionary containing http_proxy, https_proxy, and no_proxy.
+
+**Example**::
+
+  proxy:
+    http_proxy: http://squid.proxy:3728/
+    https_proxy: http://squid.proxy:3728/
+    no_proxy: localhost,127.0.0.1,10.0.2.1
+
+
 reporting
 ~~~~~~~~~
 Configure installation reporting (see Reporting section for details).
 
=== added file 'doc/topics/curthooks.rst'
--- doc/topics/curthooks.rst	1970-01-01 00:00:00 +0000
+++ doc/topics/curthooks.rst	2017-10-06 16:35:22 +0000
@@ -0,0 +1,109 @@
+========================================
+Curthooks / New OS Support 
+========================================
+Curtin has built-in support for installation of Ubuntu.
+Other operating systems are supported through a mechanism called
+'curthooks' or 'curtin-hooks'.
+
+A curtin install runs through different stages.  See the 
+:ref:`Stages <stages>`
+documentation for function of each stage.
+The stages communicate with each other via data in a working directory and
+environment variables as described in
+:ref:`Command Environment`.
+
+Curtin handles partitioning, filesystem creation and target filesystem
+population for all operating systems. Curthooks are the mechanism provided
+so that the operating system can customize itself before reboot. This
+customization typically would need to include:
+
+ - ensuring that appropriate device drivers are loaded on first boot
+ - consuming the network interfaces file and applying its declarations.
+ - ensuring that necessary packages are installed to utilize storage
+   configuration or networking configuration.
+ - making the system boot (running grub-install or equivalent).
+
+Image provided curtin-hooks
+---------------------------
+An image provides curtin hooks support by containing a file
+``/curtin/curtin-hooks``.
+
+If an Ubuntu image image contains this path it will override the builtin
+curtin support.
+
+The ``curtin-hooks`` program should be executable in the filesystem and
+will be executed without any arguments.  It will be executed in the install
+environment, *not* the target environment.  A change of root to the
+target environment can be done with ``curtin in-target``.
+
+The hook is provided with some environment variables that can be used
+to find more information.  See the :ref:`Command Environment` doc for
+details.  Specifically interesting to this stage are:
+
+ - ``OUTPUT_NETWORK_CONFIG``: This is a path to the file created during
+   network discovery stage. 
+ - ``OUTPUT_FSTAB``: This is a path to the file created during partitioning
+   stage.
+ - ``CONFIG``: This is a path to the curtin config file.  It is provided so
+   that additional configuration could be provided through to the OS
+   customization.
+
+.. **TODO**: We should add 'PYTHON' or 'CURTIN_PYTHON' to this environment
+   so that the hook can easily run a python program with the same python
+   that curtin ran with (ie, python2 or python3).
+
+
+Networking configuration
+------------------------
+Access to the network configuration that is desired is inside the config
+and is in the format described in :ref:`networking`.
+
+.. TODO: We should guarantee that the presence
+         of network config v1 in the file OUTPUT_NETWORK_CONFIG.
+
+The curtin-hooks program must read the configuration from the
+path contained in ``OUTPUT_NETWORK_CONFIG`` and then set up
+the installed system to use it.
+
+If the installed system has cloud-init at version 17.1 or higher, it may
+be possible to simply copy this section into the target in
+``/etc/cloud/cloud.cfg.d/`` and let cloud-init render the correct
+networking on first boot.
+
+Storage configuration
+---------------------
+Access to the storage configuration that was set up is inside the config
+and is in the format described in :ref:`storage`.
+
+.. TODO: We should guarantee that the presence
+         of storage config v1 in the file OUTPUT_STORAGE_CONFIG.
+         This would mean the user would not have to pull it out
+         of CONFIG.  We should guarantee its presence and format
+         even in the 'simple' path.
+
+To apply this storage configuration, the curthooks may need to:
+
+ * update /etc/fstab to add the expected mounts entries.  The environment
+   variable ``OUTPUT_FSTAB`` contains a path to a file that may be suitable
+   for use.
+
+ * install any packages that are not already installed that are required
+   to boot with the provided storage config.  For example, if the storage
+   layout includes raid you may need to install the mdadm package.
+
+ * update or create an initramfs.
+
+
+System boot
+-----------
+In Ubuntu, curtin will run 'grub-setup' and to install grub.  This covers
+putting the bootloader onto the disk(s) that are marked as
+``grub_device``.  The provided hook will need to do the equivalent
+operation.
+
+finalize hook
+-------------
+There is one other hook that curtin will invoke in an install, called
+``finalize``.  This program is invoked in the same environment as
+``curtin-hooks`` above.  It is intended to give the OS a final opportunity
+make updates before reboot.  It is called before ``late_commands``.
 
=== modified file 'doc/topics/integration-testing.rst'
--- doc/topics/integration-testing.rst	2017-06-12 20:39:06 +0000
+++ doc/topics/integration-testing.rst	2017-10-06 16:35:22 +0000
@@ -161,6 +161,12 @@
     - ``logs``: install and boot logs
     - ``collect``: data collected by the boot phase
 
+- ``CURTIN_VMTEST_TAR_DISKS``: default 0
+
+  Vmtest writes out disk image files sparsely into a disks directory
+  If this flag is set to a non-zero number, vmtest will tar all disks in
+  the directory into a single disks.tar and remove the sparse disk files.
+
 - ``CURTIN_VMTEST_TOPDIR``: default $TMPDIR/vmtest-<timestamp>
 
   Vmtest puts all test data under this value.  By default, it creates
 
=== modified file 'doc/topics/networking.rst'
--- doc/topics/networking.rst	2016-10-03 18:42:29 +0000
+++ doc/topics/networking.rst	2017-10-06 16:35:22 +0000
@@ -1,3 +1,5 @@
+.. _networking:
+
 ==========
 Networking
 ==========
 
=== modified file 'doc/topics/overview.rst'
--- doc/topics/overview.rst	2016-10-03 18:42:29 +0000
+++ doc/topics/overview.rst	2017-10-06 16:35:22 +0000
@@ -4,6 +4,8 @@
 
 Curtin is intended to be a bare bones "installer".   Its goal is to take data from a source, and get it onto disk as quick as possible and then boot it.  The key difference from traditional package based installers is that curtin assumes the thing its installing is intelligent and will do the right thing.
 
+.. _Stages:
+
 Stages
 ------
 A usage of curtin will go through the following stages:
@@ -22,6 +24,32 @@
 
 Curtin's assumption is that a fairly rich Linux (Ubuntu) environment is booted.
 
+.. _Command Environment:
+
+Command Environment
+~~~~~~~~~~~~~~~~~~~
+Stages and commands invoked by curtin always have the following environment
+variables defined.
+
+- ``WORKING_DIR``: This is for inter-command state.  It will be the same
+  directory for each command run and will only be deleted at the end of the
+  install. Files referenced in other environment variables will be in
+  this directory.
+
+- ``TARGET_MOUNT_POINT``: The path in the filesystem where the target
+  filesystem will be mounted.
+
+- ``OUTPUT_NETWORK_CONFIG``: After the network discovery stage, this file
+  should contain networking config information that should then be written
+  to the target.
+
+- ``OUTPUT_FSTAB``: After partitioning and filesystem creation, this file
+  will contain fstab(5) style content representing mounts.
+
+- ``CONFIG``: This variable contains a path to a yaml formatted file with
+  the fully rendered config.
+
+
 Early Commands
 ~~~~~~~~~~~~~~
 Early commands are executed on the system, and non-zero exit status will terminate the installation process.  These commands are intended to be used for things like
@@ -48,32 +76,23 @@
   10_wipe_filesystems: curtin wipe --quick --all-unused-disks
   50_setup_raid: curtin disk-setup --all-disks raid0 /
 
-**Command environment**
-
-Partitioning commands have the following environment variables available to them:
-
-- ``WORKING_DIR``: This is simply for some sort of inter-command state.  It will be the same directory for each command run and will only be deleted at the end of all partitioning_commands.
-- ``OUTPUT_FSTAB``: This is the target path for a fstab file.  After all partitioning commands have been run, a file should exist, formatted per fstab(5) that describes how the filesystems should be mounted.
-- ``TARGET_MOUNT_POINT``:
-
-
-Network Discovery and Setup
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Networking is done in a similar fashion to partitioning.  A series of commands, specified in the config are run.  At the end of these commands, a interfaces(5) style file is expected to be written to ``OUTPUT_INTERFACES``.
-
-Note, that as with fstab, this file is not copied verbatim to the target filesystem, but rather made available to the OS customization stage.  That stage may just copy the file verbatim, but may also parse it, and use that as input.
-
-**Config Example**::
-
- network_commands:
-  10_netconf: curtin network copy-existing
-
-**Command environment**
-
-Networking commands have the following environment variables available to them:
-
-- ``WORKING_DIR``: This is simply for some sort of inter-command state.  It will be the same directory for each command run and will only be deleted at the end of all network_commands.
-- ``OUTPUT_INTERFACES``: This is the target path for an interfaces style file. After all commands have been run, a file should exist, formatted per interfaces(5) that describes the systems network setup.
+
+Network Discovery
+~~~~~~~~~~~~~~~~~
+Networking configuration is *discovered* in the 'network' stage.
+The default command run at this stage is ``curtin net-meta auto``.  After
+execution, it will write the discovered networking to the file specified
+in the environment variable ``OUTPUT_NETWORK_CONFIG``.  The format of this
+file is as described in :ref:`networking`.
+
+If curtin's config has a network section, the net-meta will simply parrot the
+data to the output file.  If there is no network section, then its default
+behavior is to copy existing config from the running environment.
+
+Note, that as with fstab, this file is not copied verbatim to the target
+filesystem, but rather made available to the OS customization stage.  That
+stage may just copy the file verbatim, but may also parse it, and apply the
+settings.
 
 Extraction of sources
 ~~~~~~~~~~~~~~~~~~~~~
@@ -88,27 +107,6 @@
 
  wget $URL | tar -Sxvzf 
 
-Hook for installed OS to customize itself
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-After extraction of sources, the source that was extracted is then given a chance to customize itself for the system.  This customization may include:
- - ensuring that appropriate device drivers are loaded on first boot
- - consuming the network interfaces file and applying its declarations.
- - ensuring that necessary packages 
-
-**Config Example**::
-
- config_hook: {{TARGET_MP}}/opt/curtin/config-hook
-
-**Command environment**
- - ``INTERFACES``: This is a path to the file created during networking stage
- - ``FSTAB``: This is a path to the file created during partitioning stage
- - ``CONFIG``: This is a path to the curtin config file.  It is provided so that additional configuration could be provided through to the OS customization.
-
-**Helpers**
-
-Curtin provides some helpers to make the OS customization easier.
- - `curtin in-target`: run the command while chrooted into the target.
-
 Final Commands
 ~~~~~~~~~~~~~~
 
 
=== modified file 'doc/topics/reporting.rst'
--- doc/topics/reporting.rst	2016-10-03 18:42:29 +0000
+++ doc/topics/reporting.rst	2017-10-06 16:35:22 +0000
@@ -10,6 +10,7 @@
 Reporting consists of notification of a series of 'events.  Each event has:
  - **event_type**: 'start' or 'finish'
  - **description**: human readable text
+ - **level**: the log level of the event, DEBUG/INFO/WARN etc.
  - **name**: and id for this event
  - **result**: only present when event_type is 'finish', its value is one of "SUCCESS", "WARN", or "FAIL".  A result of WARN indicates something is likely wrong, but a non-fatal error.  A result of "FAIL" is fatal.
  - **origin**: literal value 'curtin'
@@ -75,6 +76,34 @@
 is specified then all messages with a lower priority than specified will be
 ignored. Default is INFO.
 
+Journald Reporter
+-----------------
+
+The journald reporter sends the events to systemd's `journald`_.  To enable,
+provide curtin with config like::
+
+  reporting:
+    mylistener:
+      type: journald
+      identifier: "my_identifier"
+      level: DEBUG
+
+The event's fields are mapped to fields of the resulting journal entry
+as follows:
+
+- **description** maps to **CURTIN_MESSAGE**
+- **level** maps to **PRIORITY**
+- **name** maps to **CURTIN_NAME**
+- **event_type** maps to **CURTIN_EVENT_TYPE**
+- **result**, if present, maps to **CURTIN_RESULT**
+
+The configured `identifier`, which defaults to "curtin_event", becomes
+the entry's **SYSLOG_IDENTIFIER**.
+
+The python-systemd package must be installed to use this handler.
+
+.. _`journald`: https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html
+
 Example Events
 ~~~~~~~~~~~~~~
 The following is an example event that would be posted::
 
=== modified file 'doc/topics/storage.rst'
--- doc/topics/storage.rst	2017-06-12 20:39:06 +0000
+++ doc/topics/storage.rst	2017-10-06 16:35:22 +0000
@@ -1,3 +1,5 @@
+.. _storage:
+
 =======
 Storage
 =======
 
=== modified file 'examples/network-ipv6-bond-vlan.yaml'
--- examples/network-ipv6-bond-vlan.yaml	2016-10-03 18:42:29 +0000
+++ examples/network-ipv6-bond-vlan.yaml	2017-10-06 16:35:22 +0000
@@ -3,10 +3,10 @@
   config:
     - name: interface0
       type: physical
-      mac_address: BC:76:4E:06:96:B3
+      mac_address: bc:76:4e:06:96:b3
     - name: interface1
       type: physical
-      mac_address: BC:76:4E:04:88:41
+      mac_address: bc:76:4e:04:88:41
     - type: bond
       bond_interfaces:
         - interface0
 
=== modified file 'examples/tests/bonding_network.yaml'
--- examples/tests/bonding_network.yaml	2016-10-03 18:00:41 +0000
+++ examples/tests/bonding_network.yaml	2017-10-06 16:35:22 +0000
@@ -16,8 +16,7 @@
           mac_address: "52:54:00:12:34:04"
         # Bond.
         - type: bond
-          name: bond0
-          mac_address: "52:54:00:12:34:06"
+          name: bond1
           bond_interfaces:
             - interface1
             - interface2
@@ -26,8 +25,6 @@
           subnets:
               - type: static
                 address: 10.23.23.2/24
-              - type: static
-                address: 10.23.24.2/24
 
 curthooks_commands:
     # use curtin to disable open-iscsi ifupdown hooks for precise; they're
 
=== modified file 'examples/tests/centos_basic.yaml'
--- examples/tests/centos_basic.yaml	2017-01-18 16:01:35 +0000
+++ examples/tests/centos_basic.yaml	2017-10-06 16:35:22 +0000
@@ -9,5 +9,6 @@
           mac_address: "52:54:00:12:34:00"
           subnets:
               - type: static
-                address: 10.0.2.15/24
+                address: 10.0.2.15
+                netmask: 255.255.255.0
                 gateway: 10.0.2.2
 
=== added file 'examples/tests/centos_defaults.yaml'
--- examples/tests/centos_defaults.yaml	1970-01-01 00:00:00 +0000
+++ examples/tests/centos_defaults.yaml	2017-10-06 16:35:22 +0000
@@ -0,0 +1,91 @@
+hook_commands:
+    builtin: null
+
+# To force curtin to run centos_apply_network_config vmtest, uncomment
+# _ammend_centos_curthooks: True
+
+write_files:
+  grub_serial_console:
+    path: '/root/curtin-send-console-to-serial'
+    permissions: '0755'
+    owner: 'root:root'
+    content: |
+      # update grub1 and grub2 configs to write to serial console.
+      CONPARM="console=ttyS0,115200"
+      grub1conf="/boot/grub/grub.conf"
+      grub2conf="/boot/grub2/grub.cfg"
+      grub2def="/etc/default/grub"
+
+      rerror() { perror "$?" "$@"; return $r; }
+      perror() { local r="$1"; shift; error "$@"; return $r; }
+      error() { echo "GRUB_SERIAL:" "ERROR:" "$@" 1>&2; }
+      info() { echo "GRUB_SERIAL:" "$@" 1>&2; }
+      fail() { error "$@"; exit 1; }
+      bk() {
+          local ofile="$1" bk="$1.dist.curtin"
+          shift
+          [ -e "$ofile" ] || return 0
+          cp "$ofile" "$bk" || rerror "failed backup ($ofile -> $bk):" "$@";
+      }
+      
+      update_grub1() {
+          local cfg="$1" r=""
+          [ -e "$cfg" ] ||
+              { info "no grub1 cfg '$cfg'"; return 0; }
+          bk "$cfg" "grub1 config" || return
+          if ! grep "^serial" "$cfg"; then
+              cat >> "$cfg" <<EOF
+      #curtin added
+      serial --unit=0 --speed=115200
+      terminal --timeout=2 serial console
+      EOF
+              r=$?
+              [ $r -eq 0 ] ||
+                  { perror $r "failed to append to grub1 cfg '$cfg'"; return; }
+          fi
+          sed -i -e '/linux16/n' -e '/console=/n' \
+              -e "s/root=\([^ ]*\)/root=\1 ${CONPARM}/" "$cfg" ||
+              { rerror "failed to update grub1 cfg '$cfg'."; return; }
+          info "updated grub1 cfg '$cfg'."
+      }
+      
+      update_grub2() {
+          local cfg="$1" defgrub="$2"
+          [ -e "$cfg" ] || { info "no grub2 config '$cfg'"; return 0; }
+          bk "$cfg" "grub2 config" || return
+          sed -i -e '/kernel/n' -e '/console=/n' \
+            -e "s/root=\([^ ]*\)/root=\1 ${CONPARM}/" "$cfg" ||
+            { rerror "failed to update grub2 '$cfg'"; return; }
+      
+          # update /etc/default/grub. any GRUB_CMDLINE_LINUX remove 
+          # any console= and add conparm at the beginning.
+          local var="GRUB_CMDLINE_LINUX" msg="updated grub2 '$cfg'."
+          if [ ! -e "$defgrub" ]; then
+              msg="$msg. no defaults file '$defgrub'."
+          else
+              bk "$defgrub" "grub2 defaults file" || return
+              msg="$msg. updated defaults file '$defgrub'."
+              sed -i \
+                  -e "/$var=/!n" \
+                  -e 's/console=[^ "]*//g' \
+                  -e "s/$var=\"/$var=\"${CONPARM}/" "$defgrub" ||
+                  { rerror "grub2 default update failed on $defgrub"; return; }
+          fi
+          info "$msg"
+      }
+
+      update_grub1 "$grub1conf" || fail "failed update grub1"
+      update_grub2 "$grub2conf" "$grub2def" || fail "failed update grub2"
+
+late_commands:
+ # centos66 images include grub 0.97 which will detect vmtests' ephemeral disk
+ # and the install disk which leaves grub configured with two disks.  When
+ # vmtest reboots into installed disk, there is only one disk and the grub
+ # map is no longer valid.  Here in 00_grub, we switch hd1 to hd0.  MAAS
+ # is not affected as their ephemeral image (iscsi or http) is not discovered
+ # by grub and therefor the device.map doesn't contain a second device. Cent7
+ # has grub2 which uses root by UUID
+ 00_grub1_boot: [curtin, in-target, --, sed, -i.curtin, -e,
+                 's|(hd1,0)|(hd0,0)|g', /boot/grub/grub.conf]
+ # vmtest wants output to go to serial console so we update grub inside.
+ 00_grub_serial: [curtin, in-target, --, '/root/curtin-send-console-to-serial']
 
=== added file 'examples/tests/journald_reporter.yaml'
--- examples/tests/journald_reporter.yaml	1970-01-01 00:00:00 +0000
+++ examples/tests/journald_reporter.yaml	2017-10-06 16:35:22 +0000
@@ -0,0 +1,20 @@
+reporting:
+  journald:
+    type: journald
+    level: DEBUG
+
+journal_cmds:
+  - &copy_journal_log |
+    journalctl -b -o short-precise --no-pager -t curtin_event \
+    > ${TARGET_MOUNT_POINT}/root/journalctl.curtin_events.log
+
+  # use sed to make the json file loadable (listify the json)
+  - &copy_journal_json |
+    journalctl -b -o json-pretty --no-pager -t curtin_event \
+     | sed -e '1i [' -e 's|^}|},|g'  -e '$s|^},|}|' -e '$a]' \
+     > ${TARGET_MOUNT_POINT}/root/journalctl.curtin_events.json
+
+# extract the journald entries for curtin
+late_commands:
+ 00_copy_journal__log: [sh, -c, *copy_journal_log]
+ 01_copy_journal_json: [sh, -c, *copy_journal_json]
 
=== modified file 'examples/tests/network_alias.yaml'
--- examples/tests/network_alias.yaml	2016-10-03 18:42:29 +0000
+++ examples/tests/network_alias.yaml	2017-10-06 16:35:22 +0000
@@ -8,29 +8,27 @@
           mac_address: "52:54:00:12:34:00"
           subnets:
               - type: static
-                address: 192.168.1.2/24
-                mtu: 1501
+                address: 10.47.98.1/24
               - type: static
                 address: 2001:4800:78ff:1b:be76:4eff:fe06:ffac
                 netmask: 'ffff:ffff:ffff:ffff::'
-                mtu: 1480
         # multi_v4_alias: multiple v4 addrs on same interface
         - type: physical
           name: interface1
           mac_address: "52:54:00:12:34:02"
           subnets:
               - type: static
-                address: 192.168.2.2/22
+                address: 192.168.20.2/24
                 routes:
-                  - network: 192.168.0.0
-                    netmask: 255.255.252.0
-                    gateway: 192.168.2.1
+                  - gateway: 192.168.20.1
+                    netmask: 255.255.255.0
+                    network: 10.242.47.0
               - type: static
-                address: 10.23.23.7/23
+                address: 10.23.22.7/23
                 routes:
-                  - gateway: 10.23.23.1
-                    netmask: 255.255.254.0
-                    network: 10.23.22.0
+                  - gateway: 10.23.22.2
+                    netmask: 255.255.255.0
+                    network: 10.49.253.0
         # multi_v6_alias: multiple v6 addrs on same interface
         - type: physical
           name: interface2
@@ -51,17 +49,17 @@
           mac_address: "52:54:00:12:34:06"
           subnets:
               - type: static
-                address: 192.168.7.7/22
+                address: 192.168.80.8/24
                 routes:
-                  - network: 192.168.0.0
-                    netmask: 255.255.252.0
-                    gateway: 192.168.7.1
+                  - gateway: 192.168.80.1
+                    netmask: 255.255.255.0
+                    network: 10.189.34.0
               - type: static
-                address: 10.99.99.23/23
+                address: 10.99.10.23/23
                 routes:
-                  - gateway: 10.99.99.1
-                    netmask: 255.255.254.0
-                    network: 10.99.98.0
+                  - gateway: 10.99.10.1
+                    netmask: 255.255.255.0
+                    network: 10.77.154.0
               - type: static
                 address: 2001:4800:78ff:1b:be76:4eff:beef:4000
                 netmask: 'ffff:ffff:ffff:ffff::'
@@ -86,17 +84,17 @@
                 address: 2001:4800:78ff:1b:be76:4eff:debe:9000
                 netmask: 'ffff:ffff:ffff:ffff::'
               - type: static
-                address: 192.168.100.100/22
+                address: 192.168.100.100/24
                 routes:
-                  - network: 192.168.0.0
-                    netmask: 255.255.252.0
-                    gateway: 192.168.100.1
+                  - gateway: 192.168.100.1
+                    netmask: 255.255.255.0
+                    network: 10.28.219.0
               - type: static
                 address: 10.17.142.2/23
                 routes:
                   - gateway: 10.17.142.1
-                    netmask: 255.255.254.0
-                    network: 10.17.142.0
+                    netmask: 255.255.255.0
+                    network: 10.82.49.0
         # multi_v6_and_v4_mix_order: multiple v4 and v6 addr, mixed order
         - type: physical
           name: interface5
@@ -109,17 +107,17 @@
                 address: 2001:4800:78ff:1b:be76:4eff:baaf:c000
                 netmask: 'ffff:ffff:ffff:ffff::'
               - type: static
-                address: 192.168.200.200/22
+                address: 192.168.200.200/24
                 routes:
-                  - network: 192.168.0.0
-                    netmask: 255.255.252.0
-                    gateway: 192.168.200.1
+                  - gateway: 192.168.200.1
+                    netmask: 255.255.255.0
+                    network: 10.71.23.0
               - type: static
                 address: 10.252.2.2/23
                 routes:
                   - gateway: 10.252.2.1
-                    netmask: 255.255.254.0
-                    network: 10.252.2.0
+                    netmask: 255.255.255.0
+                    network: 10.3.7.0
               - type: static
                 address: 2001:4800:78ff:1b:be76:4eff:baaf:b000
                 netmask: 'ffff:ffff:ffff:ffff::'
 
=== modified file 'examples/tests/network_static_routes.yaml'
--- examples/tests/network_static_routes.yaml	2017-02-08 22:22:44 +0000
+++ examples/tests/network_static_routes.yaml	2017-10-06 16:35:22 +0000
@@ -10,18 +10,13 @@
               - address: 172.23.31.42/26
                 gateway: 172.23.31.2
                 type: static
-      - type: route
-        id: 4
-        metric: 0
-        destination: 10.0.0.0/12
-        gateway: 172.23.31.1
-      - type: route
-        id: 5
-        metric: 0
-        destination: 192.168.0.0/16
-        gateway: 172.23.31.1
-      - type: route
-        id: 6
-        metric: 1
-        destination: 10.200.0.0/16
-        gateway: 172.23.31.1
+                routes:
+                  - gateway: 172.23.31.1
+                    network: 10.0.0.0/12
+                    metric: 0
+                  - gateway: 172.23.31.1
+                    network: 192.168.0.0/16
+                    metric: 0
+                  - gateway: 172.23.31.1
+                    network: 10.200.0.0/16
+                    metric: 1
 
=== added file 'examples/tests/network_v2_passthrough.yaml'
--- examples/tests/network_v2_passthrough.yaml	1970-01-01 00:00:00 +0000
+++ examples/tests/network_v2_passthrough.yaml	2017-10-06 16:35:22 +0000
@@ -0,0 +1,8 @@
+showtrace: true
+network:
+    version: 2
+    ethernets:
+        interface0:
+            match:
+                mac_address: "52:54:00:12:34:00"
+            dhcp4: true
 
=== modified file 'setup.py'
--- setup.py	2016-10-03 18:42:29 +0000
+++ setup.py	2017-10-06 16:35:22 +0000
@@ -1,6 +1,7 @@
 from distutils.core import setup
 from glob import glob
 import os
+import sys
 
 import curtin
 
@@ -8,6 +9,19 @@
 def is_f(p):
     return os.path.isfile(p)
 
+
+def in_virtualenv():
+    try:
+        if sys.real_prefix == sys.prefix:
+            return False
+        else:
+            return True
+    except AttributeError:
+        return False
+
+
+USR = "usr" if in_virtualenv() else "/usr"
+
 setup(
     name="curtin",
     description='The curtin installer',
@@ -27,9 +41,9 @@
     ],
     scripts=glob('bin/*'),
     data_files=[
-        ('/usr/share/doc/curtin',
+        (USR + '/share/doc/curtin',
          [f for f in glob('doc/*') if is_f(f)]),
-        ('/usr/lib/curtin/helpers',
+        (USR + '/lib/curtin/helpers',
          [f for f in glob('helpers/*') if is_f(f)])
     ]
 )
 
=== modified file 'tests/unittests/helpers.py'
--- tests/unittests/helpers.py	2017-02-08 22:22:44 +0000
+++ tests/unittests/helpers.py	2017-10-06 16:35:22 +0000
@@ -19,6 +19,10 @@
 import imp
 import importlib
 import mock
+import os
+import shutil
+import tempfile
+from unittest import TestCase
 
 
 def builtin_module_name():
@@ -43,3 +47,35 @@
     m_patch = '{}.open'.format(mod_name)
     with mock.patch(m_patch, m_open, create=True):
         yield m_open
+
+
+class CiTestCase(TestCase):
+    """Common testing class which all curtin unit tests subclass."""
+
+    def add_patch(self, target, attr, **kwargs):
+        """Patches specified target object and sets it as attr on test
+        instance also schedules cleanup"""
+        if 'autospec' not in kwargs:
+            kwargs['autospec'] = True
+        m = mock.patch(target, **kwargs)
+        p = m.start()
+        self.addCleanup(m.stop)
+        setattr(self, attr, p)
+
+    def tmp_dir(self, dir=None, cleanup=True):
+        """Return a full path to a temporary directory for the test run."""
+        if dir is None:
+            tmpd = tempfile.mkdtemp(
+                prefix="curtin-ci-%s." % self.__class__.__name__)
+        else:
+            tmpd = tempfile.mkdtemp(dir=dir)
+        self.addCleanup(shutil.rmtree, tmpd)
+        return tmpd
+
+    def tmp_path(self, path, _dir=None):
+        # return an absolute path to 'path' under dir.
+        # if dir is None, one will be created with tmp_dir()
+        # the file is not created or modified.
+        if _dir is None:
+            _dir = self.tmp_dir()
+        return os.path.normpath(os.path.abspath(os.path.join(_dir, path)))
 
=== modified file 'tests/unittests/test_apt_custom_sources_list.py'
--- tests/unittests/test_apt_custom_sources_list.py	2016-10-03 18:42:29 +0000
+++ tests/unittests/test_apt_custom_sources_list.py	2017-10-06 16:35:22 +0000
@@ -3,10 +3,7 @@
 """
 import logging
 import os
-import shutil
-import tempfile
 
-from unittest import TestCase
 
 import yaml
 import mock
@@ -14,6 +11,7 @@
 
 from curtin import util
 from curtin.commands import apt_config
+from .helpers import CiTestCase
 
 LOG = logging.getLogger(__name__)
 
@@ -85,12 +83,11 @@
 """)
 
 
-class TestAptSourceConfigSourceList(TestCase):
+class TestAptSourceConfigSourceList(CiTestCase):
     """TestAptSourceConfigSourceList - Class to test sources list rendering"""
     def setUp(self):
         super(TestAptSourceConfigSourceList, self).setUp()
-        self.new_root = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, self.new_root)
+        self.new_root = self.tmp_dir()
         # self.patchUtils(self.new_root)
 
     @staticmethod
 
=== modified file 'tests/unittests/test_apt_source.py'
--- tests/unittests/test_apt_source.py	2017-03-01 16:13:56 +0000
+++ tests/unittests/test_apt_source.py	2017-10-06 16:35:22 +0000
@@ -4,11 +4,8 @@
 import glob
 import os
 import re
-import shutil
 import socket
-import tempfile
 
-from unittest import TestCase
 
 import mock
 from mock import call
@@ -16,6 +13,7 @@
 from curtin import util
 from curtin import gpg
 from curtin.commands import apt_config
+from .helpers import CiTestCase
 
 
 EXPECTEDKEY = u"""-----BEGIN PGP PUBLIC KEY BLOCK-----
@@ -62,14 +60,13 @@
 ChrootableTargetStr = "curtin.commands.apt_config.util.ChrootableTarget"
 
 
-class TestAptSourceConfig(TestCase):
+class TestAptSourceConfig(CiTestCase):
     """ TestAptSourceConfig
     Main Class to test apt configs
     """
     def setUp(self):
         super(TestAptSourceConfig, self).setUp()
-        self.tmp = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, self.tmp)
+        self.tmp = self.tmp_dir()
         self.aptlistfile = os.path.join(self.tmp, "single-deb.list")
         self.aptlistfile2 = os.path.join(self.tmp, "single-deb2.list")
         self.aptlistfile3 = os.path.join(self.tmp, "single-deb3.list")
@@ -930,7 +927,7 @@
             orig, apt_config.disable_suites(["proposed"], orig, rel))
 
 
-class TestDebconfSelections(TestCase):
+class TestDebconfSelections(CiTestCase):
 
     @mock.patch("curtin.commands.apt_config.debconf_set_selections")
     def test_no_set_sel_if_none_to_set(self, m_set_sel):
 
=== modified file 'tests/unittests/test_basic.py'
--- tests/unittests/test_basic.py	2013-07-29 16:12:09 +0000
+++ tests/unittests/test_basic.py	2017-10-06 16:35:22 +0000
@@ -1,7 +1,7 @@
-from unittest import TestCase
-
-
-class TestImport(TestCase):
+from .helpers import CiTestCase
+
+
+class TestImport(CiTestCase):
     def test_import(self):
         import curtin
         self.assertFalse(getattr(curtin, 'BOGUS_ENTRY', None))
 
=== modified file 'tests/unittests/test_block.py'
--- tests/unittests/test_block.py	2017-06-12 20:39:06 +0000
+++ tests/unittests/test_block.py	2017-10-06 16:35:22 +0000
@@ -1,19 +1,16 @@
-from unittest import TestCase
 import functools
 import os
 import mock
-import tempfile
-import shutil
 import sys
 
 from collections import OrderedDict
 
-from .helpers import simple_mocked_open
+from .helpers import CiTestCase, simple_mocked_open
 from curtin import util
 from curtin import block
 
 
-class TestBlock(TestCase):
+class TestBlock(CiTestCase):
 
     @mock.patch("curtin.block.util")
     def test_get_volume_uuid(self, mock_util):
@@ -103,7 +100,7 @@
             block.lookup_disk(serial)
 
 
-class TestSysBlockPath(TestCase):
+class TestSysBlockPath(CiTestCase):
     @mock.patch("curtin.block.get_blockdev_for_partition")
     @mock.patch("os.path.exists")
     def test_existing_valid_devname(self, m_os_path_exists, m_get_blk):
@@ -177,19 +174,13 @@
                          block.sys_block_path('/dev/cciss/c0d0p1'))
 
 
-class TestWipeFile(TestCase):
+class TestWipeFile(CiTestCase):
     def __init__(self, *args, **kwargs):
         super(TestWipeFile, self).__init__(*args, **kwargs)
 
-    def tfile(self, *args):
-        # return a temp file in a dir that will be cleaned up
-        tmpdir = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, tmpdir)
-        return os.path.sep.join([tmpdir] + list(args))
-
     def test_non_exist_raises_file_not_found(self):
         try:
-            p = self.tfile("enofile")
+            p = self.tmp_path("enofile")
             block.wipe_file(p)
             raise Exception("%s did not raise exception" % p)
         except Exception as e:
@@ -198,7 +189,7 @@
 
     def test_non_exist_dir_raises_file_not_found(self):
         try:
-            p = self.tfile("enodir", "file")
+            p = self.tmp_path(os.path.sep.join(["enodir", "file"]))
             block.wipe_file(p)
             raise Exception("%s did not raise exception" % p)
         except Exception as e:
@@ -207,7 +198,7 @@
 
     def test_default_is_zero(self):
         flen = 1024
-        myfile = self.tfile("def_zero")
+        myfile = self.tmp_path("def_zero")
         util.write_file(myfile, flen * b'\1', omode="wb")
         block.wipe_file(myfile)
         found = util.load_file(myfile, decode=False)
@@ -219,7 +210,7 @@
         def reader(size):
             return size * b'\1'
 
-        myfile = self.tfile("reader_used")
+        myfile = self.tmp_path("reader_used")
         # populate with nulls
         util.write_file(myfile, flen * b'\0', omode="wb")
         block.wipe_file(myfile, reader=reader, buflen=flen)
@@ -236,15 +227,15 @@
             data['x'] = data['x'][size:]
             return buf
 
-        myfile = self.tfile("reader_twice")
+        myfile = self.tmp_path("reader_twice")
         util.write_file(myfile, flen * b'\xff', omode="wb")
         block.wipe_file(myfile, reader=reader, buflen=20)
         found = util.load_file(myfile, decode=False)
         self.assertEqual(found, expected)
 
     def test_reader_fhandle(self):
-        srcfile = self.tfile("fhandle_src")
-        trgfile = self.tfile("fhandle_trg")
+        srcfile = self.tmp_path("fhandle_src")
+        trgfile = self.tmp_path("fhandle_trg")
         data = '\n'.join(["this is source file." for f in range(0, 10)] + [])
         util.write_file(srcfile, data)
         util.write_file(trgfile, 'a' * len(data))
@@ -254,7 +245,7 @@
         self.assertEqual(data, found)
 
     def test_exclusive_open_raise_missing(self):
-        myfile = self.tfile("no-such-file")
+        myfile = self.tmp_path("no-such-file")
 
         with self.assertRaises(ValueError):
             with block.exclusive_open(myfile) as fp:
@@ -265,7 +256,7 @@
     @mock.patch('os.open')
     def test_exclusive_open(self, mock_os_open, mock_os_fdopen, mock_os_close):
         flen = 1024
-        myfile = self.tfile("my_exclusive_file")
+        myfile = self.tmp_path("my_exclusive_file")
         util.write_file(myfile, flen * b'\1', omode="wb")
         mock_fd = 3
         mock_os_open.return_value = mock_fd
@@ -288,7 +279,7 @@
                                                     mock_os_close,
                                                     mock_util_fuser):
         flen = 1024
-        myfile = self.tfile("my_exclusive_file")
+        myfile = self.tmp_path("my_exclusive_file")
         util.write_file(myfile, flen * b'\1', omode="wb")
         mock_os_open.side_effect = OSError("NO_O_EXCL")
         mock_holders.return_value = ['md1']
@@ -310,7 +301,7 @@
     def test_exclusive_open_fdopen_failure(self, mock_os_open,
                                            mock_os_fdopen, mock_os_close):
         flen = 1024
-        myfile = self.tfile("my_exclusive_file")
+        myfile = self.tmp_path("my_exclusive_file")
         util.write_file(myfile, flen * b'\1', omode="wb")
         mock_fd = 3
         mock_os_open.return_value = mock_fd
@@ -328,7 +319,7 @@
             self.assertEqual([], mock_os_close.call_args_list)
 
 
-class TestWipeVolume(TestCase):
+class TestWipeVolume(CiTestCase):
     dev = '/dev/null'
 
     @mock.patch('curtin.block.lvm')
@@ -366,7 +357,7 @@
             block.wipe_volume(self.dev, mode='invalidmode')
 
 
-class TestBlockKnames(TestCase):
+class TestBlockKnames(CiTestCase):
     """Tests for some of the kname functions in block"""
     def test_determine_partition_kname(self):
         part_knames = [(('sda', 1), 'sda1'),
@@ -430,7 +421,7 @@
                 block.kname_to_path(kname)
 
 
-class TestPartTableSignature(TestCase):
+class TestPartTableSignature(CiTestCase):
     blockdev = '/dev/null'
     dos_content = b'\x00' * 0x1fe + b'\x55\xAA' + b'\x00' * 0xf00
     gpt_content = b'\x00' * 0x200 + b'EFI PART' + b'\x00' * (0x200 - 8)
@@ -493,7 +484,7 @@
                     block.check_efi_signature(self.blockdev))
 
 
-class TestNonAscii(TestCase):
+class TestNonAscii(CiTestCase):
     @mock.patch('curtin.block.util.subp')
     def test_lsblk(self, mock_subp):
         # lsblk can write non-ascii data, causing shlex to blow up
@@ -519,14 +510,7 @@
         block.blkid()
 
 
-class TestSlaveKnames(TestCase):
-    def add_patch(self, target, attr, autospec=True):
-        """Patches specified target object and sets it as attr on test
-        instance also schedules cleanup"""
-        m = mock.patch(target, autospec=autospec)
-        p = m.start()
-        self.addCleanup(m.stop)
-        setattr(self, attr, p)
+class TestSlaveKnames(CiTestCase):
 
     def setUp(self):
         super(TestSlaveKnames, self).setUp()
 
=== modified file 'tests/unittests/test_block_iscsi.py'
--- tests/unittests/test_block_iscsi.py	2017-06-12 20:39:06 +0000
+++ tests/unittests/test_block_iscsi.py	2017-10-06 16:35:22 +0000
@@ -1,23 +1,13 @@
 import mock
+import os
 
-from unittest import TestCase
 from curtin.block import iscsi
-
-
-class IscsiTestBase(TestCase):
-    def setUp(self):
-        super(IscsiTestBase, self).setUp()
-
-    def add_patch(self, target, attr):
-        """Patches specified target object and sets it as attr on test
-        instance also schedules cleanup"""
-        m = mock.patch(target, autospec=True)
-        p = m.start()
-        self.addCleanup(m.stop)
-        setattr(self, attr, p)
-
-
-class TestBlockIscsiPortalParsing(IscsiTestBase):
+from curtin import util
+from .helpers import CiTestCase
+
+
+class TestBlockIscsiPortalParsing(CiTestCase):
+
     def test_iscsi_portal_parsing_string(self):
         with self.assertRaisesRegexp(ValueError, 'not a string'):
             iscsi.assert_valid_iscsi_portal(1234)
@@ -490,7 +480,7 @@
         self.assertEquals(i.target, 'iqn.2017-04.com.example.test:target-name')
 
 
-class TestBlockIscsiVolPath(IscsiTestBase):
+class TestBlockIscsiVolPath(CiTestCase):
     # non-iscsi backed disk returns false
     # regular iscsi-backed disk returns true
     # layered setup without an iscsi member returns false
@@ -569,4 +559,183 @@
         with self.assertRaises(ValueError):
             iscsi.volpath_is_iscsi(None)
 
+
+class TestBlockIscsiDiskFromConfig(CiTestCase):
+    # Test iscsi parsing of storage config for iscsi configure disks
+
+    def setUp(self):
+        super(TestBlockIscsiDiskFromConfig, self).setUp()
+        self.add_patch('curtin.block.iscsi.util.subp', 'mock_subp')
+
+    def test_parse_iscsi_disk_from_config(self):
+        """Test parsing iscsi volume path creates the same iscsi disk"""
+        target = 'curtin-659d5f45-4f23-46cb-b826-f2937b896e09'
+        iscsi_path = 'iscsi:10.245.168.20::20112:1:' + target
+        cfg = {
+            'storage': {
+                'config': [{'type': 'disk',
+                            'id': 'iscsidev1',
+                            'path': iscsi_path,
+                            'name': 'iscsi_disk1',
+                            'ptable': 'msdos',
+                            'wipe': 'superblock'}]
+                }
+        }
+        expected_iscsi_disk = iscsi.IscsiDisk(iscsi_path)
+        iscsi_disk = iscsi.get_iscsi_disks_from_config(cfg).pop()
+        # utilize IscsiDisk str method for equality check
+        self.assertEqual(str(expected_iscsi_disk), str(iscsi_disk))
+
+    def test_parse_iscsi_disk_from_config_no_iscsi(self):
+        """Test parsing storage config with no iscsi disks included"""
+        cfg = {
+            'storage': {
+                'config': [{'type': 'disk',
+                            'id': 'ssd1',
+                            'path': 'dev/slash/foo1',
+                            'name': 'the-fast-one',
+                            'ptable': 'gpt',
+                            'wipe': 'superblock'}]
+                }
+        }
+        expected_iscsi_disks = []
+        iscsi_disks = iscsi.get_iscsi_disks_from_config(cfg)
+        self.assertEqual(expected_iscsi_disks, iscsi_disks)
+
+    def test_parse_iscsi_disk_from_config_invalid_iscsi(self):
+        """Test parsing storage config with no iscsi disks included"""
+        cfg = {
+            'storage': {
+                'config': [{'type': 'disk',
+                            'id': 'iscsidev2',
+                            'path': 'iscsi:garbage',
+                            'name': 'noob-city',
+                            'ptable': 'msdos',
+                            'wipe': 'superblock'}]
+                }
+        }
+        with self.assertRaises(ValueError):
+            iscsi.get_iscsi_disks_from_config(cfg)
+
+    def test_parse_iscsi_disk_from_config_empty(self):
+        """Test parse_iscsi_disks handles empty/invalid config"""
+        expected_iscsi_disks = []
+        iscsi_disks = iscsi.get_iscsi_disks_from_config({})
+        self.assertEqual(expected_iscsi_disks, iscsi_disks)
+
+        cfg = {'storage': {'config': []}}
+        iscsi_disks = iscsi.get_iscsi_disks_from_config(cfg)
+        self.assertEqual(expected_iscsi_disks, iscsi_disks)
+
+    def test_parse_iscsi_disk_from_config_none(self):
+        """Test parse_iscsi_disks handles no config"""
+        expected_iscsi_disks = []
+        iscsi_disks = iscsi.get_iscsi_disks_from_config({})
+        self.assertEqual(expected_iscsi_disks, iscsi_disks)
+
+        cfg = None
+        iscsi_disks = iscsi.get_iscsi_disks_from_config(cfg)
+        self.assertEqual(expected_iscsi_disks, iscsi_disks)
+
+
+class TestBlockIscsiDisconnect(CiTestCase):
+    # test that when disconnecting iscsi targets we
+    # check that the target has an active session before
+    # issuing a disconnect command
+
+    def setUp(self):
+        super(TestBlockIscsiDisconnect, self).setUp()
+        self.add_patch('curtin.block.iscsi.util.subp', 'mock_subp')
+        self.add_patch('curtin.block.iscsi.iscsiadm_sessions',
+                       'mock_iscsi_sessions')
+        # fake target_root + iscsi nodes dir
+        self.target_path = self.tmp_dir()
+        self.iscsi_nodes = os.path.join(self.target_path, 'etc/iscsi/nodes')
+        util.ensure_dir(self.iscsi_nodes)
+
+    def _fmt_disconnect(self, target, portal):
+        return ['iscsiadm', '--mode=node', '--targetname=%s' % target,
+                '--portal=%s' % portal, '--logout']
+
+    def _setup_nodes(self, sessions, connection):
+        # setup iscsi_nodes dir (<fakeroot>/etc/iscsi/nodes) with content
+        for s in sessions:
+            sdir = os.path.join(self.iscsi_nodes, s)
+            connpath = os.path.join(sdir, connection)
+            util.ensure_dir(sdir)
+            util.write_file(connpath, content="")
+
+    def test_disconnect_target_disk(self):
+        """Test iscsi disconnecting multiple sessions, all present"""
+
+        sessions = [
+            'curtin-53ab23ff-a887-449a-80a8-288151208091',
+            'curtin-94b62de1-c579-42c0-879e-8a28178e64c5',
+            'curtin-556aeecd-a227-41b7-83d7-2bb471c574b4',
+            'curtin-fd0f644b-7858-420f-9997-3ea2aefe87b9'
+        ]
+        connection = '10.245.168.20,16395,1'
+        self._setup_nodes(sessions, connection)
+
+        self.mock_iscsi_sessions.return_value = "\n".join(sessions)
+
+        iscsi.disconnect_target_disks(self.target_path)
+
+        expected_calls = []
+        for session in sessions:
+            (host, port, _) = connection.split(',')
+            disconnect = self._fmt_disconnect(session, "%s:%s" % (host, port))
+            calls = [
+                mock.call(['sync']),
+                mock.call(disconnect, capture=True, log_captured=True),
+                mock.call(['udevadm', 'settle']),
+            ]
+            expected_calls.extend(calls)
+
+        self.mock_subp.assert_has_calls(expected_calls, any_order=True)
+
+    def test_disconnect_target_disk_skip_disconnected(self):
+        """Test iscsi does not attempt to disconnect already closed sessions"""
+        sessions = [
+            'curtin-53ab23ff-a887-449a-80a8-288151208091',
+            'curtin-94b62de1-c579-42c0-879e-8a28178e64c5',
+            'curtin-556aeecd-a227-41b7-83d7-2bb471c574b4',
+            'curtin-fd0f644b-7858-420f-9997-3ea2aefe87b9'
+        ]
+        connection = '10.245.168.20,16395,1'
+        self._setup_nodes(sessions, connection)
+        # Test with all sessions are already disconnected
+        self.mock_iscsi_sessions.return_value = ""
+
+        iscsi.disconnect_target_disks(self.target_path)
+
+        self.mock_subp.assert_has_calls([], any_order=True)
+
+    @mock.patch('curtin.block.iscsi.iscsiadm_logout')
+    def test_disconnect_target_disk_raises_runtime_error(self, mock_logout):
+        """Test iscsi raises RuntimeError if we fail to logout"""
+        sessions = [
+            'curtin-53ab23ff-a887-449a-80a8-288151208091',
+        ]
+        connection = '10.245.168.20,16395,1'
+        self._setup_nodes(sessions, connection)
+        self.mock_iscsi_sessions.return_value = "\n".join(sessions)
+        mock_logout.side_effect = util.ProcessExecutionError()
+
+        with self.assertRaises(RuntimeError):
+            iscsi.disconnect_target_disks(self.target_path)
+
+        expected_calls = []
+        for session in sessions:
+            (host, port, _) = connection.split(',')
+            disconnect = self._fmt_disconnect(session, "%s:%s" % (host, port))
+            calls = [
+                mock.call(['sync']),
+                mock.call(disconnect, capture=True, log_captured=True),
+                mock.call(['udevadm', 'settle']),
+            ]
+            expected_calls.extend(calls)
+
+        self.mock_subp.assert_has_calls([], any_order=True)
+
 # vi: ts=4 expandtab syntax=python
 
=== modified file 'tests/unittests/test_block_lvm.py'
--- tests/unittests/test_block_lvm.py	2016-10-03 18:42:29 +0000
+++ tests/unittests/test_block_lvm.py	2017-10-06 16:35:22 +0000
@@ -1,10 +1,10 @@
 from curtin.block import lvm
 
-from unittest import TestCase
+from .helpers import CiTestCase
 import mock
 
 
-class TestBlockLvm(TestCase):
+class TestBlockLvm(CiTestCase):
     vg_name = 'ubuntu-volgroup'
 
     @mock.patch('curtin.block.lvm.util')
 
=== modified file 'tests/unittests/test_block_mdadm.py'
--- tests/unittests/test_block_mdadm.py	2017-06-12 20:39:06 +0000
+++ tests/unittests/test_block_mdadm.py	2017-10-06 16:35:22 +0000
@@ -1,27 +1,15 @@
-from unittest import TestCase
 from mock import call, patch
 from curtin.block import dev_short
 from curtin.block import mdadm
 from curtin import util
+from .helpers import CiTestCase
 import os
 import subprocess
 import textwrap
 
 
-class MdadmTestBase(TestCase):
-    def setUp(self):
-        super(MdadmTestBase, self).setUp()
-
-    def add_patch(self, target, attr):
-        """Patches specified target object and sets it as attr on test
-        instance also schedules cleanup"""
-        m = patch(target, autospec=True)
-        p = m.start()
-        self.addCleanup(m.stop)
-        setattr(self, attr, p)
-
-
-class TestBlockMdadmAssemble(MdadmTestBase):
+class TestBlockMdadmAssemble(CiTestCase):
+
     def setUp(self):
         super(TestBlockMdadmAssemble, self).setUp()
         self.add_patch('curtin.block.mdadm.util', 'mock_util')
@@ -94,7 +82,7 @@
             rcs=[0, 1, 2])
 
 
-class TestBlockMdadmCreate(MdadmTestBase):
+class TestBlockMdadmCreate(CiTestCase):
     def setUp(self):
         super(TestBlockMdadmCreate, self).setUp()
         self.add_patch('curtin.block.mdadm.util', 'mock_util')
@@ -243,7 +231,7 @@
         self.mock_util.subp.assert_has_calls(expected_calls)
 
 
-class TestBlockMdadmExamine(MdadmTestBase):
+class TestBlockMdadmExamine(CiTestCase):
     def setUp(self):
         super(TestBlockMdadmExamine, self).setUp()
         self.add_patch('curtin.block.mdadm.util', 'mock_util')
@@ -328,7 +316,7 @@
         self.assertEqual(data, {})
 
 
-class TestBlockMdadmStop(MdadmTestBase):
+class TestBlockMdadmStop(CiTestCase):
     def setUp(self):
         super(TestBlockMdadmStop, self).setUp()
         self.add_patch('curtin.block.mdadm.util.lsb_release', 'mock_util_lsb')
@@ -495,7 +483,7 @@
         self.mock_util_write_file.assert_has_calls(expected_writes)
 
 
-class TestBlockMdadmRemove(MdadmTestBase):
+class TestBlockMdadmRemove(CiTestCase):
     def setUp(self):
         super(TestBlockMdadmRemove, self).setUp()
         self.add_patch('curtin.block.mdadm.util', 'mock_util')
@@ -521,7 +509,7 @@
         self.mock_util.subp.assert_has_calls(expected_calls)
 
 
-class TestBlockMdadmQueryDetail(MdadmTestBase):
+class TestBlockMdadmQueryDetail(CiTestCase):
     def setUp(self):
         super(TestBlockMdadmQueryDetail, self).setUp()
         self.add_patch('curtin.block.mdadm.util', 'mock_util')
@@ -599,7 +587,7 @@
                          '93a73e10:427f280b:b7076c02:204b8f7a')
 
 
-class TestBlockMdadmDetailScan(MdadmTestBase):
+class TestBlockMdadmDetailScan(CiTestCase):
     def setUp(self):
         super(TestBlockMdadmDetailScan, self).setUp()
         self.add_patch('curtin.block.mdadm.util', 'mock_util')
@@ -634,7 +622,7 @@
         self.assertEqual(None, data)
 
 
-class TestBlockMdadmMdHelpers(MdadmTestBase):
+class TestBlockMdadmMdHelpers(CiTestCase):
     def setUp(self):
         super(TestBlockMdadmMdHelpers, self).setUp()
         self.add_patch('curtin.block.mdadm.util', 'mock_util')
 
=== modified file 'tests/unittests/test_block_mkfs.py'
--- tests/unittests/test_block_mkfs.py	2016-10-03 18:42:29 +0000
+++ tests/unittests/test_block_mkfs.py	2017-10-06 16:35:22 +0000
@@ -1,10 +1,10 @@
 from curtin.block import mkfs
 
-from unittest import TestCase
+from .helpers import CiTestCase
 import mock
 
 
-class TestBlockMkfs(TestCase):
+class TestBlockMkfs(CiTestCase):
     test_uuid = "fb26cc6c-ae73-11e5-9e38-2fb63f0c3155"
 
     def _get_config(self, fstype):
 
=== modified file 'tests/unittests/test_clear_holders.py'
--- tests/unittests/test_clear_holders.py	2017-06-12 20:39:06 +0000
+++ tests/unittests/test_clear_holders.py	2017-10-06 16:35:22 +0000
@@ -1,12 +1,12 @@
-from unittest import TestCase
 import mock
-
-from curtin.block import clear_holders
 import os
 import textwrap
 
-
-class TestClearHolders(TestCase):
+from curtin.block import clear_holders
+from .helpers import CiTestCase
+
+
+class TestClearHolders(CiTestCase):
     test_blockdev = '/dev/null'
     test_syspath = '/sys/class/block/null'
     remove_retries = [0.2] * 150  # clear_holders defaults to 30 seconds
 
=== added file 'tests/unittests/test_commands_apply_net.py'
--- tests/unittests/test_commands_apply_net.py	1970-01-01 00:00:00 +0000
+++ tests/unittests/test_commands_apply_net.py	2017-10-06 16:35:22 +0000
@@ -0,0 +1,334 @@
+from mock import patch, call
+import copy
+import os
+
+from curtin.commands import apply_net
+from curtin import util
+from .helpers import CiTestCase
+
+
+class TestApplyNet(CiTestCase):
+
+    def setUp(self):
+        super(TestApplyNet, self).setUp()
+
+        base = 'curtin.commands.apply_net.'
+        patches = [
+            (base + '_maybe_remove_legacy_eth0', 'm_legacy'),
+            (base + '_disable_ipv6_privacy_extensions', 'm_ipv6_priv'),
+            (base + '_patch_ifupdown_ipv6_mtu_hook', 'm_ipv6_mtu'),
+            ('curtin.net.netconfig_passthrough_available', 'm_netpass_avail'),
+            ('curtin.net.render_netconfig_passthrough', 'm_netpass_render'),
+            ('curtin.net.parse_net_config_data', 'm_net_parsedata'),
+            ('curtin.net.render_network_state', 'm_net_renderstate'),
+            ('curtin.net.network_state.from_state_file', 'm_ns_from_file'),
+            ('curtin.config.load_config', 'm_load_config'),
+        ]
+        for (tgt, attr) in patches:
+            self.add_patch(tgt, attr)
+
+        self.target = "my_target"
+        self.network_config = {
+            'network': {
+                'version': 1,
+                'config': {},
+            }
+        }
+        self.ns = {
+            'interfaces': {},
+            'routes': [],
+            'dns': {
+                'nameservers': [],
+                'search': [],
+            }
+        }
+
+    def test_apply_net_notarget(self):
+        self.assertRaises(Exception,
+                          apply_net.apply_net, None, "", "")
+
+    def test_apply_net_nostate_or_config(self):
+        self.assertRaises(Exception,
+                          apply_net.apply_net, "")
+
+    def test_apply_net_target_and_state(self):
+        self.m_ns_from_file.return_value = self.ns
+
+        self.assertRaises(ValueError,
+                          apply_net.apply_net, self.target,
+                          network_state=self.ns, network_config=None)
+
+    def test_apply_net_target_and_config(self):
+        self.m_load_config.return_value = self.network_config
+        self.m_netpass_avail.return_value = False
+        self.m_net_parsedata.return_value = self.ns
+
+        apply_net.apply_net(self.target, network_state=None,
+                            network_config=self.network_config)
+
+        self.m_netpass_avail.assert_called_with(self.target)
+
+        self.m_net_renderstate.assert_called_with(target=self.target,
+                                                  network_state=self.ns)
+        self.m_legacy.assert_called_with(self.target)
+        self.m_ipv6_priv.assert_called_with(self.target)
+        self.m_ipv6_mtu.assert_called_with(self.target)
+
+    def test_apply_net_target_and_config_passthrough(self):
+        self.m_load_config.return_value = self.network_config
+        self.m_netpass_avail.return_value = True
+
+        netcfg = "network_config.yaml"
+        apply_net.apply_net(self.target, network_state=None,
+                            network_config=netcfg)
+
+        self.assertFalse(self.m_ns_from_file.called)
+        self.m_load_config.assert_called_with(netcfg)
+        self.m_netpass_avail.assert_called_with(self.target)
+        nc = self.network_config
+        self.m_netpass_render.assert_called_with(self.target, netconfig=nc)
+
+        self.assertFalse(self.m_net_renderstate.called)
+        self.m_legacy.assert_called_with(self.target)
+        self.m_ipv6_priv.assert_called_with(self.target)
+        self.m_ipv6_mtu.assert_called_with(self.target)
+
+    def test_apply_net_target_and_config_passthrough_nonet(self):
+        nc = {'storage': {}}
+        self.m_load_config.return_value = nc
+        self.m_netpass_avail.return_value = True
+
+        netcfg = "network_config.yaml"
+
+        apply_net.apply_net(self.target, network_state=None,
+                            network_config=netcfg)
+
+        self.assertFalse(self.m_ns_from_file.called)
+        self.m_load_config.assert_called_with(netcfg)
+        self.m_netpass_avail.assert_called_with(self.target)
+        self.m_netpass_render.assert_called_with(self.target, netconfig=nc)
+
+        self.assertFalse(self.m_net_renderstate.called)
+        self.m_legacy.assert_called_with(self.target)
+        self.m_ipv6_priv.assert_called_with(self.target)
+        self.m_ipv6_mtu.assert_called_with(self.target)
+
+    def test_apply_net_target_and_config_passthrough_v2_not_available(self):
+        nc = copy.deepcopy(self.network_config)
+        nc['network']['version'] = 2
+        self.m_load_config.return_value = nc
+        self.m_netpass_avail.return_value = False
+        self.m_net_parsedata.return_value = self.ns
+
+        netcfg = "network_config.yaml"
+
+        apply_net.apply_net(self.target, network_state=None,
+                            network_config=netcfg)
+
+        self.assertFalse(self.m_ns_from_file.called)
+        self.m_load_config.assert_called_with(netcfg)
+        self.m_netpass_avail.assert_called_with(self.target)
+        self.assertFalse(self.m_netpass_render.called)
+        self.m_net_parsedata.assert_called_with(nc['network'])
+
+        self.m_net_renderstate.assert_called_with(
+            target=self.target, network_state=self.ns)
+        self.m_legacy.assert_called_with(self.target)
+        self.m_ipv6_priv.assert_called_with(self.target)
+        self.m_ipv6_mtu.assert_called_with(self.target)
+
+
+class TestApplyNetPatchIfupdown(CiTestCase):
+
+    @patch('curtin.util.write_file')
+    def test_apply_ipv6_mtu_hook(self, mock_write):
+        target = 'mytarget'
+        prehookfn = 'if-pre-up.d/mtuipv6'
+        posthookfn = 'if-up.d/mtuipv6'
+        mode = 0o755
+
+        apply_net._patch_ifupdown_ipv6_mtu_hook(target,
+                                                prehookfn=prehookfn,
+                                                posthookfn=posthookfn)
+
+        precfg = util.target_path(target, path=prehookfn)
+        postcfg = util.target_path(target, path=posthookfn)
+        precontents = apply_net.IFUPDOWN_IPV6_MTU_PRE_HOOK
+        postcontents = apply_net.IFUPDOWN_IPV6_MTU_POST_HOOK
+
+        hook_calls = [
+            call(precfg, precontents, mode=mode),
+            call(postcfg, postcontents, mode=mode),
+        ]
+        mock_write.assert_has_calls(hook_calls)
+
+    @patch('curtin.util.write_file')
+    def test_apply_ipv6_mtu_hook_write_fail(self, mock_write):
+        """Write failure raises IOError"""
+        target = 'mytarget'
+        prehookfn = 'if-pre-up.d/mtuipv6'
+        posthookfn = 'if-up.d/mtuipv6'
+        mock_write.side_effect = (IOError)
+
+        self.assertRaises(IOError,
+                          apply_net._patch_ifupdown_ipv6_mtu_hook,
+                          target,
+                          prehookfn=prehookfn,
+                          posthookfn=posthookfn)
+        self.assertEqual(1, mock_write.call_count)
+
+    @patch('curtin.util.write_file')
+    def test_apply_ipv6_mtu_hook_invalid_target(self, mock_write):
+        """Invalid target path fail before calling util.write_file"""
+        invalid_target = {}
+        prehookfn = 'if-pre-up.d/mtuipv6'
+        posthookfn = 'if-up.d/mtuipv6'
+
+        self.assertRaises(ValueError,
+                          apply_net._patch_ifupdown_ipv6_mtu_hook,
+                          invalid_target,
+                          prehookfn=prehookfn,
+                          posthookfn=posthookfn)
+        self.assertEqual(0, mock_write.call_count)
+
+    @patch('curtin.util.write_file')
+    def test_apply_ipv6_mtu_hook_invalid_prepost_fn(self, mock_write):
+        """Invalid prepost filenames fail before calling util.write_file"""
+        target = "mytarget"
+        invalid_prehookfn = {'a': 1}
+        invalid_posthookfn = {'b': 2}
+
+        self.assertRaises(ValueError,
+                          apply_net._patch_ifupdown_ipv6_mtu_hook,
+                          target,
+                          prehookfn=invalid_prehookfn,
+                          posthookfn=invalid_posthookfn)
+        self.assertEqual(0, mock_write.call_count)
+
+
+class TestApplyNetPatchIpv6Priv(CiTestCase):
+
+    @patch('curtin.util.del_file')
+    @patch('curtin.util.load_file')
+    @patch('os.path')
+    @patch('curtin.util.write_file')
+    def test_disable_ipv6_priv_extentions(self, mock_write, mock_ospath,
+                                          mock_load, mock_del):
+        target = 'mytarget'
+        path = 'etc/sysctl.d/10-ipv6-privacy.conf'
+        ipv6_priv_contents = (
+            'net.ipv6.conf.all.use_tempaddr = 2\n'
+            'net.ipv6.conf.default.use_tempaddr = 2')
+        expected_ipv6_priv_contents = '\n'.join(
+            ["# IPv6 Privacy Extensions (RFC 4941)",
+             "# Disabled by curtin",
+             "# net.ipv6.conf.all.use_tempaddr = 2",
+             "# net.ipv6.conf.default.use_tempaddr = 2"])
+        mock_ospath.exists.return_value = True
+        mock_load.side_effect = [ipv6_priv_contents]
+
+        apply_net._disable_ipv6_privacy_extensions(target)
+
+        cfg = util.target_path(target, path=path)
+        mock_write.assert_called_with(cfg, expected_ipv6_priv_contents)
+
+    @patch('curtin.util.load_file')
+    @patch('os.path')
+    def test_disable_ipv6_priv_extentions_decoderror(self, mock_ospath,
+                                                     mock_load):
+        target = 'mytarget'
+        mock_ospath.exists.return_value = True
+
+        # simulate loading of binary data
+        mock_load.side_effect = (Exception)
+
+        self.assertRaises(Exception,
+                          apply_net._disable_ipv6_privacy_extensions,
+                          target)
+
+    @patch('curtin.util.load_file')
+    @patch('os.path')
+    def test_disable_ipv6_priv_extentions_notfound(self, mock_ospath,
+                                                   mock_load):
+        target = 'mytarget'
+        path = 'foo.conf'
+        mock_ospath.exists.return_value = False
+
+        apply_net._disable_ipv6_privacy_extensions(target, path=path)
+
+        # source file not found
+        cfg = util.target_path(target, path)
+        mock_ospath.exists.assert_called_with(cfg)
+        self.assertEqual(0, mock_load.call_count)
+
+
+class TestApplyNetRemoveLegacyEth0(CiTestCase):
+
+    @patch('curtin.util.del_file')
+    @patch('curtin.util.load_file')
+    @patch('os.path')
+    def test_remove_legacy_eth0(self, mock_ospath, mock_load, mock_del):
+        target = 'mytarget'
+        path = 'eth0.cfg'
+        cfg = util.target_path(target, path)
+        legacy_eth0_contents = (
+            'auto eth0\n'
+            'iface eth0 inet dhcp')
+
+        mock_ospath.exists.return_value = True
+        mock_load.side_effect = [legacy_eth0_contents]
+
+        apply_net._maybe_remove_legacy_eth0(target, path)
+
+        mock_del.assert_called_with(cfg)
+
+    @patch('curtin.util.del_file')
+    @patch('curtin.util.load_file')
+    @patch('os.path')
+    def test_remove_legacy_eth0_nomatch(self, mock_ospath, mock_load,
+                                        mock_del):
+        target = 'mytarget'
+        path = 'eth0.cfg'
+        legacy_eth0_contents = "nomatch"
+        mock_ospath.join.side_effect = os.path.join
+        mock_ospath.exists.return_value = True
+        mock_load.side_effect = [legacy_eth0_contents]
+
+        self.assertRaises(Exception,
+                          apply_net._maybe_remove_legacy_eth0,
+                          target, path)
+
+        self.assertEqual(0, mock_del.call_count)
+
+    @patch('curtin.util.del_file')
+    @patch('curtin.util.load_file')
+    @patch('os.path')
+    def test_remove_legacy_eth0_badload(self, mock_ospath, mock_load,
+                                        mock_del):
+        target = 'mytarget'
+        path = 'eth0.cfg'
+        mock_ospath.exists.return_value = True
+        mock_load.side_effect = (Exception)
+
+        self.assertRaises(Exception,
+                          apply_net._maybe_remove_legacy_eth0,
+                          target, path)
+
+        self.assertEqual(0, mock_del.call_count)
+
+    @patch('curtin.util.del_file')
+    @patch('curtin.util.load_file')
+    @patch('os.path')
+    def test_remove_legacy_eth0_notfound(self, mock_ospath, mock_load,
+                                         mock_del):
+        target = 'mytarget'
+        path = 'eth0.conf'
+        mock_ospath.exists.return_value = False
+
+        apply_net._maybe_remove_legacy_eth0(target, path)
+
+        # source file not found
+        cfg = util.target_path(target, path)
+        mock_ospath.exists.assert_called_with(cfg)
+        self.assertEqual(0, mock_load.call_count)
+        self.assertEqual(0, mock_del.call_count)
 
=== modified file 'tests/unittests/test_commands_block_meta.py'
--- tests/unittests/test_commands_block_meta.py	2017-06-12 20:39:06 +0000
+++ tests/unittests/test_commands_block_meta.py	2017-10-06 16:35:22 +0000
@@ -1,24 +1,11 @@
-from unittest import TestCase
 from mock import patch, call
 from argparse import Namespace
 
 from curtin.commands import block_meta
-
-
-class BlockMetaTestBase(TestCase):
-    def setUp(self):
-        super(BlockMetaTestBase, self).setUp()
-
-    def add_patch(self, target, attr):
-        """Patches specified target object and sets it as attr on test
-        instance also schedules cleanup"""
-        m = patch(target, autospec=True)
-        p = m.start()
-        self.addCleanup(m.stop)
-        setattr(self, attr, p)
-
-
-class TestBlockMetaSimple(BlockMetaTestBase):
+from .helpers import CiTestCase
+
+
+class TestBlockMetaSimple(CiTestCase):
     def setUp(self):
         super(TestBlockMetaSimple, self).setUp()
         self.target = "my_target"
@@ -120,10 +107,10 @@
             [call(['mount', devname, self.target])])
 
 
-class TestBlockMeta(BlockMetaTestBase):
+class TestBlockMeta(CiTestCase):
+
     def setUp(self):
         super(TestBlockMeta, self).setUp()
-        # self.target = tempfile.mkdtemp()
 
         basepath = 'curtin.commands.block_meta.'
         self.add_patch(basepath + 'get_path_to_storage_volume', 'mock_getpath')
 
=== added file 'tests/unittests/test_commands_install.py'
--- tests/unittests/test_commands_install.py	1970-01-01 00:00:00 +0000
+++ tests/unittests/test_commands_install.py	2017-10-06 16:35:22 +0000
@@ -0,0 +1,22 @@
+import copy
+
+from curtin.commands import install
+from .helpers import CiTestCase
+
+
+class TestMigrateProxy(CiTestCase):
+    def test_legacy_moved_over(self):
+        """Legacy setting should get moved over."""
+        proxy = "http://my.proxy:3128"
+        cfg = {'http_proxy': proxy}
+        install.migrate_proxy_settings(cfg)
+        self.assertEqual(cfg, {'proxy': {'http_proxy': proxy}})
+
+    def test_no_legacy_new_only(self):
+        """If only new 'proxy', then no change is expected."""
+        proxy = "http://my.proxy:3128"
+        cfg = {'proxy': {'http_proxy': proxy, 'https_proxy': proxy,
+                         'no_proxy': "10.2.2.2"}}
+        expected = copy.deepcopy(cfg)
+        install.migrate_proxy_settings(cfg)
+        self.assertEqual(expected, cfg)
 
=== modified file 'tests/unittests/test_config.py'
--- tests/unittests/test_config.py	2015-10-02 16:19:07 +0000
+++ tests/unittests/test_config.py	2017-10-06 16:35:22 +0000
@@ -1,12 +1,12 @@
-from unittest import TestCase
 import copy
 import json
 import textwrap
 
 from curtin import config
-
-
-class TestMerge(TestCase):
+from .helpers import CiTestCase
+
+
+class TestMerge(CiTestCase):
     def test_merge_cfg_string(self):
         d1 = {'str1': 'str_one'}
         d2 = {'dict1': {'d1.e1': 'd1-e1'}}
@@ -16,7 +16,7 @@
         self.assertEqual(d1, expected)
 
 
-class TestCmdArg2Cfg(TestCase):
+class TestCmdArg2Cfg(CiTestCase):
     def test_cmdarg_flat(self):
         self.assertEqual(config.cmdarg2cfg("foo=bar"), {'foo': 'bar'})
 
@@ -50,7 +50,7 @@
         self.assertEqual(via_merge, via_merge_cmdarg)
 
 
-class TestConfigArchive(TestCase):
+class TestConfigArchive(CiTestCase):
     def test_archive_dict(self):
         myarchive = _replace_consts(textwrap.dedent("""
             _ARCH_HEAD_
 
=== modified file 'tests/unittests/test_curthooks.py'
--- tests/unittests/test_curthooks.py	2017-06-12 20:39:06 +0000
+++ tests/unittests/test_curthooks.py	2017-10-06 16:35:22 +0000
@@ -1,29 +1,14 @@
 import os
-from unittest import TestCase
 from mock import call, patch, MagicMock
-import shutil
-import tempfile
 
 from curtin.commands import curthooks
 from curtin import util
 from curtin import config
 from curtin.reporter import events
-
-
-class CurthooksBase(TestCase):
-    def setUp(self):
-        super(CurthooksBase, self).setUp()
-
-    def add_patch(self, target, attr, autospec=True):
-        """Patches specified target object and sets it as attr on test
-        instance also schedules cleanup"""
-        m = patch(target, autospec=autospec)
-        p = m.start()
-        self.addCleanup(m.stop)
-        setattr(self, attr, p)
-
-
-class TestGetFlashKernelPkgs(CurthooksBase):
+from .helpers import CiTestCase
+
+
+class TestGetFlashKernelPkgs(CiTestCase):
     def setUp(self):
         super(TestGetFlashKernelPkgs, self).setUp()
         self.add_patch('curtin.util.subp', 'mock_subp')
@@ -57,7 +42,7 @@
         self.mock_is_uefi_bootable.assert_called_once_with()
 
 
-class TestCurthooksInstallKernel(CurthooksBase):
+class TestCurthooksInstallKernel(CiTestCase):
     def setUp(self):
         super(TestCurthooksInstallKernel, self).setUp()
         self.add_patch('curtin.util.has_pkg_available', 'mock_haspkg')
@@ -70,7 +55,7 @@
                                       'fallback-package': 'mock-fallback',
                                       'mapping': {}}}
         # Tests don't actually install anything so we just need a name
-        self.target = tempfile.mktemp()
+        self.target = self.tmp_dir()
 
     def test__installs_flash_kernel_packages_when_needed(self):
         kernel_package = self.kernel_cfg.get('kernel', {}).get('package', {})
@@ -94,14 +79,11 @@
             [kernel_package], target=self.target)
 
 
-class TestUpdateInitramfs(CurthooksBase):
+class TestUpdateInitramfs(CiTestCase):
     def setUp(self):
         super(TestUpdateInitramfs, self).setUp()
         self.add_patch('curtin.util.subp', 'mock_subp')
-        self.target = tempfile.mkdtemp()
-
-    def tearDown(self):
-        shutil.rmtree(self.target)
+        self.target = self.tmp_dir()
 
     def _mnt_call(self, point):
         target = os.path.join(self.target, point)
@@ -134,7 +116,7 @@
         self.mock_subp.assert_has_calls(subp_calls)
 
 
-class TestInstallMissingPkgs(CurthooksBase):
+class TestInstallMissingPkgs(CiTestCase):
     def setUp(self):
         super(TestInstallMissingPkgs, self).setUp()
         self.add_patch('platform.machine', 'mock_machine')
@@ -176,11 +158,38 @@
         self.assertEqual([], self.mock_install_packages.call_args_list)
 
 
-class TestSetupGrub(CurthooksBase):
+class TestSetupZipl(CiTestCase):
+
+    def setUp(self):
+        super(TestSetupZipl, self).setUp()
+        self.target = self.tmp_dir()
+
+    @patch('curtin.block.get_devices_for_mp')
+    @patch('platform.machine')
+    def test_noop_non_s390x(self, m_machine, m_get_devices):
+        m_machine.return_value = 'non-s390x'
+        curthooks.setup_zipl(None, self.target)
+        self.assertEqual(0, m_get_devices.call_count)
+
+    @patch('curtin.block.get_devices_for_mp')
+    @patch('platform.machine')
+    def test_setup_zipl_writes_etc_zipl_conf(self, m_machine, m_get_devices):
+        m_machine.return_value = 's390x'
+        m_get_devices.return_value = ['/dev/mapper/ubuntu--vg-root']
+        curthooks.setup_zipl(None, self.target)
+        m_get_devices.assert_called_with(self.target)
+        with open(os.path.join(self.target, 'etc', 'zipl.conf')) as stream:
+            content = stream.read()
+        self.assertIn(
+            '# This has been modified by the MAAS curtin installer',
+            content)
+
+
+class TestSetupGrub(CiTestCase):
 
     def setUp(self):
         super(TestSetupGrub, self).setUp()
-        self.target = tempfile.mkdtemp()
+        self.target = self.tmp_dir()
         self.add_patch('curtin.util.lsb_release', 'mock_lsb_release')
         self.mock_lsb_release.return_value = {
             'codename': 'xenial',
@@ -203,9 +212,6 @@
         self.mock_in_chroot_subp.side_effect = iter(self.in_chroot_subp_output)
         self.mock_chroot.return_value = self.mock_in_chroot
 
-    def tearDown(self):
-        shutil.rmtree(self.target)
-
     def test_uses_old_grub_install_devices_in_cfg(self):
         cfg = {
             'grub_install_devices': ['/dev/vdb']
@@ -434,17 +440,13 @@
             self.mock_in_chroot_subp.call_args_list[0][0])
 
 
-class TestUbuntuCoreHooks(CurthooksBase):
+class TestUbuntuCoreHooks(CiTestCase):
     def setUp(self):
         super(TestUbuntuCoreHooks, self).setUp()
         self.target = None
 
-    def tearDown(self):
-        if self.target:
-            shutil.rmtree(self.target)
-
     def test_target_is_ubuntu_core(self):
-        self.target = tempfile.mkdtemp()
+        self.target = self.tmp_dir()
         ubuntu_core_path = os.path.join(self.target, 'system-data',
                                         'var/lib/snapd')
         util.ensure_dir(ubuntu_core_path)
@@ -457,7 +459,7 @@
         self.assertFalse(is_core)
 
     def test_target_is_ubuntu_core_noncore_target(self):
-        self.target = tempfile.mkdtemp()
+        self.target = self.tmp_dir()
         non_core_path = os.path.join(self.target, 'curtin')
         util.ensure_dir(non_core_path)
         self.assertTrue(os.path.isdir(non_core_path))
@@ -469,7 +471,7 @@
     @patch('curtin.commands.curthooks.handle_cloudconfig')
     def test_curthooks_no_config(self, mock_handle_cc, mock_del_file,
                                  mock_write_file):
-        self.target = tempfile.mkdtemp()
+        self.target = self.tmp_dir()
         cfg = {}
         curthooks.ubuntu_core_curthooks(cfg, target=self.target)
         self.assertEqual(len(mock_handle_cc.call_args_list), 0)
@@ -478,7 +480,7 @@
 
     @patch('curtin.commands.curthooks.handle_cloudconfig')
     def test_curthooks_cloud_config_remove_disabled(self, mock_handle_cc):
-        self.target = tempfile.mkdtemp()
+        self.target = self.tmp_dir()
         uc_cloud = os.path.join(self.target, 'system-data', 'etc/cloud')
         cc_disabled = os.path.join(uc_cloud, 'cloud-init.disabled')
         cc_path = os.path.join(uc_cloud, 'cloud.cfg.d')
@@ -496,7 +498,7 @@
         curthooks.ubuntu_core_curthooks(cfg, target=self.target)
 
         mock_handle_cc.assert_called_with(cfg.get('cloudconfig'),
-                                          target=cc_path)
+                                          base_dir=cc_path)
         self.assertFalse(os.path.exists(cc_disabled))
 
     @patch('curtin.util.write_file')
@@ -504,7 +506,7 @@
     @patch('curtin.commands.curthooks.handle_cloudconfig')
     def test_curthooks_cloud_config(self, mock_handle_cc, mock_del_file,
                                     mock_write_file):
-        self.target = tempfile.mkdtemp()
+        self.target = self.tmp_dir()
         cfg = {
             'cloudconfig': {
                 'file1': {
@@ -518,7 +520,7 @@
         cc_path = os.path.join(self.target,
                                'system-data/etc/cloud/cloud.cfg.d')
         mock_handle_cc.assert_called_with(cfg.get('cloudconfig'),
-                                          target=cc_path)
+                                          base_dir=cc_path)
         self.assertEqual(len(mock_write_file.call_args_list), 0)
 
     @patch('curtin.util.write_file')
@@ -526,7 +528,7 @@
     @patch('curtin.commands.curthooks.handle_cloudconfig')
     def test_curthooks_net_config(self, mock_handle_cc, mock_del_file,
                                   mock_write_file):
-        self.target = tempfile.mkdtemp()
+        self.target = self.tmp_dir()
         cfg = {
             'network': {
                 'version': '1',
@@ -541,13 +543,13 @@
         netcfg_path = os.path.join(self.target,
                                    'system-data',
                                    'etc/cloud/cloud.cfg.d',
-                                   '50-network-config.cfg')
+                                   '50-curtin-networking.cfg')
         netcfg = config.dump_config({'network': cfg.get('network')})
         mock_write_file.assert_called_with(netcfg_path,
                                            content=netcfg)
         self.assertEqual(len(mock_del_file.call_args_list), 0)
 
-    @patch('curtin.commands.curthooks.write_files')
+    @patch('curtin.commands.curthooks.futil.write_files')
     def test_handle_cloudconfig(self, mock_write_files):
         cc_target = "tmpXXXX/systemd-data/etc/cloud/cloud.cfg.d"
         cloudconfig = {
@@ -561,20 +563,202 @@
         }
 
         expected_cfg = {
-            'write_files': {
-                'file1': {
-                    'path': '50-cloudconfig-file1.cfg',
-                    'content': cloudconfig['file1']['content']},
-                'foobar': {
-                    'path': '50-cloudconfig-foobar.cfg',
-                    'content': cloudconfig['foobar']['content']}
-            }
+            'file1': {
+                'path': '50-cloudconfig-file1.cfg',
+                'content': cloudconfig['file1']['content']},
+            'foobar': {
+                'path': '50-cloudconfig-foobar.cfg',
+                'content': cloudconfig['foobar']['content']}
         }
-        curthooks.handle_cloudconfig(cloudconfig, target=cc_target)
+        curthooks.handle_cloudconfig(cloudconfig, base_dir=cc_target)
         mock_write_files.assert_called_with(expected_cfg, cc_target)
 
     def test_handle_cloudconfig_bad_config(self):
         with self.assertRaises(ValueError):
-            curthooks.handle_cloudconfig([], target="foobar")
+            curthooks.handle_cloudconfig([], base_dir="foobar")
+
+
+class TestDetectRequiredPackages(CiTestCase):
+    test_config = {
+        'storage': {
+            1: {
+                'bcache': {
+                    'type': 'bcache', 'name': 'bcache0', 'id': 'cache0',
+                    'backing_device': 'sda3', 'cache_device': 'sdb'},
+                'lvm_partition': {
+                    'id': 'lvol1', 'name': 'lv1', 'volgroup': 'vg1',
+                    'type': 'lvm_partition'},
+                'lvm_volgroup': {
+                    'id': 'vol1', 'name': 'vg1', 'devices': ['sda', 'sdb'],
+                    'type': 'lvm_volgroup'},
+                'raid': {
+                    'id': 'mddevice', 'name': 'md0', 'type': 'raid',
+                    'raidlevel': 5, 'devices': ['sda1', 'sdb1', 'sdc1']},
+                'ext2': {
+                    'id': 'format0', 'fstype': 'ext2', 'type': 'format'},
+                'ext3': {
+                    'id': 'format1', 'fstype': 'ext3', 'type': 'format'},
+                'ext4': {
+                    'id': 'format2', 'fstype': 'ext4', 'type': 'format'},
+                'btrfs': {
+                    'id': 'format3', 'fstype': 'btrfs', 'type': 'format'},
+                'xfs': {
+                    'id': 'format4', 'fstype': 'xfs', 'type': 'format'}}
+        },
+        'network': {
+            1: {
+                'bond': {
+                    'name': 'bond0', 'type': 'bond',
+                    'bond_interfaces': ['interface0', 'interface1'],
+                    'params': {'bond-mode': 'active-backup'},
+                    'subnets': [
+                        {'type': 'static', 'address': '10.23.23.2/24'},
+                        {'type': 'static', 'address': '10.23.24.2/24'}]},
+                'vlan': {
+                    'id': 'interface1.2667', 'mtu': 1500, 'name':
+                    'interface1.2667', 'type': 'vlan', 'vlan_id': 2667,
+                    'vlan_link': 'interface1',
+                    'subnets': [{'address': '10.245.184.2/24',
+                                 'dns_nameservers': [], 'type': 'static'}]},
+                'bridge': {
+                    'name': 'br0', 'bridge_interfaces': ['eth0', 'eth1'],
+                    'type': 'bridge', 'params': {
+                        'bridge_stp': 'off', 'bridge_fd': 0,
+                        'bridge_maxwait': 0},
+                    'subnets': [
+                        {'type': 'static', 'address': '192.168.14.2/24'},
+                        {'type': 'static', 'address': '2001:1::1/64'}]}},
+            2: {
+                'vlan': {
+                    'vlans': {
+                        'en-intra': {'id': 1, 'link': 'eno1', 'dhcp4': 'yes'},
+                        'en-vpn': {'id': 2, 'link': 'eno1'}}},
+                'bridge': {
+                    'bridges': {
+                        'br0': {
+                            'interfaces': ['wlp1s0', 'switchports'],
+                            'dhcp4': True}}}}
+        },
+    }
+
+    def _fmt_config(self, config_items):
+        res = {}
+        for item, item_confs in config_items.items():
+            version = item_confs['version']
+            res[item] = {'version': version}
+            if version == 1:
+                res[item]['config'] = [self.test_config[item][version][i]
+                                       for i in item_confs['items']]
+            elif version == 2 and item == 'network':
+                for cfg_item in item_confs['items']:
+                    res[item].update(self.test_config[item][version][cfg_item])
+            else:
+                raise NotImplementedError
+        return res
+
+    def _test_req_mappings(self, req_mappings):
+        for (config_items, expected_reqs) in req_mappings:
+            cfg = self._fmt_config(config_items)
+            actual_reqs = curthooks.detect_required_packages(cfg)
+            self.assertEqual(set(actual_reqs), set(expected_reqs),
+                             'failed for config: {}'.format(config_items))
+
+    def test_storage_v1_detect(self):
+        self._test_req_mappings((
+            ({'storage': {
+                'version': 1,
+                'items': ('lvm_partition', 'lvm_volgroup', 'btrfs', 'xfs')}},
+             ('lvm2', 'xfsprogs', 'btrfs-tools')),
+            ({'storage': {
+                'version': 1,
+                'items': ('raid', 'bcache', 'ext3', 'xfs')}},
+             ('mdadm', 'bcache-tools', 'e2fsprogs', 'xfsprogs')),
+            ({'storage': {
+                'version': 1,
+                'items': ('raid', 'lvm_volgroup', 'lvm_partition', 'ext3',
+                          'ext4', 'btrfs')}},
+             ('lvm2', 'mdadm', 'e2fsprogs', 'btrfs-tools')),
+            ({'storage': {
+                'version': 1,
+                'items': ('bcache', 'lvm_volgroup', 'lvm_partition', 'ext2')}},
+             ('bcache-tools', 'lvm2', 'e2fsprogs')),
+        ))
+
+    def test_network_v1_detect(self):
+        self._test_req_mappings((
+            ({'network': {
+                'version': 1,
+                'items': ('bridge',)}},
+             ('bridge-utils',)),
+            ({'network': {
+                'version': 1,
+                'items': ('vlan', 'bond')}},
+             ('vlan', 'ifenslave')),
+            ({'network': {
+                'version': 1,
+                'items': ('bond', 'bridge')}},
+             ('ifenslave', 'bridge-utils')),
+            ({'network': {
+                'version': 1,
+                'items': ('vlan', 'bridge', 'bond')}},
+             ('ifenslave', 'bridge-utils', 'vlan')),
+        ))
+
+    def test_mixed_v1_detect(self):
+        self._test_req_mappings((
+            ({'storage': {
+                'version': 1,
+                'items': ('raid', 'bcache', 'ext4')},
+              'network': {
+                  'version': 1,
+                  'items': ('vlan',)}},
+             ('mdadm', 'bcache-tools', 'e2fsprogs', 'vlan')),
+            ({'storage': {
+                'version': 1,
+                'items': ('lvm_partition', 'lvm_volgroup', 'xfs')},
+              'network': {
+                  'version': 1,
+                  'items': ('bridge', 'bond')}},
+             ('lvm2', 'xfsprogs', 'bridge-utils', 'ifenslave')),
+            ({'storage': {
+                'version': 1,
+                'items': ('ext3', 'ext4', 'btrfs')},
+              'network': {
+                  'version': 1,
+                  'items': ('bond', 'vlan')}},
+             ('e2fsprogs', 'btrfs-tools', 'vlan', 'ifenslave')),
+        ))
+
+    def test_network_v2_detect(self):
+        self._test_req_mappings((
+            ({'network': {
+                'version': 2,
+                'items': ('bridge',)}},
+             ('bridge-utils',)),
+            ({'network': {
+                'version': 2,
+                'items': ('vlan',)}},
+             ('vlan',)),
+            ({'network': {
+                'version': 2,
+                'items': ('vlan', 'bridge')}},
+             ('vlan', 'bridge-utils')),
+        ))
+
+    def test_mixed_storage_v1_network_v2_detect(self):
+        self._test_req_mappings((
+            ({'network': {
+                'version': 2,
+                'items': ('bridge', 'vlan')},
+             'storage': {
+                 'version': 1,
+                 'items': ('raid', 'bcache', 'ext4')}},
+             ('vlan', 'bridge-utils', 'mdadm', 'bcache-tools', 'e2fsprogs')),
+        ))
+
+    def test_invalid_version_in_config(self):
+        with self.assertRaises(ValueError):
+            curthooks.detect_required_packages({'network': {'version': 3}})
+
 
 # vi: ts=4 expandtab syntax=python
 
=== modified file 'tests/unittests/test_feature.py'
--- tests/unittests/test_feature.py	2017-06-12 20:39:06 +0000
+++ tests/unittests/test_feature.py	2017-10-06 16:35:22 +0000
@@ -1,9 +1,9 @@
-from unittest import TestCase
+from .helpers import CiTestCase
 
 import curtin
 
 
-class TestExportsFeatures(TestCase):
+class TestExportsFeatures(CiTestCase):
     def test_has_storage_v1(self):
         self.assertIn('STORAGE_CONFIG_V1', curtin.FEATURES)
 
@@ -15,3 +15,6 @@
 
     def test_has_reporting_events_webhook(self):
         self.assertIn('REPORTING_EVENTS_WEBHOOK', curtin.FEATURES)
+
+    def test_has_centos_apply_network_config(self):
+        self.assertIn('CENTOS_APPLY_NETWORK_CONFIG', curtin.FEATURES)
 
=== modified file 'tests/unittests/test_gpg.py'
--- tests/unittests/test_gpg.py	2017-02-08 22:22:44 +0000
+++ tests/unittests/test_gpg.py	2017-10-06 16:35:22 +0000
@@ -1,12 +1,12 @@
-from unittest import TestCase
 from mock import call, patch
 import textwrap
 
 from curtin import gpg
 from curtin import util
-
-
-class TestCurtinGpg(TestCase):
+from .helpers import CiTestCase
+
+
+class TestCurtinGpg(CiTestCase):
 
     @patch('curtin.util.subp')
     def test_export_armour(self, mock_subp):
 
=== modified file 'tests/unittests/test_make_dname.py'
--- tests/unittests/test_make_dname.py	2016-10-03 18:42:29 +0000
+++ tests/unittests/test_make_dname.py	2017-10-06 16:35:22 +0000
@@ -1,13 +1,13 @@
-from unittest import TestCase
 import mock
 
 import textwrap
 import uuid
 
 from curtin.commands import block_meta
-
-
-class TestMakeDname(TestCase):
+from .helpers import CiTestCase
+
+
+class TestMakeDname(CiTestCase):
     state = {'scratch': '/tmp/null'}
     rules_d = '/tmp/null/rules.d'
     rule_file = '/tmp/null/rules.d/{}.rules'
 
=== modified file 'tests/unittests/test_net.py'
--- tests/unittests/test_net.py	2017-03-01 16:13:56 +0000
+++ tests/unittests/test_net.py	2017-10-06 16:35:22 +0000
@@ -1,15 +1,14 @@
-from unittest import TestCase
+import mock
 import os
-import shutil
-import tempfile
 import yaml
 
-from curtin import net
+from curtin import config, net, util
 import curtin.net.network_state as network_state
+from .helpers import CiTestCase
 from textwrap import dedent
 
 
-class TestNetParserData(TestCase):
+class TestNetParserData(CiTestCase):
 
     def test_parse_deb_config_data_ignores_comments(self):
         contents = dedent("""\
@@ -234,13 +233,11 @@
             }, ifaces)
 
 
-class TestNetParser(TestCase):
+class TestNetParser(CiTestCase):
 
     def setUp(self):
-        self.target = tempfile.mkdtemp()
-
-    def tearDown(self):
-        shutil.rmtree(self.target)
+        super(TestNetParser, self).setUp()
+        self.target = self.tmp_dir()
 
     def make_config(self, path=None, name=None, contents=None,
                     parse=True):
@@ -386,9 +383,10 @@
         self.assertEqual({}, observed)
 
 
-class TestNetConfig(TestCase):
+class TestNetConfig(CiTestCase):
     def setUp(self):
-        self.target = tempfile.mkdtemp()
+        super(TestNetConfig, self).setUp()
+        self.target = self.tmp_dir()
         self.config_f = os.path.join(self.target, 'config')
         self.config = '''
 # YAML example of a simple network config
@@ -435,9 +433,6 @@
         ns.parse_config()
         return ns
 
-    def tearDown(self):
-        shutil.rmtree(self.target)
-
     def test_parse_net_config_data(self):
         ns = self.get_net_state()
         net_state_from_cls = ns.network_state
@@ -503,24 +498,19 @@
             auto interface1
             iface interface1 inet manual
                 bond-mode active-backup
-                bond-master bond0
+                bond-master bond1
 
             auto interface2
             iface interface2 inet manual
                 bond-mode active-backup
-                bond-master bond0
+                bond-master bond1
 
-            auto bond0
-            iface bond0 inet static
+            auto bond1
+            iface bond1 inet static
                 address 10.23.23.2/24
                 bond-mode active-backup
-                hwaddress ether 52:54:00:12:34:06
                 bond-slaves none
 
-            # control-alias bond0
-            iface bond0 inet static
-                address 10.23.24.2/24
-
             source /etc/network/interfaces.d/*.cfg
             """)
         net_ifaces = net.render_interfaces(ns.network_state)
@@ -654,6 +644,91 @@
         self.assertEqual(sorted(ifaces.split('\n')),
                          sorted(net_ifaces.split('\n')))
 
+    @mock.patch('curtin.util.subp')
+    @mock.patch('curtin.util.which')
+    @mock.patch.object(util.ChrootableTarget, "__enter__", new=lambda a: a)
+    def test_netconfig_passthrough_available(self, mock_which, mock_subp):
+        cloud_init = '/usr/bin/cloud-init'
+        mock_which.return_value = cloud_init
+        mock_subp.return_value = ("NETWORK_CONFIG_V1\nNETWORK_CONFIG_V2\n", '')
+
+        available = net.netconfig_passthrough_available(self.target)
+
+        self.assertEqual(True, available,
+                         "netconfig passthrough was NOT available")
+        mock_which.assert_called_with('cloud-init', target=self.target)
+        mock_subp.assert_called_with([cloud_init, 'features'],
+                                     capture=True, target=self.target)
+
+    @mock.patch('curtin.net.LOG')
+    @mock.patch('curtin.util.subp')
+    @mock.patch('curtin.util.which')
+    @mock.patch.object(util.ChrootableTarget, "__enter__", new=lambda a: a)
+    def test_netconfig_passthrough_available_no_cloudinit(self, mock_which,
+                                                          mock_subp, mock_log):
+        mock_which.return_value = None
+
+        available = net.netconfig_passthrough_available(self.target)
+
+        self.assertEqual(False, available,
+                         "netconfig passthrough was available")
+        self.assertTrue(mock_log.warning.called)
+        self.assertFalse(mock_subp.called)
+
+    @mock.patch('curtin.util.subp')
+    @mock.patch('curtin.util.which')
+    @mock.patch.object(util.ChrootableTarget, "__enter__", new=lambda a: a)
+    def test_netconfig_passthrough_available_feature_not_found(self,
+                                                               mock_which,
+                                                               mock_subp):
+        cloud_init = '/usr/bin/cloud-init'
+        mock_which.return_value = cloud_init
+        mock_subp.return_value = ('NETWORK_CONFIG_V1\n', '')
+
+        available = net.netconfig_passthrough_available(self.target)
+
+        self.assertEqual(False, available,
+                         "netconfig passthrough was available")
+        mock_which.assert_called_with('cloud-init', target=self.target)
+        mock_subp.assert_called_with([cloud_init, 'features'],
+                                     capture=True, target=self.target)
+
+    @mock.patch('curtin.net.LOG')
+    @mock.patch('curtin.util.subp')
+    @mock.patch('curtin.util.which')
+    @mock.patch.object(util.ChrootableTarget, "__enter__", new=lambda a: a)
+    def test_netconfig_passthrough_available_exc(self, mock_which, mock_subp,
+                                                 mock_log):
+        cloud_init = '/usr/bin/cloud-init'
+        mock_which.return_value = cloud_init
+        mock_subp.side_effect = util.ProcessExecutionError
+
+        available = net.netconfig_passthrough_available(self.target)
+
+        self.assertEqual(False, available,
+                         "netconfig passthrough was available")
+        mock_which.assert_called_with('cloud-init', target=self.target)
+        mock_subp.assert_called_with([cloud_init, 'features'],
+                                     capture=True, target=self.target)
+        self.assertTrue(mock_log.warning.called)
+
+    @mock.patch('curtin.util.write_file')
+    def test_render_netconfig_passthrough(self, mock_writefile):
+        netcfg = yaml.safe_load(self.config)
+        pt_config = 'etc/cloud/cloud.cfg.d/50-curtin-networking.cfg'
+        target_config = os.path.sep.join((self.target, pt_config),)
+
+        net.render_netconfig_passthrough(self.target, netconfig=netcfg)
+
+        content = config.dump_config(netcfg)
+        mock_writefile.assert_called_with(target_config, content=content)
+
+    def test_render_netconfig_passthrough_nonetcfg(self):
+        netcfg = None
+        self.assertRaises(ValueError,
+                          net.render_netconfig_passthrough,
+                          self.target, netconfig=netcfg)
+
     def test_routes_rendered(self):
         # as reported in bug 1649652
         conf = [
 
=== modified file 'tests/unittests/test_partitioning.py'
--- tests/unittests/test_partitioning.py	2015-10-02 16:19:07 +0000
+++ tests/unittests/test_partitioning.py	2017-10-06 16:35:22 +0000
@@ -1,6 +1,7 @@
-import unittest
+from unittest import skip
 import mock
 import curtin.commands.block_meta
+from .helpers import CiTestCase
 
 from sys import version_info
 if version_info.major == 2:
@@ -11,8 +12,8 @@
 parted = None  # FIXME: remove these tests entirely. This is here for flake8
 
 
[e-mail]
-class TestBlock(unittest.TestCase):
+@skip
+class TestBlock(CiTestCase):
     storage_config = {
         "sda": {"id": "sda", "type": "disk", "ptable": "msdos",
                 "serial": "DISK_1", "grub_device": "True"},
 
=== added file 'tests/unittests/test_public.py'
--- tests/unittests/test_public.py	1970-01-01 00:00:00 +0000
+++ tests/unittests/test_public.py	2017-10-06 16:35:22 +0000
@@ -0,0 +1,54 @@
+
+from curtin import block
+from curtin import config
+from curtin import futil
+from curtin import util
+
+from curtin.commands import curthooks
+from .helpers import CiTestCase
+
+
+class TestPublicAPI(CiTestCase):
+    """Test entry points known to be used externally.
+
+    Curtin's only known external library user is the curthooks
+    that are present in the MAAS images.  This will test for presense
+    of the modules and entry points that are used there.
+
+    This unit test is present to just test entry points.  Function
+    behavior should be present elsewhere."""
+
+    def assert_has_callables(self, module, expected):
+        self.assertEqual(expected, _module_has(module, expected, callable))
+
+    def test_block(self):
+        """Verify expected attributes in curtin.block."""
+        self.assert_has_callables(
+            block,
+            ['get_devices_for_mp', 'get_blockdev_for_partition', '_lsblock'])
+
+    def test_config(self):
+        """Verify exported attributes in curtin.config."""
+        self.assert_has_callables(config, ['load_config'])
+
+    def test_util(self):
+        """Verify exported attributes in curtin.util."""
+        self.assert_has_callables(
+            util, ['RunInChroot', 'load_command_environment'])
+
+    def test_centos_apply_network_config(self):
+        """MAAS images use centos_apply_network_config from cmd.curthooks."""
+        self.assert_has_callables(curthooks, ['centos_apply_network_config'])
+
+    def test_futil(self):
+        """Verify exported attributes in curtin.futil."""
+        self.assert_has_callables(futil, ['write_files'])
+
+
+def _module_has(module, names, nfilter=None):
+    found = [(name, getattr(module, name))
+             for name in names if hasattr(module, name)]
+    if nfilter is not None:
+        found = [(name, attr) for name, attr in found if nfilter(attr)]
+
+    return [name for name, _ in found]
 
=== modified file 'tests/unittests/test_reporter.py'
--- tests/unittests/test_reporter.py	2017-03-01 16:13:56 +0000
+++ tests/unittests/test_reporter.py	2017-10-06 16:35:22 +0000
@@ -21,7 +21,6 @@
     unicode_literals,
     )
 
-from unittest import TestCase
 from mock import patch
 
 from curtin.reporter.legacy import (
@@ -39,13 +38,12 @@
 from curtin.reporter import handlers
 from curtin import url_helper
 from curtin.reporter import events
+from .helpers import CiTestCase
 
-import os
-import tempfile
 import base64
 
 
-class TestLegacyReporter(TestCase):
+class TestLegacyReporter(CiTestCase):
 
     @patch('curtin.reporter.legacy.LOG')
     def test_load_reporter_logs_empty_cfg(self, mock_LOG):
@@ -72,7 +70,7 @@
         self.assertTrue(mock_LOG.error.called)
 
 
-class TestMAASReporter(TestCase):
+class TestMAASReporter(CiTestCase):
     def test_load_factory_raises_exception_wrong_options(self):
         options = {'wrong': 'wrong'}
         self.assertRaises(
@@ -86,7 +84,7 @@
         self.assertIsInstance(reporter, MAASReporter)
 
 
-class TestReporter(TestCase):
+class TestReporter(CiTestCase):
     config = {'element1': {'type': 'webhook', 'level': 'INFO',
                            'consumer_key': "ck_foo",
                            'consumer_secret': 'cs_foo',
@@ -175,39 +173,32 @@
     @patch('curtin.reporter.events.report_event')
     def test_report_finished_post_files(self, mock_report_event):
         test_data = b'abcdefg'
-        tmp = tempfile.mkstemp()
-        try:
-            with open(tmp[1], 'wb') as fp:
-                fp.write(test_data)
-            events.report_finish_event(self.ev_name, self.ev_desc,
-                                       post_files=[tmp[1]])
-            event = self._get_reported_event(mock_report_event)
-            files = event.as_dict().get('files')
-            self.assertTrue(len(files) == 1)
-            self.assertEqual(files[0].get('path'), tmp[1])
-            self.assertEqual(files[0].get('encoding'), 'base64')
-            self.assertEqual(files[0].get('content'),
-                             base64.b64encode(test_data).decode())
-        finally:
-            os.remove(tmp[1])
+        tmpfname = self.tmp_path('testfile')
+        with open(tmpfname, 'wb') as fp:
+            fp.write(test_data)
+        events.report_finish_event(self.ev_name, self.ev_desc,
+                                   post_files=[tmpfname])
+        event = self._get_reported_event(mock_report_event)
+        files = event.as_dict().get('files')
+        self.assertTrue(len(files) == 1)
+        self.assertEqual(files[0].get('path'), tmpfname)
+        self.assertEqual(files[0].get('encoding'), 'base64')
+        self.assertEqual(files[0].get('content'),
+                         base64.b64encode(test_data).decode())
 
     @patch('curtin.url_helper.OauthUrlHelper')
     def test_webhook_handler_post_files(self, mock_url_helper):
         test_data = b'abcdefg'
-        tmp = tempfile.mkstemp()
-        tmpfname = tmp[1]
-        try:
-            with open(tmpfname, 'wb') as fp:
-                fp.write(test_data)
-            event = events.FinishReportingEvent('test_event_name',
-                                                'test event description',
-                                                post_files=[tmpfname],
-                                                level='INFO')
-            webhook_handler = handlers.WebHookHandler('127.0.0.1:8000',
-                                                      level='INFO')
-            webhook_handler.publish_event(event)
-            webhook_handler.oauth_helper.geturl.assert_called_with(
-                url='127.0.0.1:8000', data=event.as_dict(),
-                headers=webhook_handler.headers, retries=None)
-        finally:
-            os.remove(tmpfname)
+        tmpfname = self.tmp_path('testfile')
+        with open(tmpfname, 'wb') as fp:
+            fp.write(test_data)
+        event = events.FinishReportingEvent('test_event_name',
+                                            'test event description',
+                                            post_files=[tmpfname],
+                                            level='INFO')
+        webhook_handler = handlers.WebHookHandler('127.0.0.1:8000',
+                                                  level='INFO')
+        webhook_handler.publish_event(event)
+        webhook_handler.oauth_helper.geturl.assert_called_with(
+            url='127.0.0.1:8000', data=event.as_dict(),
+            headers=webhook_handler.headers, retries=None)
 
=== modified file 'tests/unittests/test_util.py'
--- tests/unittests/test_util.py	2017-06-12 20:39:06 +0000
+++ tests/unittests/test_util.py	2017-10-06 16:35:22 +0000
@@ -1,16 +1,14 @@
-from unittest import TestCase, skipIf
+from unittest import skipIf
 import mock
 import os
 import stat
-import shutil
-import tempfile
 from textwrap import dedent
 
 from curtin import util
-from .helpers import simple_mocked_open
-
-
-class TestLogTimer(TestCase):
+from .helpers import CiTestCase, simple_mocked_open
+
+
+class TestLogTimer(CiTestCase):
     def test_logger_called(self):
         data = {}
 
@@ -24,16 +22,14 @@
         self.assertIn("mymessage", data['msg'])
 
 
-class TestDisableDaemons(TestCase):
+class TestDisableDaemons(CiTestCase):
     prcpath = "usr/sbin/policy-rc.d"
 
     def setUp(self):
-        self.target = tempfile.mkdtemp()
+        super(TestDisableDaemons, self).setUp()
+        self.target = self.tmp_dir()
         self.temp_prc = os.path.join(self.target, self.prcpath)
 
-    def tearDown(self):
-        shutil.rmtree(self.target)
-
     def test_disable_daemons_in_root_works(self):
         ret = util.disable_daemons_in_root(self.target)
         self.assertTrue(ret)
@@ -55,8 +51,10 @@
         self.assertTrue(os.path.exists(self.temp_prc))
 
 
-class TestWhich(TestCase):
+class TestWhich(CiTestCase):
+
     def setUp(self):
+        super(TestWhich, self).setUp()
         self.orig_is_exe = util.is_exe
         util.is_exe = self.my_is_exe
         self.orig_path = os.environ.get("PATH")
@@ -103,8 +101,10 @@
         self.assertEqual(found, "/usr/bin2/fuzz")
 
 
-class TestLsbRelease(TestCase):
+class TestLsbRelease(CiTestCase):
+
     def setUp(self):
+        super(TestLsbRelease, self).setUp()
         self._reset_cache()
 
     def _reset_cache(self):
@@ -143,7 +143,7 @@
         self.assertEqual(util.lsb_release(), expected)
 
 
-class TestSubp(TestCase):
+class TestSubp(CiTestCase):
 
     stdin2err = ['bash', '-c', 'cat >&2']
     stdin2out = ['cat']
@@ -160,6 +160,12 @@
         decode_type = str
         nodecode_type = bytes
 
+    def setUp(self):
+        super(TestSubp, self).setUp()
+        self.add_patch(
+            'curtin.util._get_unshare_pid_args', 'mock_get_unshare_pid_args',
+            return_value=[])
+
     def printf_cmd(self, *args):
         # bash's printf supports \xaa.  So does /usr/bin/printf
         # but by using bash, we remove dependency on another program.
@@ -296,12 +302,29 @@
         calls = m_popen.call_args_list
         popen_args, popen_kwargs = calls[-1]
         target = util.target_path(kwargs.get('target', None))
+        unshcmd = self.mock_get_unshare_pid_args.return_value
         if target == "/":
-            self.assertEqual(cmd, popen_args[0])
+            self.assertEqual(unshcmd + list(cmd), popen_args[0])
         else:
-            self.assertEqual(['chroot', target] + list(cmd), popen_args[0])
+            self.assertEqual(unshcmd + ['chroot', target] + list(cmd),
+                             popen_args[0])
         return calls
 
+    def test_args_can_be_a_tuple(self):
+        """subp can take a tuple for cmd rather than a list."""
+        my_cmd = tuple(['echo', 'hi', 'mom'])
+        calls = self._subp_wrap_popen(my_cmd, {})
+        args, kwargs = calls[0]
+        # subp was called with cmd as a tuple.  That may get converted to
+        # a list before subprocess.popen.  So only compare as lists.
+        self.assertEqual(1, len(calls))
+        self.assertEqual(list(my_cmd), list(args[0]))
+
+    def test_args_can_be_a_string(self):
+        """subp("cat") is acceptable, as suprocess.call("cat") works fine."""
+        out, err = util.subp("cat", data=b'hi mom', capture=True, decode=False)
+        self.assertEqual(b'hi mom', out)
+
     def test_with_target_gets_chroot(self):
         args, kwargs = self._subp_wrap_popen(["my-command"],
                                              {'target': "/mytarget"})[0]
@@ -342,8 +365,94 @@
         # since we fail a few times, it needs to have been called again.
         self.assertEqual(len(r), len(rcs))
 
-
-class TestHuman2Bytes(TestCase):
+    def test_unshare_pid_return_is_used(self):
+        """The return of _get_unshare_pid_return needs to be in command."""
+        my_unshare_cmd = ['do-unshare-command', 'arg0', 'arg1', '--']
+        self.mock_get_unshare_pid_args.return_value = my_unshare_cmd
+        my_kwargs = {'target': '/target', 'unshare_pid': True}
+        r = self._subp_wrap_popen(['apt-get', 'install'], my_kwargs)
+        self.assertEqual(1, len(r))
+        args, kwargs = r[0]
+        self.assertEqual(
+            [mock.call(my_kwargs['unshare_pid'], my_kwargs['target'])],
+            self.mock_get_unshare_pid_args.call_args_list)
+        expected = (my_unshare_cmd + ['chroot', '/target'] +
+                    ['apt-get', 'install'])
+        self.assertEqual(expected, args[0])
+
+
+class TestGetUnsharePidArgs(CiTestCase):
+    """Test the internal implementation for when to unshare."""
+
+    def setUp(self):
+        super(TestGetUnsharePidArgs, self).setUp()
+        self.add_patch('curtin.util._has_unshare_pid', 'mock_has_unshare_pid',
+                       return_value=True)
+        # our trusty tox environment with mock 1.0.1 will stack trace
+        # if autospec is not disabled here.
+        self.add_patch('curtin.util.os.geteuid', 'mock_geteuid',
+                       autospec=False, return_value=0)
+
+    def assertOff(self, result):
+        self.assertEqual([], result)
+
+    def assertOn(self, result):
+        self.assertEqual(['unshare', '--fork', '--pid', '--'], result)
+
+    def test_unshare_pid_none_and_not_root_means_off(self):
+        """If not root, then expect off."""
+        self.assertOff(util._get_unshare_pid_args(None, "/foo", 500))
+        self.assertOff(util._get_unshare_pid_args(None, "/", 500))
+
+        self.mock_geteuid.return_value = 500
+        self.assertOff(util._get_unshare_pid_args(None, "/"))
+        self.assertOff(
+            util._get_unshare_pid_args(unshare_pid=None, target="/foo"))
+
+    def test_unshare_pid_none_and_no_unshare_pid_means_off(self):
+        """No unshare support and unshare_pid is None means off."""
+        self.mock_has_unshare_pid.return_value = False
+        self.assertOff(util._get_unshare_pid_args(None, "/target", 0))
+
+    def test_unshare_pid_true_and_no_unshare_pid_raises(self):
+        """Passing unshare_pid in as True and no command should raise."""
+        self.mock_has_unshare_pid.return_value = False
+        expected_msg = 'no unshare command'
+        with self.assertRaisesRegexp(RuntimeError, expected_msg):
+            util._get_unshare_pid_args(True)
+
+        with self.assertRaisesRegexp(RuntimeError, expected_msg):
+            util._get_unshare_pid_args(True, "/foo", 0)
+
+    def test_unshare_pid_true_and_not_root_raises(self):
+        """When unshare_pid is True for non-root an error is raised."""
+        expected_msg = 'euid.* != 0'
+        with self.assertRaisesRegexp(RuntimeError, expected_msg):
+            util._get_unshare_pid_args(True, "/foo", 500)
+
+        self.mock_geteuid.return_value = 500
+        with self.assertRaisesRegexp(RuntimeError, expected_msg):
+            util._get_unshare_pid_args(True)
+
+    def test_euid0_target_not_slash(self):
+        """If root and target is not /, then expect on."""
+        self.assertOn(util._get_unshare_pid_args(None, target="/foo", euid=0))
+
+    def test_euid0_target_slash(self):
+        """If root and target is /, then expect off."""
+        self.assertOff(util._get_unshare_pid_args(None, "/", 0))
+        self.assertOff(util._get_unshare_pid_args(None, target=None, euid=0))
+
+    def test_unshare_pid_of_false_means_off(self):
+        """Any unshare_pid value false-ish other than None means no unshare."""
+        self.assertOff(
+            util._get_unshare_pid_args(unshare_pid=False, target=None))
+        self.assertOff(util._get_unshare_pid_args(False, "/target", 1))
+        self.assertOff(util._get_unshare_pid_args(False, "/", 0))
+        self.assertOff(util._get_unshare_pid_args("", "/target", 0))
+
+
+class TestHuman2Bytes(CiTestCase):
     GB = 1024 * 1024 * 1024
     MB = 1024 * 1024
 
@@ -397,52 +506,42 @@
                 util.bytes2human(util.human2bytes(size_str)), size_str)
 
 
-class TestSetUnExecutable(TestCase):
+class TestSetUnExecutable(CiTestCase):
     tmpf = None
     tmpd = None
 
-    def tearDown(self):
-        if self.tmpf:
-            if os.path.exists(self.tmpf):
-                os.unlink(self.tmpf)
-            self.tmpf = None
-        if self.tmpd:
-            shutil.rmtree(self.tmpd)
-            self.tmpd = None
-
-    def tempfile(self, data=None):
-        fp, self.tmpf = tempfile.mkstemp()
-        if data:
-            fp.write(data)
-        os.close(fp)
-        return self.tmpf
+    def setUp(self):
+        super(CiTestCase, self).setUp()
+        self.tmpd = self.tmp_dir()
 
     def test_change_needed_returns_original_mode(self):
-        tmpf = self.tempfile()
+        tmpf = self.tmp_path('testfile')
+        util.write_file(tmpf, '')
         os.chmod(tmpf, 0o755)
         ret = util.set_unexecutable(tmpf)
         self.assertEqual(ret, 0o0755)
 
     def test_no_change_needed_returns_none(self):
-        tmpf = self.tempfile()
+        tmpf = self.tmp_path('testfile')
+        util.write_file(tmpf, '')
         os.chmod(tmpf, 0o600)
         ret = util.set_unexecutable(tmpf)
         self.assertEqual(ret, None)
 
     def test_change_does_as_expected(self):
-        tmpf = self.tempfile()
+        tmpf = self.tmp_path('testfile')
+        util.write_file(tmpf, '')
         os.chmod(tmpf, 0o755)
         ret = util.set_unexecutable(tmpf)
         self.assertEqual(ret, 0o0755)
         self.assertEqual(stat.S_IMODE(os.stat(tmpf).st_mode), 0o0644)
 
     def test_strict_no_exists_raises_exception(self):
-        self.tmpd = tempfile.mkdtemp()
         bogus = os.path.join(self.tmpd, 'bogus')
         self.assertRaises(ValueError, util.set_unexecutable, bogus, True)
 
 
-class TestTargetPath(TestCase):
+class TestTargetPath(CiTestCase):
     def test_target_empty_string(self):
         self.assertEqual("/etc/passwd", util.target_path("", "/etc/passwd"))
 
@@ -484,7 +583,7 @@
                          util.target_path("/target/", "///my/path/"))
 
 
-class TestRunInChroot(TestCase):
+class TestRunInChroot(CiTestCase):
     """Test the legacy 'RunInChroot'.
 
     The test works by mocking ChrootableTarget's __enter__ to do nothing.
@@ -514,7 +613,7 @@
         m_subp.assert_called_with(cmd, target=target)
 
 
-class TestLoadFile(TestCase):
+class TestLoadFile(CiTestCase):
     """Test utility 'load_file'"""
 
     def test_load_file_simple(self):
@@ -545,7 +644,7 @@
             self.assertEqual(loaded_contents, contents)
 
 
-class TestIpAddress(TestCase):
+class TestIpAddress(CiTestCase):
     """Test utility 'is_valid_ip{,v4,v6}_address'"""
 
     def test_is_valid_ipv6_address(self):
@@ -570,10 +669,11 @@
             '2002:4559:1FE2:0000:0000:0000:4559:1FE2'))
 
 
-class TestLoadCommandEnvironment(TestCase):
+class TestLoadCommandEnvironment(CiTestCase):
+
     def setUp(self):
-        self.tmpd = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, self.tmpd)
+        super(TestLoadCommandEnvironment, self).setUp()
+        self.tmpd = self.tmp_dir()
         all_names = {
             'CONFIG',
             'OUTPUT_FSTAB',
@@ -616,7 +716,7 @@
             self.fail("unexpected key error raised: %s" % e)
 
 
-class TestWaitForRemoval(TestCase):
+class TestWaitForRemoval(CiTestCase):
     def test_wait_for_removal_missing_path(self):
         with self.assertRaises(ValueError):
             util.wait_for_removal(None)
@@ -684,14 +784,12 @@
         ])
 
 
-class TestGetEFIBootMGR(TestCase):
+class TestGetEFIBootMGR(CiTestCase):
 
     def setUp(self):
         super(TestGetEFIBootMGR, self).setUp()
-        mock_chroot = mock.patch(
-            'curtin.util.ChrootableTarget', autospec=False)
-        self.mock_chroot = mock_chroot.start()
-        self.addCleanup(mock_chroot.stop)
+        self.add_patch(
+            'curtin.util.ChrootableTarget', 'mock_chroot', autospec=False)
         self.mock_in_chroot = mock.MagicMock()
         self.mock_in_chroot.__enter__.return_value = self.mock_in_chroot
         self.in_chroot_subp_output = []
@@ -753,4 +851,55 @@
         }, observed)
 
 
+class TestUsesSystemd(CiTestCase):
+
+    def setUp(self):
+        super(TestUsesSystemd, self).setUp()
+        self._reset_cache()
+        self.add_patch('curtin.util.os.path.isdir', 'mock_isdir')
+
+    def _reset_cache(self):
+        util._USES_SYSTEMD = None
+
+    def test_uses_systemd_on_systemd(self):
+        """ Test that uses_systemd returns True if sdpath is a dir """
+        # systemd_enabled
+        self.mock_isdir.return_value = True
+        result = util.uses_systemd()
+        self.assertEqual(True, result)
+        self.assertEqual(1, len(self.mock_isdir.call_args_list))
+
+    def test_uses_systemd_cached(self):
+        """Test that we cache the uses_systemd result"""
+
+        # reset_cache should ensure it's unset
+        self.assertEqual(None, util._USES_SYSTEMD)
+
+        # systemd enabled
+        self.mock_isdir.return_value = True
+
+        # first time
+        first_result = util.uses_systemd()
+
+        # check the cache value
+        self.assertEqual(first_result, util._USES_SYSTEMD)
+
+        # second time
+        second_result = util.uses_systemd()
+
+        # results should match between tries
+        self.assertEqual(True, first_result)
+        self.assertEqual(True, second_result)
+
+        # isdir should only be called once
+        self.assertEqual(1, len(self.mock_isdir.call_args_list))
+
+    def test_uses_systemd_on_non_systemd(self):
+        """ Test that uses_systemd returns False if sdpath is not a dir """
+        # systemd not available
+        self.mock_isdir.return_value = False
+        result = util.uses_systemd()
+        self.assertEqual(False, result)
+
+
 # vi: ts=4 expandtab syntax=python
 
=== modified file 'tests/unittests/test_version.py'
--- tests/unittests/test_version.py	2017-02-08 22:22:44 +0000
+++ tests/unittests/test_version.py	2017-10-06 16:35:22 +0000
@@ -1,28 +1,16 @@
-from unittest import TestCase
 import mock
 import subprocess
 import os
 
 from curtin import version
 from curtin import __version__ as old_version
-
-
-class CurtinVersionBase(TestCase):
-    def setUp(self):
-        super(CurtinVersionBase, self).setUp()
-
-    def add_patch(self, target, attr):
-        """Patches specified target object and sets it as attr on test
-        instance also schedules cleanup"""
-        m = mock.patch(target, autospec=True)
-        p = m.start()
-        self.addCleanup(m.stop)
-        setattr(self, attr, p)
-
-
-class TestCurtinVersion(CurtinVersionBase):
-
-    def setUp(self):
+from .helpers import CiTestCase
+
+
+class TestCurtinVersion(CiTestCase):
+
+    def setUp(self):
+        super(TestCurtinVersion, self).setUp()
         self.add_patch('subprocess.check_output', 'mock_subp')
         self.add_patch('os.path', 'mock_path')
 
 
=== modified file 'tests/vmtests/__init__.py'
--- tests/vmtests/__init__.py	2017-06-12 20:39:06 +0000
+++ tests/vmtests/__init__.py	2017-10-06 16:35:22 +0000
@@ -38,6 +38,7 @@
 CURTIN_VMTEST_IMAGE_SYNC = os.environ.get("CURTIN_VMTEST_IMAGE_SYNC", "1")
 IMAGE_SYNCS = []
 TARGET_IMAGE_FORMAT = "raw"
+TAR_DISKS = bool(int(os.environ.get("CURTIN_VMTEST_TAR_DISKS", "0")))
 
 
 DEFAULT_BRIDGE = os.environ.get("CURTIN_VMTEST_BRIDGE", "user")
@@ -335,7 +336,12 @@
     __test__ = False
     arch_skip = []
     boot_timeout = BOOT_TIMEOUT
-    collect_scripts = []
+    collect_scripts = [textwrap.dedent("""
+        cd OUTPUT_COLLECT_D
+        dpkg-query --show \
+            --showformat='${db:Status-Abbrev}\t${Package}\t${Version}\n' \
+            > debian-packages.txt 2> debian-packages.txt.err
+    """)]
     conf_file = "examples/tests/basic.yaml"
     nr_cpus = None
     dirty_disks = False
@@ -368,6 +374,8 @@
     target_krel = None
     target_ftype = "vmtest.root-tgz"
 
+    _debian_packages = None
+
     def shortDescription(self):
         return None
 
@@ -593,7 +601,7 @@
             logger.debug("Interface name: {}".format(ifname))
             iface = interfaces.get(ifname)
             hwaddr = iface.get('mac_address')
-            if hwaddr:
+            if iface['type'] == 'physical' and hwaddr:
                 macs.append(hwaddr)
         netdevs = []
         if len(macs) > 0:
@@ -685,6 +693,12 @@
             configs.append(excfg)
             logger.debug('Added extra config {}'.format(excfg))
 
+        if cls.target_distro == "centos":
+            centos_default = 'examples/tests/centos_defaults.yaml'
+            configs.append(centos_default)
+            logger.info('Detected centos, adding default config %s',
+                        centos_default)
+
         if cls.multipath:
             disks = disks * cls.multipath_num_paths
 
@@ -871,8 +885,11 @@
             raise
 
         # capture curtin install log and webhook timings
-        util.subp(["tools/curtin-log-print", "--dumpfiles", cls.td.logs,
-                   cls.reporting_log], capture=True)
+        try:
+            util.subp(["tools/curtin-log-print", "--dumpfiles", cls.td.logs,
+                      cls.reporting_log], capture=True)
+        except util.ProcessExecutionError as error:
+            logger.debug('tools/curtin-log-print failed: %s', error)
 
         logger.info(
             "%s: setUpClass finished. took %.02f seconds. Running testcases.",
@@ -929,7 +946,8 @@
         clean_working_dir(cls.td.tmpdir, success,
                           keep_pass=KEEP_DATA['pass'],
                           keep_fail=KEEP_DATA['fail'])
-
+        if TAR_DISKS:
+            tar_disks(cls.td.tmpdir)
         cls.cleanIscsiState(success,
                             keep_pass=KEEP_DATA['pass'],
                             keep_fail=KEEP_DATA['fail'])
@@ -1143,6 +1161,18 @@
                 fp.write(json.dumps(data, indent=2, sort_keys=True,
                                     separators=(',', ': ')) + "\n")
 
+    @property
+    def debian_packages(self):
+        if self._debian_packages is None:
+            data = self.load_collect_file("debian-packages.txt")
+            pkgs = {}
+            for line in data.splitlines():
+                # lines are <status>\t<
+                status, pkg, ver = line.split('\t')
+                pkgs[pkg] = {'status': status, 'version': ver}
+            self._debian_packages = pkgs
+        return self._debian_packages
+
 
 class PsuedoVMBaseClass(VMBaseClass):
     # This mimics much of the VMBaseClass just with faster setUpClass
@@ -1332,8 +1362,13 @@
     output_device = '/dev/disk/by-id/virtio-%s' % OUTPUT_DISK_NAME
 
     collect_prep = textwrap.dedent("mkdir -p " + output_dir)
-    collect_post = textwrap.dedent(
-        'tar -C "%s" -cf "%s" .' % (output_dir, output_device))
+    collect_post = textwrap.dedent("""\
+        cd {output_dir}\n
+        # remove any symlinks, but archive information about them.
+        # %Y target's file type, %P = path, %l = target of symlink
+        find -type l -printf "%Y\t%P\t%l\n" -delete > symlinks.txt
+        tar -cf "{output_device}" .
+        """).format(output_dir=output_dir, output_device=output_device)
 
     # copy /root for curtin config and install.log
     copy_rootdir = textwrap.dedent("cp -a /root " + output_dir)
@@ -1410,6 +1445,23 @@
     KEEP_DATA.update(data)
 
 
+def tar_disks(tmpdir, outfile="disks.tar", diskmatch=".img"):
+    """ Tar up files in ``tmpdir``/disks that ends with the pattern supplied"""
+
+    disks_dir = os.path.join(tmpdir, "disks")
+    if os.path.exists(disks_dir):
+        outfile = os.path.join(disks_dir, outfile)
+        disks = [os.path.join(disks_dir, disk) for disk in
+                 os.listdir(disks_dir) if disk.endswith(diskmatch)]
+        cmd = ["tar", "--create", "--file=%s" % outfile,
+               "--verbose", "--remove-files", "--sparse"]
+        cmd.extend(disks)
+        logger.info('Taring %s disks sparsely to %s', len(disks), outfile)
+        util.subp(cmd, capture=True)
+    else:
+        logger.error('Failed to find "disks" dir under tmpdir: %s', tmpdir)
+
+
 def boot_log_wrap(name, func, cmd, console_log, timeout, purpose):
     logger.debug("%s[%s]: booting with timeout=%s log=%s cmd: %s",
                  name, purpose, timeout, console_log, ' '.join(cmd))
 
=== modified file 'tests/vmtests/releases.py'
--- tests/vmtests/releases.py	2017-06-12 20:39:06 +0000
+++ tests/vmtests/releases.py	2017-10-06 16:35:22 +0000
@@ -77,22 +77,10 @@
     target_release = "trusty"
 
 
-class _VividBase(_UbuntuBase):
-    release = "vivid"
-
-
-class _WilyBase(_UbuntuBase):
-    release = "wily"
-
-
 class _XenialBase(_UbuntuBase):
     release = "xenial"
 
 
-class _YakketyBase(_UbuntuBase):
-    release = "yakkety"
-
-
 class _ZestyBase(_UbuntuBase):
     release = "zesty"
 
@@ -110,10 +98,7 @@
     trusty_hwe_w = _TrustyHWEW
     trusty_hwe_x = _TrustyHWEX
     trustyfromxenial = _TrustyFromXenial
-    vivid = _VividBase
-    wily = _WilyBase
     xenial = _XenialBase
-    yakkety = _YakketyBase
     zesty = _ZestyBase
     artful = _ArtfulBase
 
 
=== modified file 'tests/vmtests/test_apt_config_cmd.py'
--- tests/vmtests/test_apt_config_cmd.py	2017-06-12 20:39:06 +0000
+++ tests/vmtests/test_apt_config_cmd.py	2017-10-06 16:35:22 +0000
@@ -55,10 +55,6 @@
     __test__ = True
 
 
-class YakketyTestAptConfigCMDCMD(relbase.yakkety, TestAptConfigCMD):
-    __test__ = True
-
-
 class ZestyTestAptConfigCMDCMD(relbase.zesty, TestAptConfigCMD):
     __test__ = True
 
 
=== modified file 'tests/vmtests/test_basic.py'
--- tests/vmtests/test_basic.py	2017-06-12 20:39:06 +0000
+++ tests/vmtests/test_basic.py	2017-10-06 16:35:22 +0000
@@ -202,19 +202,10 @@
     __test__ = True
 
 
-class WilyTestBasic(relbase.wily, TestBasicAbs):
-    # EOL - 2016-07-28
-    __test__ = False
-
-
 class XenialTestBasic(relbase.xenial, TestBasicAbs):
     __test__ = True
 
 
-class YakketyTestBasic(relbase.yakkety, TestBasicAbs):
-    __test__ = True
-
-
 class ZestyTestBasic(relbase.zesty, TestBasicAbs):
     __test__ = True
 
@@ -323,10 +314,6 @@
     __test__ = True
 
 
-class YakketyTestScsiBasic(relbase.yakkety, TestBasicScsiAbs):
-    __test__ = True
-
-
 class ZestyTestScsiBasic(relbase.zesty, TestBasicScsiAbs):
     __test__ = True
 
 
=== modified file 'tests/vmtests/test_bcache_basic.py'
--- tests/vmtests/test_bcache_basic.py	2017-06-12 20:39:06 +0000
+++ tests/vmtests/test_bcache_basic.py	2017-10-06 16:35:22 +0000
@@ -59,10 +59,6 @@
     __test__ = True
 
 
-class YakketyBcacheBasic(relbase.yakkety, TestBcacheBasic):
-    __test__ = True
-
-
 class ZestyBcacheBasic(relbase.zesty, TestBcacheBasic):
     __test__ = True
 
 
=== modified file 'tests/vmtests/test_centos_basic.py'
--- tests/vmtests/test_centos_basic.py	2017-01-18 16:01:35 +0000
+++ tests/vmtests/test_centos_basic.py	2017-10-06 16:35:22 +0000
@@ -1,5 +1,6 @@
 from . import VMBaseClass
 from .releases import centos_base_vm_classes as relbase
+from .test_network import TestNetworkBaseTestsAbs
 
 import textwrap
 
@@ -9,10 +10,20 @@
     __test__ = False
     conf_file = "examples/tests/centos_basic.yaml"
     extra_kern_args = "BOOTIF=eth0-52:54:00:12:34:00"
+    # XXX: command | tee output is required for Centos under SELinux
+    # http://danwalsh.livejournal.com/22860.html
     collect_scripts = [textwrap.dedent(
         """
         cd OUTPUT_COLLECT_D
         cat /etc/fstab > fstab
+        rpm -qa | cat >rpm_qa
+        ifconfig -a | cat >ifconfig_a
+        ip a | cat >ip_a
+        cp -a /etc/sysconfig/network-scripts .
+        cp -a /var/log/messages .
+        cp -a /var/log/cloud-init* .
+        cp -a /var/lib/cloud ./var_lib_cloud
+        cp -a /run/cloud-init ./run_cloud-init
         """)]
     fstab_expected = {
         'LABEL=cloudimg-rootfs': '/',
@@ -40,3 +51,27 @@
     # FIXME: test is disabled because the grub config script in target
     #        specifies drive using hd(1,0) syntax, which breaks when the
     #        installation medium is removed. other than this, the install works
+
+
+class CentosTestBasicNetworkAbs(TestNetworkBaseTestsAbs):
+    conf_file = "examples/tests/centos_basic.yaml"
+    extra_kern_args = "BOOTIF=eth0-52:54:00:12:34:00"
+    collect_scripts = TestNetworkBaseTestsAbs.collect_scripts + [
+        textwrap.dedent("""
+            cd OUTPUT_COLLECT_D
+            cp -a /etc/sysconfig/network-scripts .
+            cp -a /var/log/cloud-init* .
+            cp -a /var/lib/cloud ./var_lib_cloud
+            cp -a /run/cloud-init ./run_cloud-init
+        """)]
+
+    def test_etc_network_interfaces(self):
+        pass
+
+    def test_etc_resolvconf(self):
+        pass
+
+
+class Centos70BasicNetworkFromXenialTestBasic(relbase.centos70fromxenial,
+                                              CentosTestBasicNetworkAbs):
+    __test__ = True
 
=== modified file 'tests/vmtests/test_iscsi.py'
--- tests/vmtests/test_iscsi.py	2017-06-12 20:39:06 +0000
+++ tests/vmtests/test_iscsi.py	2017-10-06 16:35:22 +0000
@@ -59,10 +59,6 @@
     __test__ = True
 
 
-class YakketyTestIscsiBasic(relbase.yakkety, TestBasicIscsiAbs):
-    __test__ = True
-
-
 class ZestyTestIscsiBasic(relbase.zesty, TestBasicIscsiAbs):
     __test__ = True
 
 
=== added file 'tests/vmtests/test_journald_reporter.py'
--- tests/vmtests/test_journald_reporter.py	1970-01-01 00:00:00 +0000
+++ tests/vmtests/test_journald_reporter.py	2017-10-06 16:35:22 +0000
@@ -0,0 +1,52 @@
+from . import VMBaseClass