Merge ~ubuntu-server/cloud-init:derived-repo-v3-new-apt-spec into cloud-init:master

[Scott Moser]

Apt: add new apt configuration format

This adds an improved apt configuration format that is fully backwards
compatible with previous behavior. This is mostly copied from curtin's
implementation.

It does:
 * clean up and centralizes many of the top level 'apt_*' values that
   previously existed into a single top level 'apt'key.
 * support a 'source' in apt/sources/entry that has only a key
 * documents new features and adds tests.

See the added doc/examples/cloud-config-apt.txt for more information.

b7d9a53... by Scott Moser on 2016-08-18

fix error in disable_suites if empty lines in sources.

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 05ad4b0..609dbb5 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -23,80 +23,182 @@ import os
 import re
 
 from cloudinit import gpg
+from cloudinit import log as logging
 from cloudinit import templater
 from cloudinit import util
 
-distros = ['ubuntu', 'debian']
-
-PROXY_TPL = "Acquire::HTTP::Proxy \"%s\";\n"
-APT_CONFIG_FN = "/etc/apt/apt.conf.d/94cloud-init-config"
-APT_PROXY_FN = "/etc/apt/apt.conf.d/95cloud-init-proxy"
+LOG = logging.getLogger(__name__)
 
 # this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar')
 ADD_APT_REPO_MATCH = r"^[\w-]+:\w"
 
+# place where apt stores cached repository data
+APT_LISTS = "/var/lib/apt/lists"
 
-def handle(name, cfg, cloud, log, _args):
-    if util.is_false(cfg.get('apt_configure_enabled', True)):
-        log.debug("Skipping module named %s, disabled by config.", name)
-        return
-
-    release = get_release()
-    mirrors = find_apt_mirror_info(cloud, cfg)
-    if not mirrors or "primary" not in mirrors:
-        log.debug(("Skipping module named %s,"
-                   " no package 'mirror' located"), name)
-        return
-
-    # backwards compatibility
-    mirror = mirrors["primary"]
-    mirrors["mirror"] = mirror
-
-    log.debug("Mirror info: %s" % mirrors)
-
-    if not util.get_cfg_option_bool(cfg,
-                                    'apt_preserve_sources_list', False):
-        generate_sources_list(cfg, release, mirrors, cloud, log)
-        old_mirrors = cfg.get('apt_old_mirrors',
-                              {"primary": "archive.ubuntu.com/ubuntu",
-                               "security": "security.ubuntu.com/ubuntu"})
-        rename_apt_lists(old_mirrors, mirrors)
+# Files to store proxy information
+APT_CONFIG_FN = "/etc/apt/apt.conf.d/94cloud-init-config"
+APT_PROXY_FN = "/etc/apt/apt.conf.d/90cloud-init-aptproxy"
+
+# Default keyserver to use
+DEFAULT_KEYSERVER = "keyserver.ubuntu.com"
+
+# Default archive mirrors
+PRIMARY_ARCH_MIRRORS = {"PRIMARY": "http://archive.ubuntu.com/ubuntu/",
+                        "SECURITY": "http://security.ubuntu.com/ubuntu/"}
+PORTS_MIRRORS = {"PRIMARY": "http://ports.ubuntu.com/ubuntu-ports",
+                 "SECURITY": "http://ports.ubuntu.com/ubuntu-ports"}
+PRIMARY_ARCHES = ['amd64', 'i386']
+PORTS_ARCHES = ['s390x', 'arm64', 'armhf', 'powerpc', 'ppc64el']
+
+
+def get_default_mirrors(arch=None, target=None):
+    """returns the default mirrors for the target. These depend on the
+       architecture, for more see:
+       https://wiki.ubuntu.com/UbuntuDevelopment/PackageArchive#Ports"""
+    if arch is None:
+        arch = util.get_architecture(target)
+    if arch in PRIMARY_ARCHES:
+        return PRIMARY_ARCH_MIRRORS.copy()
+    if arch in PORTS_ARCHES:
+        return PORTS_MIRRORS.copy()
+    raise ValueError("No default mirror known for arch %s" % arch)
+
+
+def handle(name, ocfg, cloud, log, _):
+    """process the config for apt_config. This can be called from
+       curthooks if a global apt config was provided or via the "apt"
+       standalone command."""
+    # keeping code close to curtin codebase via entry handler
+    target = None
+    if log is not None:
+        global LOG
+        LOG = log
+    # feed back converted config, but only work on the subset under 'apt'
+    ocfg = convert_to_v3_apt_format(ocfg)
+    cfg = ocfg.get('apt', {})
+
+    if not isinstance(cfg, dict):
+        raise ValueError("Expected dictionary for 'apt' config, found %s",
+                         type(cfg))
+
+    LOG.debug("handling apt (module %s) with apt config '%s'", name, cfg)
+
+    release = util.lsb_release(target=target)['codename']
+    arch = util.get_architecture(target)
+    mirrors = find_apt_mirror_info(cfg, cloud, arch=arch)
+    LOG.debug("Apt Mirror info: %s", mirrors)
+
+    apply_debconf_selections(cfg, target)
+
+    if util.is_false(cfg.get('preserve_sources_list', False)):
+        generate_sources_list(cfg, release, mirrors, cloud)
+        rename_apt_lists(mirrors, target)
 
     try:
         apply_apt_config(cfg, APT_PROXY_FN, APT_CONFIG_FN)
-    except Exception as e:
-        log.warn("failed to proxy or apt config info: %s", e)
+    except (IOError, OSError):
+        LOG.exception("Failed to apply proxy or apt config info:")
 
-    # Process 'apt_sources'
-    if 'apt_sources' in cfg:
+    # Process 'apt_source -> sources {dict}'
+    if 'sources' in cfg:
         params = mirrors
         params['RELEASE'] = release
-        params['MIRROR'] = mirror
+        params['MIRROR'] = mirrors["MIRROR"]
 
+        matcher = None
         matchcfg = cfg.get('add_apt_repo_match', ADD_APT_REPO_MATCH)
         if matchcfg:
             matcher = re.compile(matchcfg).search
+
+        add_apt_sources(cfg['sources'], cloud, target=target,
+                        template_params=params, aa_repo_match=matcher)
+
+
+def debconf_set_selections(selections, target=None):
+    util.subp(['debconf-set-selections'], data=selections, target=target,
+              capture=True)
+
+
+def dpkg_reconfigure(packages, target=None):
+    # For any packages that are already installed, but have preseed data
+    # we populate the debconf database, but the filesystem configuration
+    # would be preferred on a subsequent dpkg-reconfigure.
+    # so, what we have to do is "know" information about certain packages
+    # to unconfigure them.
+    unhandled = []
+    to_config = []
+    for pkg in packages:
+        if pkg in CONFIG_CLEANERS:
+            LOG.debug("unconfiguring %s", pkg)
+            CONFIG_CLEANERS[pkg](target)
+            to_config.append(pkg)
         else:
-            def matcher(x):
-                return False
+            unhandled.append(pkg)
+
+    if len(unhandled):
+        LOG.warn("The following packages were installed and preseeded, "
+                 "but cannot be unconfigured: %s", unhandled)
+
+    if len(to_config):
+        util.subp(['dpkg-reconfigure', '--frontend=noninteractive'] +
+                  list(to_config), data=None, target=target, capture=True)
+
+
+def apply_debconf_selections(cfg, target=None):
+    """apply_debconf_selections - push content to debconf"""
+    # debconf_selections:
+    #  set1: |
+    #   cloud-init cloud-init/datasources multiselect MAAS
+    #  set2: pkg pkg/value string bar
+    selsets = cfg.get('debconf_selections')
+    if not selsets:
+        LOG.debug("debconf_selections was not set in config")
+        return
 
-        errors = add_apt_sources(cfg['apt_sources'], params,
-                                 aa_repo_match=matcher)
-        for e in errors:
-            log.warn("Add source error: %s", ':'.join(e))
+    selections = '\n'.join(
+        [selsets[key] for key in sorted(selsets.keys())])
+    debconf_set_selections(selections.encode() + b"\n", target=target)
 
-    dconf_sel = util.get_cfg_option_str(cfg, 'debconf_selections', False)
-    if dconf_sel:
-        log.debug("Setting debconf selections per cloud config")
-        try:
-            util.subp(('debconf-set-selections', '-'), dconf_sel)
-        except Exception:
-            util.logexc(log, "Failed to run debconf-set-selections")
+    # get a complete list of packages listed in input
+    pkgs_cfgd = set()
+    for key, content in selsets.items():
+        for line in content.splitlines():
+            if line.startswith("#"):
+                continue
+            pkg = re.sub(r"[:\s].*", "", line)
+            pkgs_cfgd.add(pkg)
+
+    pkgs_installed = util.get_installed_packages(target)
+
+    LOG.debug("pkgs_cfgd: %s", pkgs_cfgd)
+    need_reconfig = pkgs_cfgd.intersection(pkgs_installed)
+
+    if len(need_reconfig) == 0:
+        LOG.debug("no need for reconfig")
+        return
+
+    dpkg_reconfigure(need_reconfig, target=target)
+
+
+def clean_cloud_init(target):
+    """clean out any local cloud-init config"""
+    flist = glob.glob(
+        util.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*"))
+
+    LOG.debug("cleaning cloud-init config from: %s", flist)
+    for dpkg_cfg in flist:
+        os.unlink(dpkg_cfg)
 
 
 def mirrorurl_to_apt_fileprefix(mirror):
+    """mirrorurl_to_apt_fileprefix
+       Convert a mirror url to the file prefix used by apt on disk to
+       store cache information for that mirror.
+       To do so do:
+       - take off ???://
+       - drop tailing /
+       - convert in string / to _"""
     string = mirror
-    # take off http:// or ftp://
     if string.endswith("/"):
         string = string[0:-1]
     pos = string.find("://")
@@ -106,174 +208,365 @@ def mirrorurl_to_apt_fileprefix(mirror):
     return string
 
 
-def rename_apt_lists(old_mirrors, new_mirrors, lists_d="/var/lib/apt/lists"):
-    for (name, omirror) in old_mirrors.items():
+def rename_apt_lists(new_mirrors, target=None):
+    """rename_apt_lists - rename apt lists to preserve old cache data"""
+    default_mirrors = get_default_mirrors(util.get_architecture(target))
+
+    pre = util.target_path(target, APT_LISTS)
+    for (name, omirror) in default_mirrors.items():
         nmirror = new_mirrors.get(name)
         if not nmirror:
             continue
-        oprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(omirror))
-        nprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(nmirror))
+
+        oprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(omirror)
+        nprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(nmirror)
         if oprefix == nprefix:
             continue
         olen = len(oprefix)
         for filename in glob.glob("%s_*" % oprefix):
-            util.rename(filename, "%s%s" % (nprefix, filename[olen:]))
-
-
-def get_release():
-    (stdout, _stderr) = util.subp(['lsb_release', '-cs'])
-    return stdout.strip()
-
-
-def generate_sources_list(cfg, codename, mirrors, cloud, log):
-    params = {'codename': codename}
+            newname = "%s%s" % (nprefix, filename[olen:])
+            LOG.debug("Renaming apt list %s to %s", filename, newname)
+            try:
+                os.rename(filename, newname)
+            except OSError:
+                # since this is a best effort task, warn with but don't fail
+                LOG.warn("Failed to rename apt list:", exc_info=True)
+
+
+def mirror_to_placeholder(tmpl, mirror, placeholder):
+    """mirror_to_placeholder
+       replace the specified mirror in a template with a placeholder string
+       Checks for existance of the expected mirror and warns if not found"""
+    if mirror not in tmpl:
+        LOG.warn("Expected mirror '%s' not found in: %s", mirror, tmpl)
+    return tmpl.replace(mirror, placeholder)
+
+
+def map_known_suites(suite):
+    """there are a few default names which will be auto-extended.
+       This comes at the inability to use those names literally as suites,
+       but on the other hand increases readability of the cfg quite a lot"""
+    mapping = {'updates': '$RELEASE-updates',
+               'backports': '$RELEASE-backports',
+               'security': '$RELEASE-security',
+               'proposed': '$RELEASE-proposed',
+               'release': '$RELEASE'}
+    try:
+        retsuite = mapping[suite]
+    except KeyError:
+        retsuite = suite
+    return retsuite
+
+
+def disable_suites(disabled, src, release):
+    """reads the config for suites to be disabled and removes those
+       from the template"""
+    if not disabled:
+        return src
+
+    retsrc = src
+    for suite in disabled:
+        suite = map_known_suites(suite)
+        releasesuite = templater.render_string(suite, {'RELEASE': release})
+        LOG.debug("Disabling suite %s as %s", suite, releasesuite)
+
+        newsrc = ""
+        for line in retsrc.splitlines(True):
+            if line.startswith("#"):
+                newsrc += line
+                continue
+
+            # sources.list allow options in cols[1] which can have spaces
+            # so the actual suite can be [2] or later. example:
+            # deb [ arch=amd64,armel k=v ] http://example.com/debian
+            cols = line.split()
+            if len(cols) > 1:
+                pcol = 2
+                if cols[1].startswith("["):
+                    for col in cols[1:]:
+                        pcol += 1
+                        if col.endswith("]"):
+                            break
+
+                if cols[pcol] == releasesuite:
+                    line = '# suite disabled by cloud-init: %s' % line
+            newsrc += line
+        retsrc = newsrc
+
+    return retsrc
+
+
+def generate_sources_list(cfg, release, mirrors, cloud):
+    """generate_sources_list
+       create a source.list file based on a custom or default template
+       by replacing mirrors and release in the template"""
+    aptsrc = "/etc/apt/sources.list"
+    params = {'RELEASE': release, 'codename': release}
     for k in mirrors:
         params[k] = mirrors[k]
+        params[k.lower()] = mirrors[k]
 
-    custtmpl = cfg.get('apt_custom_sources_list', None)
-    if custtmpl is not None:
-        templater.render_string_to_file(custtmpl,
-                                        '/etc/apt/sources.list', params)
-        return
-
-    template_fn = cloud.get_template_filename('sources.list.%s' %
-                                              (cloud.distro.name))
-    if not template_fn:
-        template_fn = cloud.get_template_filename('sources.list')
+    tmpl = cfg.get('sources_list', None)
+    if tmpl is None:
+        LOG.info("No custom template provided, fall back to builtin")
+        template_fn = cloud.get_template_filename('sources.list.%s' %
+                                                  (cloud.distro.name))
         if not template_fn:
-            log.warn("No template found, not rendering /etc/apt/sources.list")
+            template_fn = cloud.get_template_filename('sources.list')
+        if not template_fn:
+            LOG.warn("No template found, not rendering /etc/apt/sources.list")
             return
+        tmpl = util.load_file(template_fn)
 
-    templater.render_to_file(template_fn, '/etc/apt/sources.list', params)
+    rendered = templater.render_string(tmpl, params)
+    disabled = disable_suites(cfg.get('disable_suites'), rendered, release)
+    util.write_file(aptsrc, disabled, mode=0o644)
 
 
-def add_apt_key_raw(key):
+def add_apt_key_raw(key, target=None):
     """
     actual adding of a key as defined in key argument
     to the system
     """
+    LOG.debug("Adding key:\n'%s'", key)
     try:
-        util.subp(('apt-key', 'add', '-'), key)
+        util.subp(['apt-key', 'add', '-'], data=key.encode(), target=target)
     except util.ProcessExecutionError:
-        raise ValueError('failed to add apt GPG Key to apt keyring')
+        LOG.exception("failed to add apt GPG Key to apt keyring")
+        raise
 
 
-def add_apt_key(ent):
+def add_apt_key(ent, target=None):
     """
-    add key to the system as defined in ent (if any)
-    supports raw keys or keyid's
-    The latter will as a first step fetch the raw key from a keyserver
+    Add key to the system as defined in ent (if any).
+    Supports raw keys or keyid's
+    The latter will as a first step fetched to get the raw key
     """
     if 'keyid' in ent and 'key' not in ent:
-        keyserver = "keyserver.ubuntu.com"
+        keyserver = DEFAULT_KEYSERVER
         if 'keyserver' in ent:
             keyserver = ent['keyserver']
-        ent['key'] = gpg.get_key_by_id(ent['keyid'], keyserver)
+
+        ent['key'] = gpg.getkeybyid(ent['keyid'], keyserver)
 
     if 'key' in ent:
-        add_apt_key_raw(ent['key'])
+        add_apt_key_raw(ent['key'], target)
 
 
-def convert_to_new_format(srclist):
-    """convert_to_new_format
-       convert the old list based format to the new dict based one
-    """
-    srcdict = {}
-    if isinstance(srclist, list):
-        for srcent in srclist:
-            if 'filename' not in srcent:
-                # file collides for multiple !filename cases for compatibility
-                # yet we need them all processed, so not same dictionary key
-                srcent['filename'] = "cloud_config_sources.list"
-                key = util.rand_dict_key(srcdict, "cloud_config_sources.list")
-            else:
-                # all with filename use that as key (matching new format)
-                key = srcent['filename']
-            srcdict[key] = srcent
-    elif isinstance(srclist, dict):
-        srcdict = srclist
-    else:
-        raise ValueError("unknown apt_sources format")
-
-    return srcdict
+def update_packages(cloud):
+    cloud.distro.update_package_sources()
 
 
-def add_apt_sources(srclist, template_params=None, aa_repo_match=None):
+def add_apt_sources(srcdict, cloud, target=None, template_params=None,
+                    aa_repo_match=None):
     """
     add entries in /etc/apt/sources.list.d for each abbreviated
-    sources.list entry in 'srclist'.  When rendering template, also
+    sources.list entry in 'srcdict'.  When rendering template, also
     include the values in dictionary searchList
     """
     if template_params is None:
         template_params = {}
 
     if aa_repo_match is None:
-        def _aa_repo_match(x):
-            return False
-        aa_repo_match = _aa_repo_match
+        raise ValueError('did not get a valid repo matcher')
 
-    errorlist = []
-    srcdict = convert_to_new_format(srclist)
+    if not isinstance(srcdict, dict):
+        raise TypeError('unknown apt format: %s' % (srcdict))
 
     for filename in srcdict:
         ent = srcdict[filename]
+        LOG.debug("adding source/key '%s'", ent)
         if 'filename' not in ent:
             ent['filename'] = filename
 
-        # keys can be added without specifying a source
-        try:
-            add_apt_key(ent)
-        except ValueError as detail:
-            errorlist.append([ent, detail])
+        add_apt_key(ent, target)
 
         if 'source' not in ent:
-            errorlist.append(["", "missing source"])
             continue
         source = ent['source']
         source = templater.render_string(source, template_params)
 
-        if not ent['filename'].startswith(os.path.sep):
+        if not ent['filename'].startswith("/"):
             ent['filename'] = os.path.join("/etc/apt/sources.list.d/",
                                            ent['filename'])
+        if not ent['filename'].endswith(".list"):
+            ent['filename'] += ".list"
 
         if aa_repo_match(source):
             try:
-                util.subp(["add-apt-repository", source])
-            except util.ProcessExecutionError as e:
-                errorlist.append([source,
-                                  ("add-apt-repository failed. " + str(e))])
+                util.subp(["add-apt-repository", source], target=target)
+            except util.ProcessExecutionError:
+                LOG.exception("add-apt-repository failed.")
+                raise
             continue
 
+        sourcefn = util.target_path(target, ent['filename'])
         try:
             contents = "%s\n" % (source)
-            util.write_file(ent['filename'], contents, omode="ab")
-        except Exception:
-            errorlist.append([source,
-                             "failed write to file %s" % ent['filename']])
+            util.write_file(sourcefn, contents, omode="a")
+        except IOError as detail:
+            LOG.exception("failed write to file %s: %s", sourcefn, detail)
+            raise
 
-    return errorlist
+    update_packages(cloud)
 
+    return
 
-def find_apt_mirror_info(cloud, cfg):
-    """find an apt_mirror given the cloud and cfg provided."""
 
-    mirror = None
+def convert_v1_to_v2_apt_format(srclist):
+    """convert v1 apt format to v2 (dict in apt_sources)"""
+    srcdict = {}
+    if isinstance(srclist, list):
+        LOG.debug("apt config: convert V1 to V2 format (source list to dict)")
+        for srcent in srclist:
+            if 'filename' not in srcent:
+                # file collides for multiple !filename cases for compatibility
+                # yet we need them all processed, so not same dictionary key
+                srcent['filename'] = "cloud_config_sources.list"
+                key = util.rand_dict_key(srcdict, "cloud_config_sources.list")
+            else:
+                # all with filename use that as key (matching new format)
+                key = srcent['filename']
+            srcdict[key] = srcent
+    elif isinstance(srclist, dict):
+        srcdict = srclist
+    else:
+        raise ValueError("unknown apt_sources format")
+
+    return srcdict
+
+
+def convert_key(oldcfg, aptcfg, oldkey, newkey):
+    """convert an old key to the new one if the old one exists
+       returns true if a key was found and converted"""
+    if oldcfg.get(oldkey, None) is not None:
+        aptcfg[newkey] = oldcfg.get(oldkey)
+        del oldcfg[oldkey]
+        return True
+    return False
+
+
+def convert_mirror(oldcfg, aptcfg):
+    """convert old apt_mirror keys into the new more advanced mirror spec"""
+    keymap = [('apt_mirror', 'uri'),
+              ('apt_mirror_search', 'search'),
+              ('apt_mirror_search_dns', 'search_dns')]
+    converted = False
+    newmcfg = {'arches': ['default']}
+    for oldkey, newkey in keymap:
+        if convert_key(oldcfg, newmcfg, oldkey, newkey):
+            converted = True
+
+    # only insert new style config if anything was converted
+    if converted:
+        aptcfg['primary'] = [newmcfg]
+
+
+def convert_v2_to_v3_apt_format(oldcfg):
+    """convert old to new keys and adapt restructured mirror spec"""
+    oldkeys = ['apt_sources', 'apt_mirror', 'apt_mirror_search',
+               'apt_mirror_search_dns', 'apt_proxy', 'apt_http_proxy',
+               'apt_ftp_proxy', 'apt_https_proxy',
+               'apt_preserve_sources_list', 'apt_custom_sources_list',
+               'add_apt_repo_match']
+    needtoconvert = []
+    for oldkey in oldkeys:
+        if oldcfg.get(oldkey, None) is not None:
+            needtoconvert.append(oldkey)
+
+    # no old config, so no new one to be created
+    if not needtoconvert:
+        return oldcfg
+    LOG.debug("apt config: convert V2 to V3 format for keys '%s'",
+              ", ".join(needtoconvert))
+
+    if oldcfg.get('apt', None) is not None:
+        msg = ("Error in apt configuration: "
+               "old and new format of apt features are mutually exclusive "
+               "('apt':'%s' vs '%s' key)" % (oldcfg.get('apt', None),
+                                             ", ".join(needtoconvert)))
+        LOG.error(msg)
+        raise ValueError(msg)
+
+    # create new format from old keys
+    aptcfg = {}
+
+    # renames / moves under the apt key
+    convert_key(oldcfg, aptcfg, 'add_apt_repo_match', 'add_apt_repo_match')
+    convert_key(oldcfg, aptcfg, 'apt_proxy', 'proxy')
+    convert_key(oldcfg, aptcfg, 'apt_http_proxy', 'http_proxy')
+    convert_key(oldcfg, aptcfg, 'apt_https_proxy', 'https_proxy')
+    convert_key(oldcfg, aptcfg, 'apt_ftp_proxy', 'ftp_proxy')
+    convert_key(oldcfg, aptcfg, 'apt_custom_sources_list', 'sources_list')
+    convert_key(oldcfg, aptcfg, 'apt_preserve_sources_list',
+                'preserve_sources_list')
+    # dict format not changed since v2, just renamed and moved
+    convert_key(oldcfg, aptcfg, 'apt_sources', 'sources')
+
+    convert_mirror(oldcfg, aptcfg)
+
+    for oldkey in oldkeys:
+        if oldcfg.get(oldkey, None) is not None:
+            raise ValueError("old apt key '%s' left after conversion" % oldkey)
+
+    # insert new format into config and return full cfg with only v3 content
+    oldcfg['apt'] = aptcfg
+    return oldcfg
+
+
+def convert_to_v3_apt_format(cfg):
+    """convert the old list based format to the new dict based one. After that
+       convert the old dict keys/format to v3 a.k.a 'new apt config'"""
+    # V1 -> V2, the apt_sources entry from list to dict
+    apt_sources = cfg.get('apt_sources', None)
+    if apt_sources is not None:
+        cfg['apt_sources'] = convert_v1_to_v2_apt_format(apt_sources)
+
+    # V2 -> V3, move all former globals under the "apt" key
+    # Restructure into new key names and mirror hierarchy
+    cfg = convert_v2_to_v3_apt_format(cfg)
+
+    return cfg
+
+
+def search_for_mirror(candidates):
+    """
+    Search through a list of mirror urls for one that works
+    This needs to return quickly.
+    """
+    if candidates is None:
+        return None
+
+    LOG.debug("search for mirror in candidates: '%s'", candidates)
+    for cand in candidates:
+        try:
+            if util.is_resolvable_url(cand):
+                LOG.debug("found working mirror: '%s'", cand)
+                return cand
+        except Exception:
+            pass
+    return None
 
-    # this is less preferred way of specifying mirror preferred would be to
-    # use the distro's search or package_mirror.
-    mirror = cfg.get("apt_mirror", None)
 
-    search = cfg.get("apt_mirror_search", None)
-    if not mirror and search:
-        mirror = util.search_for_mirror(search)
+def search_for_mirror_dns(configured, mirrortype, cfg, cloud):
+    """
+    Try to resolve a list of predefines DNS names to pick mirrors
+    """
+    mirror = None
 
-    if (not mirror and
-            util.get_cfg_option_bool(cfg, "apt_mirror_search_dns", False)):
+    if configured:
         mydom = ""
         doms = []
 
+        if mirrortype == "primary":
+            mirrordns = "mirror"
+        elif mirrortype == "security":
+            mirrordns = "security-mirror"
+        else:
+            raise ValueError("unknown mirror type")
+
         # if we have a fqdn, then search its domain portion first
-        (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud)
+        (_, fqdn) = util.get_hostname_fqdn(cfg, cloud)
         mydom = ".".join(fqdn.split(".")[1:])
         if mydom:
             doms.append(".%s" % mydom)
@@ -282,38 +575,136 @@ def find_apt_mirror_info(cloud, cfg):
 
         mirror_list = []
         distro = cloud.distro.name
-        mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro)
+        mirrorfmt = "http://%s-%s%s/%s" % (distro, mirrordns, "%s", distro)
         for post in doms:
             mirror_list.append(mirrorfmt % (post))
 
-        mirror = util.search_for_mirror(mirror_list)
+        mirror = search_for_mirror(mirror_list)
+
+    return mirror
 
+
+def update_mirror_info(pmirror, smirror, arch, cloud):
+    """sets security mirror to primary if not defined.
+       returns defaults if no mirrors are defined"""
+    if pmirror is not None:
+        if smirror is None:
+            smirror = pmirror
+        return {'PRIMARY': pmirror,
+                'SECURITY': smirror}
+
+    # None specified at all, get default mirrors from cloud
     mirror_info = cloud.datasource.get_package_mirror_info()
+    if mirror_info:
+        # get_package_mirror_info() returns a dictionary with
+        # arbitrary key/value pairs including 'primary' and 'security' keys.
+        # caller expects dict with PRIMARY and SECURITY.
+        m = mirror_info.copy()
+        m['PRIMARY'] = m['primary']
+        m['SECURITY'] = m['security']
+
+        return m
+
+    # if neither apt nor cloud configured mirrors fall back to
+    return get_default_mirrors(arch)
+
+
+def get_arch_mirrorconfig(cfg, mirrortype, arch):
+    """out of a list of potential mirror configurations select
+       and return the one matching the architecture (or default)"""
+    # select the mirror specification (if-any)
+    mirror_cfg_list = cfg.get(mirrortype, None)
+    if mirror_cfg_list is None:
+        return None
+
+    # select the specification matching the target arch
+    default = None
+    for mirror_cfg_elem in mirror_cfg_list:
+        arches = mirror_cfg_elem.get("arches")
+        if arch in arches:
+            return mirror_cfg_elem
+        if "default" in arches:
+            default = mirror_cfg_elem
+    return default
+
+
+def get_mirror(cfg, mirrortype, arch, cloud):
+    """pass the three potential stages of mirror specification
+       returns None is neither of them found anything otherwise the first
+       hit is returned"""
+    mcfg = get_arch_mirrorconfig(cfg, mirrortype, arch)
+    if mcfg is None:
+        return None
+
+    # directly specified
+    mirror = mcfg.get("uri", None)
+
+    # fallback to search if specified
+    if mirror is None:
+        # list of mirrors to try to resolve
+        mirror = search_for_mirror(mcfg.get("search", None))
+
+    # fallback to search_dns if specified
+    if mirror is None:
+        # list of mirrors to try to resolve
+        mirror = search_for_mirror_dns(mcfg.get("search_dns", None),
+                                       mirrortype, cfg, cloud)
+
+    return mirror
+
+
+def find_apt_mirror_info(cfg, cloud, arch=None):
+    """find_apt_mirror_info
+       find an apt_mirror given the cfg provided.
+       It can check for separate config of primary and security mirrors
+       If only primary is given security is assumed to be equal to primary
+       If the generic apt_mirror is given that is defining for both
+    """
 
-    # this is a bit strange.
-    # if mirror is set, then one of the legacy options above set it
-    # but they do not cover security. so we need to get that from
-    # get_package_mirror_info
-    if mirror:
-        mirror_info.update({'primary': mirror})
+    if arch is None:
+        arch = util.get_architecture()
+        LOG.debug("got arch for mirror selection: %s", arch)
+    pmirror = get_mirror(cfg, "primary", arch, cloud)
+    LOG.debug("got primary mirror: %s", pmirror)
+    smirror = get_mirror(cfg, "security", arch, cloud)
+    LOG.debug("got security mirror: %s", smirror)
+
+    mirror_info = update_mirror_info(pmirror, smirror, arch, cloud)
+
+    # less complex replacements use only MIRROR, derive from primary
+    mirror_info["MIRROR"] = mirror_info["PRIMARY"]
 
     return mirror_info
 
 
 def apply_apt_config(cfg, proxy_fname, config_fname):
+    """apply_apt_config
+       Applies any apt*proxy config from if specified
+    """
     # Set up any apt proxy
-    cfgs = (('apt_proxy', 'Acquire::HTTP::Proxy "%s";'),
-            ('apt_http_proxy', 'Acquire::HTTP::Proxy "%s";'),
-            ('apt_ftp_proxy', 'Acquire::FTP::Proxy "%s";'),
-            ('apt_https_proxy', 'Acquire::HTTPS::Proxy "%s";'))
+    cfgs = (('proxy', 'Acquire::http::Proxy "%s";'),
+            ('http_proxy', 'Acquire::http::Proxy "%s";'),
+            ('ftp_proxy', 'Acquire::ftp::Proxy "%s";'),
+            ('https_proxy', 'Acquire::https::Proxy "%s";'))
 
     proxies = [fmt % cfg.get(name) for (name, fmt) in cfgs if cfg.get(name)]
     if len(proxies):
+        LOG.debug("write apt proxy info to %s", proxy_fname)
         util.write_file(proxy_fname, '\n'.join(proxies) + '\n')
     elif os.path.isfile(proxy_fname):
         util.del_file(proxy_fname)
+        LOG.debug("no apt proxy configured, removed %s", proxy_fname)
 
-    if cfg.get('apt_config', None):
-        util.write_file(config_fname, cfg.get('apt_config'))
+    if cfg.get('conf', None):
+        LOG.debug("write apt config info to %s", config_fname)
+        util.write_file(config_fname, cfg.get('conf'))
     elif os.path.isfile(config_fname):
         util.del_file(config_fname)
+        LOG.debug("no apt config configured, removed %s", config_fname)
+
+
+CONFIG_CLEANERS = {
+    'cloud-init': clean_cloud_init,
+}
+
+# vi: ts=4 expandtab syntax=python
diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py
index 6a76d78..5bbff51 100644
--- a/cloudinit/gpg.py
+++ b/cloudinit/gpg.py
@@ -36,11 +36,11 @@ def export_armour(key):
     return armour
 
 
-def receive_key(key, keyserver):
+def recv_key(key, keyserver):
     """Receive gpg key from the specified keyserver"""
     LOG.debug('Receive gpg key "%s"', key)
     try:
-        util.subp(["gpg", "--keyserver", keyserver, "--recv-keys", key],
+        util.subp(["gpg", "--keyserver", keyserver, "--recv", key],
                   capture=True)
     except util.ProcessExecutionError as error:
         raise ValueError(('Failed to import key "%s" '
@@ -57,12 +57,12 @@ def delete_key(key):
         LOG.warn('Failed delete key "%s": %s', key, error)
 
 
-def get_key_by_id(keyid, keyserver="keyserver.ubuntu.com"):
+def getkeybyid(keyid, keyserver='keyserver.ubuntu.com'):
     """get gpg keyid from keyserver"""
     armour = export_armour(keyid)
     if not armour:
         try:
-            receive_key(keyid, keyserver=keyserver)
+            recv_key(keyid, keyserver=keyserver)
             armour = export_armour(keyid)
         except ValueError:
             LOG.exception('Failed to obtain gpg key %s', keyid)
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 226628c..db80ca9 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -61,6 +61,10 @@ from cloudinit import version
 
 from cloudinit.settings import (CFG_BUILTIN)
 
+try:
+    string_types = (basestring,)
+except NameError:
+    string_types = (str,)
 
 _DNS_REDIRECT_IP = None
 LOG = logging.getLogger(__name__)
@@ -82,6 +86,71 @@ CONTAINER_TESTS = (['systemd-detect-virt', '--quiet', '--container'],
 
 PROC_CMDLINE = None
 
+_LSB_RELEASE = {}
+
+
+def get_architecture(target=None):
+    out, _ = subp(['dpkg', '--print-architecture'], capture=True,
+                  target=target)
+    return out.strip()
+
+
+def _lsb_release(target=None):
+    fmap = {'Codename': 'codename', 'Description': 'description',
+            'Distributor ID': 'id', 'Release': 'release'}
+
+    data = {}
+    try:
+        out, _ = subp(['lsb_release', '--all'], capture=True, target=target)
+        for line in out.splitlines():
+            fname, _, val = line.partition(":")
+            if fname in fmap:
+                data[fmap[fname]] = val.strip()
+        missing = [k for k in fmap.values() if k not in data]
+        if len(missing):
+            LOG.warn("Missing fields in lsb_release --all output: %s",
+                     ','.join(missing))
+
+    except ProcessExecutionError as err:
+        LOG.warn("Unable to get lsb_release --all: %s", err)
+        data = {v: "UNAVAILABLE" for v in fmap.values()}
+
+    return data
+
+
+def lsb_release(target=None):
+    if target_path(target) != "/":
+        # do not use or update cache if target is provided
+        return _lsb_release(target)
+
+    global _LSB_RELEASE
+    if not _LSB_RELEASE:
+        data = _lsb_release()
+        _LSB_RELEASE.update(data)
+    return _LSB_RELEASE
+
+
+def target_path(target, path=None):
+    # return 'path' inside target, accepting target as None
+    if target in (None, ""):
+        target = "/"
+    elif not isinstance(target, string_types):
+        raise ValueError("Unexpected input for target: %s" % target)
+    else:
+        target = os.path.abspath(target)
+        # abspath("//") returns "//" specifically for 2 slashes.
+        if target.startswith("//"):
+            target = target[1:]
+
+    if not path:
+        return target
+
+    # os.path.join("/etc", "/foo") returns "/foo". Chomp all leading /.
+    while len(path) and path[0] == "/":
+        path = path[1:]
+
+    return os.path.join(target, path)
+
 
 def decode_binary(blob, encoding='utf-8'):
     # Converts a binary type into a text type using given encoding.
@@ -1688,10 +1757,20 @@ def delete_dir_contents(dirname):
 
 
 def subp(args, data=None, rcs=None, env=None, capture=True, shell=False,
-         logstring=False):
+         logstring=False, decode="replace", target=None):
+
+    # not supported in cloud-init (yet), for now kept in the call signature
+    # to ease maintaining code shared between cloud-init and curtin
+    if target is not None:
+        raise ValueError("target arg not supported by cloud-init")
+
     if rcs is None:
         rcs = [0]
+
+    devnull_fp = None
     try:
+        if target_path(target) != "/":
+            args = ['chroot', target] + list(args)
 
         if not logstring:
             LOG.debug(("Running command %s with allowed return codes %s"
@@ -1700,33 +1779,52 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False,
             LOG.debug(("Running hidden command to protect sensitive "
                        "input/output logstring: %s"), logstring)
 
-        if not capture:
-            stdout = None
-            stderr = None
-        else:
+        stdin = None
+        stdout = None
+        stderr = None
+        if capture:
             stdout = subprocess.PIPE
             stderr = subprocess.PIPE
-        stdin = subprocess.PIPE
-        kws = dict(stdout=stdout, stderr=stderr, stdin=stdin,
-                   env=env, shell=shell)
-        if six.PY3:
-            # Use this so subprocess output will be (Python 3) str, not bytes.
-            kws['universal_newlines'] = True
-        sp = subprocess.Popen(args, **kws)
+        if data is None:
+            # using devnull assures any reads get null, rather
+            # than possibly waiting on input.
+            devnull_fp = open(os.devnull)
+            stdin = devnull_fp
+        else:
+            stdin = subprocess.PIPE
+            if not isinstance(data, bytes):
+                data = data.encode()
+
+        sp = subprocess.Popen(args, stdout=stdout,
+                              stderr=stderr, stdin=stdin,
+                              env=env, shell=shell)
         (out, err) = sp.communicate(data)
+
+        # Just ensure blank instead of none.
+        if not out and capture:
+            out = b''
+        if not err and capture:
+            err = b''
+        if decode:
+            def ldecode(data, m='utf-8'):
+                if not isinstance(data, bytes):
+                    return data
+                return data.decode(m, errors=decode)
+
+            out = ldecode(out)
+            err = ldecode(err)
     except OSError as e:
         raise ProcessExecutionError(cmd=args, reason=e,
                                     errno=e.errno)
+    finally:
+        if devnull_fp:
+            devnull_fp.close()
+
     rc = sp.returncode
     if rc not in rcs:
         raise ProcessExecutionError(stdout=out, stderr=err,
                                     exit_code=rc,
                                     cmd=args)
-    # Just ensure blank instead of none?? (iff capturing)
-    if not out and capture:
-        out = ''
-    if not err and capture:
-        err = ''
     return (out, err)
 
 
@@ -2251,3 +2349,18 @@ def message_from_string(string):
     if sys.version_info[:2] < (2, 7):
         return email.message_from_file(six.StringIO(string))
     return email.message_from_string(string)
+
+
+def get_installed_packages(target=None):
+    (out, _) = subp(['dpkg-query', '--list'], target=target, capture=True)
+
+    pkgs_inst = set()
+    for line in out.splitlines():
+        try:
+            (state, pkg, _) = line.split(None, 2)
+        except ValueError:
+            continue
+        if state.startswith("hi") or state.startswith("ii"):
+            pkgs_inst.add(re.sub(":.*", "", pkg))
+
+    return pkgs_inst
diff --git a/doc/examples/cloud-config-add-apt-repos.txt b/doc/examples/cloud-config-add-apt-repos.txt
index be9d547..22ef761 100644
--- a/doc/examples/cloud-config-add-apt-repos.txt
+++ b/doc/examples/cloud-config-add-apt-repos.txt
@@ -4,18 +4,21 @@
 #
 # Default: auto select based on cloud metadata
 #  in ec2, the default is <region>.archive.ubuntu.com
-# apt_mirror:
-#   use the provided mirror
-# apt_mirror_search:
-#   search the list for the first mirror.
-#   this is currently very limited, only verifying that
-#   the mirror is dns resolvable or an IP address
+# apt:
+#   primary:
+#     - arches [default]
+#       uri:
+#     use the provided mirror
+#       search:
+#     search the list for the first mirror.
+#     this is currently very limited, only verifying that
+#     the mirror is dns resolvable or an IP address
 #
-# if neither apt_mirror nor apt_mirror search is set (the default)
+# if neither mirror is set (the default)
 # then use the mirror provided by the DataSource found.
 # In EC2, that means using <region>.ec2.archive.ubuntu.com
-# 
-# if no mirror is provided by the DataSource, and 'apt_mirror_search_dns' is
+#
+# if no mirror is provided by the DataSource, but 'search_dns' is
 # true, then search for dns names '<distro>-mirror' in each of
 # - fqdn of this host per cloud metadata
 # - localdomain
@@ -27,8 +30,19 @@
 # up and expose them only by creating dns entries.
 #
 # if none of that is found, then the default distro mirror is used
-apt_mirror: http://us.archive.ubuntu.com/ubuntu/
-apt_mirror_search: 
- - http://local-mirror.mydomain
- - http://archive.ubuntu.com
-apt_mirror_search_dns: False
+apt:
+  primary:
+    - arches: [default]
+      uri: http://us.archive.ubuntu.com/ubuntu/
+# or
+apt:
+  primary:
+    - arches: [default]
+      search:
+        - http://local-mirror.mydomain
+        - http://archive.ubuntu.com
+# or
+apt:
+  primary:
+    - arches: [default]
+      search_dns: True
diff --git a/doc/examples/cloud-config-apt.txt b/doc/examples/cloud-config-apt.txt
new file mode 100644
index 0000000..1a0fc6f
--- /dev/null
+++ b/doc/examples/cloud-config-apt.txt
@@ -0,0 +1,328 @@
+# apt_pipelining (configure Acquire::http::Pipeline-Depth)
+# Default: disables HTTP pipelining. Certain web servers, such
+# as S3 do not pipeline properly (LP: #948461).
+# Valid options:
+#   False/default: Disables pipelining for APT
+#   None/Unchanged: Use OS default
+#   Number: Set pipelining to some number (not recommended)
+apt_pipelining: False
+
+## apt config via system_info:
+# under the 'system_info', you can customize cloud-init's interaction
+# with apt.
+#  system_info:
+#    apt_get_command: [command, argument, argument]
+#    apt_get_upgrade_subcommand: dist-upgrade
+#
+# apt_get_command:
+#  To specify a different 'apt-get' command, set 'apt_get_command'.
+#  This must be a list, and the subcommand (update, upgrade) is appended to it.
+#  default is:
+#    ['apt-get', '--option=Dpkg::Options::=--force-confold',
+#     '--option=Dpkg::options::=--force-unsafe-io', '--assume-yes', '--quiet']
+#
+# apt_get_upgrade_subcommand: "dist-upgrade"
+#  Specify a different subcommand for 'upgrade. The default is 'dist-upgrade'.
+#  This is the subcommand that is invoked for package_upgrade.
+#
+# apt_get_wrapper:
+#   command: eatmydata
+#   enabled: [True, False, "auto"]
+#
+
+# Install additional packages on first boot
+#
+# Default: none
+#
+# if packages are specified, this apt_update will be set to true
+
+packages: ['pastebinit']
+
+apt:
+  # The apt config consists of two major "areas".
+  #
+  # On one hand there is the global configuration for the apt feature.
+  #
+  # On one hand (down in this file) there is the source dictionary which allows
+  # to define various entries to be considered by apt.
+
+  ##############################################################################
+  # Section 1: global apt configuration
+  #
+  # The following examples number the top keys to ease identification in
+  # discussions.
+
+  # 1.1 preserve_sources_list
+  #
+  # Preserves the existing /etc/apt/sources.list
+  # Default: false - do overwrite sources_list. If set to true then any
+  # "mirrors" configuration will have no effect.
+  # Set to true to avoid affecting sources.list. In that case only
+  # "extra" source specifications will be written into
+  # /etc/apt/sources.list.d/*
+  preserve_sources_list: true
+
+  # 1.2 disable_suites
+  #
+  # This is an empty list by default, so nothing is disabled.
+  #
+  # If given, those suites are removed from sources.list after all other
+  # modifications have been made.
+  # Suites are even disabled if no other modification was made,
+  # but not if is preserve_sources_list is active.
+  # There is a special alias “$RELEASE” as in the sources that will be replace
+  # by the matching release.
+  #
+  # To ease configuration and improve readability the following common ubuntu
+  # suites will be automatically mapped to their full definition.
+  # updates   => $RELEASE-updates
+  # backports => $RELEASE-backports
+  # security  => $RELEASE-security
+  # proposed  => $RELEASE-proposed
+  # release   => $RELEASE
+  #
+  # There is no harm in specifying a suite to be disabled that is not found in
+  # the source.list file (just a no-op then)
+  #
+  # Note: Lines don’t get deleted, but disabled by being converted to a comment.
+  # The following example disables all usual defaults except $RELEASE-security.
+  # On top it disables a custom suite called "mysuite"
+  disable_suites: [$RELEASE-updates, backports, $RELEASE, mysuite]
+
+  # 1.3 primary/security archives
+  #
+  # Default: none - instead it is auto select based on cloud metadata
+  # so if neither "uri" nor "search", nor "search_dns" is set (the default)
+  # then use the mirror provided by the DataSource found.
+  # In EC2, that means using <region>.ec2.archive.ubuntu.com
+  #
+  # define a custom (e.g. localized) mirror that will be used in sources.list
+  # and any custom sources entries for deb / deb-src lines.
+  #
+  # One can set primary and security mirror to different uri's
+  # the child elements to the keys primary and secondary are equivalent
+  primary:
+    # arches is list of architectures the following config applies to
+    # the special keyword "default" applies to any architecture not explicitly
+    # listed.
+    - arches: [amd64, i386, default]
+      # uri is just defining the target as-is
+      uri: http://us.archive.ubuntu.com/ubuntu
+      #
+      # via search one can define lists that are tried one by one.
+      # The first with a working DNS resolution (or if it is an IP) will be
+      # picked. That way one can keep one configuration for multiple
+      # subenvironments that select the working one.
+      search:
+        - http://cool.but-sometimes-unreachable.com/ubuntu
+        - http://us.archive.ubuntu.com/ubuntu
+      # if no mirror is provided by uri or search but 'search_dns' is
+      # true, then search for dns names '<distro>-mirror' in each of
+      # - fqdn of this host per cloud metadata
+      # - localdomain
+      # - no domain (which would search domains listed in /etc/resolv.conf)
+      # If there is a dns entry for <distro>-mirror, then it is assumed that
+      # there is a distro mirror at http://<distro>-mirror.<domain>/<distro>
+      #
+      # That gives the cloud provider the opportunity to set mirrors of a distro
+      # up and expose them only by creating dns entries.
+      #
+      # if none of that is found, then the default distro mirror is used
+      search_dns: true
+      #
+      # If multiple of a category are given
+      #   1. uri
+      #   2. search
+      #   3. search_dns
+      # the first defining a valid mirror wins (in the order as defined here,
+      # not the order as listed in the config).
+      #
+    - arches: [s390x, arm64]
+      # as above, allowing to have one config for different per arch mirrors
+  # security is optional, if not defined it is set to the same value as primary
+  security:
+      uri: http://security.ubuntu.com/ubuntu
+  # If search_dns is set for security the searched pattern is:
+  #   <distro>-security-mirror
+
+  # if no mirrors are specified at all, or all lookups fail it will try
+  # to get them from the cloud datasource and if those neither provide one fall
+  # back to:
+  #   primary: http://archive.ubuntu.com/ubuntu
+  #   security: http://security.ubuntu.com/ubuntu
+
+  # 1.4 sources_list
+  #
+  # Provide a custom template for rendering sources.list
+  # without one provided cloud-init uses builtin templates for
+  # ubuntu and debian.
+  # Within these sources.list templates you can use the following replacement
+  # variables (all have sane Ubuntu defaults, but mirrors can be overwritten
+  # as needed (see above)):
+  # => $RELEASE, $MIRROR, $PRIMARY, $SECURITY
+  sources_list: | # written by cloud-init custom template
+    deb $MIRROR $RELEASE main restricted
+    deb-src $MIRROR $RELEASE main restricted
+    deb $PRIMARY $RELEASE universe restricted
+    deb $SECURITY $RELEASE-security multiverse
+
+  # 1.5 conf
+  #
+  # Any apt config string that will be made available to apt
+  # see the APT.CONF(5) man page for details what can be specified
+  conf: | # APT config
+    APT {
+      Get {
+        Assume-Yes "true";
+        Fix-Broken "true";
+      };
+    };
+
+  # 1.6 (http_|ftp_|https_)proxy
+  #
+  # Proxies are the most common apt.conf option, so that for simplified use
+  # there is a shortcut for those. Those get automatically translated into the
+  # correct Acquire::*::Proxy statements.
+  #
+  # note: proxy actually being a short synonym to http_proxy
+  proxy: http://[[user][:pass]@]host[:port]/
+  http_proxy: http://[[user][:pass]@]host[:port]/
+  ftp_proxy: ftp://[[user][:pass]@]host[:port]/
+  https_proxy: https://[[user][:pass]@]host[:port]/
+
+  # 1.7 add_apt_repo_match
+  #
+  # 'source' entries in apt-sources that match this python regex
+  # expression will be passed to add-apt-repository
+  # The following example is also the builtin default if nothing is specified
+  add_apt_repo_match: '^[\w-]+:\w'
+
+
+  ##############################################################################
+  # Section 2: source list entries
+  #
+  # This is a dictionary (unlike most block/net which are lists)
+  #
+  # The key of each source entry is the filename and will be prepended by
+  # /etc/apt/sources.list.d/ if it doesn't start with a '/'.
+  # If it doesn't end with .list it will be appended so that apt picks up it's
+  # configuration.
+  #
+  # Whenever there is no content to be written into such a file, the key is
+  # not used as filename - yet it can still be used as index for merging
+  # configuration.
+  #
+  # The values inside the entries consost of the following optional entries:
+  #   'source': a sources.list entry (some variable replacements apply)
+  #   'keyid': providing a key to import via shortid or fingerprint
+  #   'key': providing a raw PGP key
+  #   'keyserver': specify an alternate keyserver to pull keys from that
+  #                were specified by keyid
+
+  # This allows merging between multiple input files than a list like:
+  # cloud-config1
+  # sources:
+  #    s1: {'key': 'key1', 'source': 'source1'}
+  # cloud-config2
+  # sources:
+  #    s2: {'key': 'key2'}
+  #    s1: {'keyserver': 'foo'}
+  # This would be merged to
+  # sources:
+  #    s1:
+  #        keyserver: foo
+  #        key: key1
+  #        source: source1
+  #    s2:
+  #        key: key2
+  #
+  # The following examples number the subfeatures per sources entry to ease
+  # identification in discussions.
+
+
+  sources:
+    curtin-dev-ppa.list:
+      # 2.1 source
+      #
+      # Creates a file in /etc/apt/sources.list.d/ for the sources list entry
+      # based on the key: "/etc/apt/sources.list.d/curtin-dev-ppa.list"
+      source: "deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main"
+
+      # 2.2 keyid
+      #
+      # Importing a gpg key for a given key id. Used keyserver defaults to
+      # keyserver.ubuntu.com
+      keyid: F430BBA5 # GPG key ID published on a key server
+
+    ignored1:
+      # 2.3 PPA shortcut
+      #
+      # Setup correct apt sources.list line and Auto-Import the signing key
+      # from LP
+      #
+      # See https://help.launchpad.net/Packaging/PPA for more information
+      # this requires 'add-apt-repository'. This will create a file in
+      # /etc/apt/sources.list.d automatically, therefore the key here is
+      # ignored as filename in those cases.
+      source: "ppa:curtin-dev/test-archive"    # Quote the string
+
+    my-repo2.list:
+      # 2.4 replacement variables
+      #
+      # sources can use $MIRROR, $PRIMARY, $SECURITY and $RELEASE replacement
+      # variables.
+      # They will be replaced with the default or specified mirrors and the
+      # running release.
+      # The entry below would be possibly turned into:
+      #   source: deb http://archive.ubuntu.com/ubuntu xenial multiverse
+      source: deb $MIRROR $RELEASE multiverse
+
+    my-repo3.list:
+      # this would have the same end effect as 'ppa:curtin-dev/test-archive'
+      source: "deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main"
+      keyid: F430BBA5 # GPG key ID published on the key server
+      filename: curtin-dev-ppa.list
+
+    ignored2:
+      # 2.5 key only
+      #
+      # this would only import the key without adding a ppa or other source spec
+      # since this doesn't generate a source.list file the filename key is ignored
+      keyid: F430BBA5 # GPG key ID published on a key server
+
+    ignored3:
+      # 2.6 key id alternatives
+      #
+      # Keyid's can also be specified via their long fingerprints
+      keyid: B59D 5F15 97A5 04B7 E230  6DCA 0620 BBCF 0368 3F77
+
+    ignored4:
+      # 2.7 alternative keyservers
+      #
+      # One can also specify alternative keyservers to fetch keys from.
+      keyid: B59D 5F15 97A5 04B7 E230  6DCA 0620 BBCF 0368 3F77
+      keyserver: pgp.mit.edu
+
+
+    my-repo4.list:
+      # 2.8 raw key
+      #
+      # The apt signing key can also be specified by providing a pgp public key
+      # block. Providing the PGP key this way is the most robust method for
+      # specifying a key, as it removes dependency on a remote key server.
+      #
+      # As with keyid's this can be specified with or without some actual source
+      # content.
+      key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK-----
+         -----BEGIN PGP PUBLIC KEY BLOCK-----
+         Version: SKS 1.0.10
+
+         mI0ESpA3UQEEALdZKVIMq0j6qWAXAyxSlF63SvPVIgxHPb9Nk0DZUixn+akqytxG4zKCONz6
+         qLjoBBfHnynyVLfT4ihg9an1PqxRnTO+JKQxl8NgKGz6Pon569GtAOdWNKw15XKinJTDLjnj
+         9y96ljJqRcpV9t/WsIcdJPcKFR5voHTEoABE2aEXABEBAAG0GUxhdW5jaHBhZCBQUEEgZm9y
+         IEFsZXN0aWOItgQTAQIAIAUCSpA3UQIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEA7H
+         5Qi+CcVxWZ8D/1MyYvfj3FJPZUm2Yo1zZsQ657vHI9+pPouqflWOayRR9jbiyUFIn0VdQBrP
+         t0FwvnOFArUovUWoKAEdqR8hPy3M3APUZjl5K4cMZR/xaMQeQRZ5CHpS4DBKURKAHC0ltS5o
+         uBJKQOZm5iltJp15cgyIkBkGe8Mx18VFyVglAZey
+         =Y2oI
+         -----END PGP PUBLIC KEY BLOCK-----
diff --git a/doc/examples/cloud-config-chef-oneiric.txt b/doc/examples/cloud-config-chef-oneiric.txt
index 2e5f4b1..75c9aee 100644
--- a/doc/examples/cloud-config-chef-oneiric.txt
+++ b/doc/examples/cloud-config-chef-oneiric.txt
@@ -11,39 +11,40 @@
 # The default is to install from packages. 
 
 # Key from http://apt.opscode.com/[e-mail]
-apt_sources:
- - source: "deb http://apt.opscode.com/ $RELEASE-0.10 main"
-   key: |
-     -----BEGIN PGP PUBLIC KEY BLOCK-----
-     Version: GnuPG v1.4.9 (GNU/Linux)
-     
-     mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu
-     twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99
-     dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC
-     JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W
-     ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I
-     XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe
-     DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm
-     sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO
-     Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ
-     YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG
-     CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K
-     +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu5Ag0ESmkLtBAIAIO2SwlR
-     lU5i6gTOp42RHWW7/pmW78CwUqJnYqnXROrt3h9F9xrsGkH0Fh1FRtsnncgzIhvh
-     DLQnRHnkXm0ws0jV0PF74ttoUT6BLAUsFi2SPP1zYNJ9H9fhhK/pjijtAcQwdgxu
-     wwNJ5xCEscBZCjhSRXm0d30bK1o49Cow8ZIbHtnXVP41c9QWOzX/LaGZsKQZnaMx
-     EzDk8dyyctR2f03vRSVyTFGgdpUcpbr9eTFVgikCa6ODEBv+0BnCH6yGTXwBid9g
-     w0o1e/2DviKUWCC+AlAUOubLmOIGFBuI4UR+rux9affbHcLIOTiKQXv79lW3P7W8
-     AAfniSQKfPWXrrcAAwUH/2XBqD4Uxhbs25HDUUiM/m6Gnlj6EsStg8n0nMggLhuN
-     QmPfoNByMPUqvA7sULyfr6xCYzbzRNxABHSpf85FzGQ29RF4xsA4vOOU8RDIYQ9X
-     Q8NqqR6pydprRFqWe47hsAN7BoYuhWqTtOLSBmnAnzTR5pURoqcquWYiiEavZixJ
-     3ZRAq/HMGioJEtMFrvsZjGXuzef7f0ytfR1zYeLVWnL9Bd32CueBlI7dhYwkFe+V
-     Ep5jWOCj02C1wHcwt+uIRDJV6TdtbIiBYAdOMPk15+VBdweBXwMuYXr76+A7VeDL
-     zIhi7tKFo6WiwjKZq0dzctsJJjtIfr4K4vbiD9Ojg1iISQQYEQIACQUCSmkLtAIb
-     DAAKCRApQKupg++CauISAJ9CxYPOKhOxalBnVTLeNUkAHGg2gACeIsbobtaD4ZHG
-     0GLl8EkfA8uhluM=
-     =zKAm
-     -----END PGP PUBLIC KEY BLOCK-----
+apt:
+  sources:
+   - source: "deb http://apt.opscode.com/ $RELEASE-0.10 main"
+     key: |
+       -----BEGIN PGP PUBLIC KEY BLOCK-----
+       Version: GnuPG v1.4.9 (GNU/Linux)
+
+       mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu
+       twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99
+       dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC
+       JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W
+       ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I
+       XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe
+       DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm
+       sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO
+       Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ
+       YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG
+       CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K
+       +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu5Ag0ESmkLtBAIAIO2SwlR
+       lU5i6gTOp42RHWW7/pmW78CwUqJnYqnXROrt3h9F9xrsGkH0Fh1FRtsnncgzIhvh
+       DLQnRHnkXm0ws0jV0PF74ttoUT6BLAUsFi2SPP1zYNJ9H9fhhK/pjijtAcQwdgxu
+       wwNJ5xCEscBZCjhSRXm0d30bK1o49Cow8ZIbHtnXVP41c9QWOzX/LaGZsKQZnaMx
+       EzDk8dyyctR2f03vRSVyTFGgdpUcpbr9eTFVgikCa6ODEBv+0BnCH6yGTXwBid9g
+       w0o1e/2DviKUWCC+AlAUOubLmOIGFBuI4UR+rux9affbHcLIOTiKQXv79lW3P7W8
+       AAfniSQKfPWXrrcAAwUH/2XBqD4Uxhbs25HDUUiM/m6Gnlj6EsStg8n0nMggLhuN
+       QmPfoNByMPUqvA7sULyfr6xCYzbzRNxABHSpf85FzGQ29RF4xsA4vOOU8RDIYQ9X
+       Q8NqqR6pydprRFqWe47hsAN7BoYuhWqTtOLSBmnAnzTR5pURoqcquWYiiEavZixJ
+       3ZRAq/HMGioJEtMFrvsZjGXuzef7f0ytfR1zYeLVWnL9Bd32CueBlI7dhYwkFe+V
+       Ep5jWOCj02C1wHcwt+uIRDJV6TdtbIiBYAdOMPk15+VBdweBXwMuYXr76+A7VeDL
+       zIhi7tKFo6WiwjKZq0dzctsJJjtIfr4K4vbiD9Ojg1iISQQYEQIACQUCSmkLtAIb
+       DAAKCRApQKupg++CauISAJ9CxYPOKhOxalBnVTLeNUkAHGg2gACeIsbobtaD4ZHG
+       0GLl8EkfA8uhluM=
+       =zKAm
+       -----END PGP PUBLIC KEY BLOCK-----
 
 chef:
 
diff --git a/doc/examples/cloud-config-chef.txt b/doc/examples/cloud-config-chef.txt
index b886cba..75d78a1 100644
--- a/doc/examples/cloud-config-chef.txt
+++ b/doc/examples/cloud-config-chef.txt
@@ -11,39 +11,40 @@
 # The default is to install from packages. 
 
 # Key from http://apt.opscode.com/[e-mail]
-apt_sources:
- - source: "deb http://apt.opscode.com/ $RELEASE-0.10 main"
-   key: |
-     -----BEGIN PGP PUBLIC KEY BLOCK-----
-     Version: GnuPG v1.4.9 (GNU/Linux)
-     
-     mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu
-     twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99
-     dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC
-     JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W
-     ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I
-     XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe
-     DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm
-     sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO
-     Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ
-     YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG
-     CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K
-     +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu5Ag0ESmkLtBAIAIO2SwlR
-     lU5i6gTOp42RHWW7/pmW78CwUqJnYqnXROrt3h9F9xrsGkH0Fh1FRtsnncgzIhvh
-     DLQnRHnkXm0ws0jV0PF74ttoUT6BLAUsFi2SPP1zYNJ9H9fhhK/pjijtAcQwdgxu
-     wwNJ5xCEscBZCjhSRXm0d30bK1o49Cow8ZIbHtnXVP41c9QWOzX/LaGZsKQZnaMx
-     EzDk8dyyctR2f03vRSVyTFGgdpUcpbr9eTFVgikCa6ODEBv+0BnCH6yGTXwBid9g
-     w0o1e/2DviKUWCC+AlAUOubLmOIGFBuI4UR+rux9affbHcLIOTiKQXv79lW3P7W8
-     AAfniSQKfPWXrrcAAwUH/2XBqD4Uxhbs25HDUUiM/m6Gnlj6EsStg8n0nMggLhuN
-     QmPfoNByMPUqvA7sULyfr6xCYzbzRNxABHSpf85FzGQ29RF4xsA4vOOU8RDIYQ9X
-     Q8NqqR6pydprRFqWe47hsAN7BoYuhWqTtOLSBmnAnzTR5pURoqcquWYiiEavZixJ
-     3ZRAq/HMGioJEtMFrvsZjGXuzef7f0ytfR1zYeLVWnL9Bd32CueBlI7dhYwkFe+V
-     Ep5jWOCj02C1wHcwt+uIRDJV6TdtbIiBYAdOMPk15+VBdweBXwMuYXr76+A7VeDL
-     zIhi7tKFo6WiwjKZq0dzctsJJjtIfr4K4vbiD9Ojg1iISQQYEQIACQUCSmkLtAIb
-     DAAKCRApQKupg++CauISAJ9CxYPOKhOxalBnVTLeNUkAHGg2gACeIsbobtaD4ZHG
-     0GLl8EkfA8uhluM=
-     =zKAm
-     -----END PGP PUBLIC KEY BLOCK-----
+apt:
+  sources:
+   - source: "deb http://apt.opscode.com/ $RELEASE-0.10 main"
+     key: |
+       -----BEGIN PGP PUBLIC KEY BLOCK-----
+       Version: GnuPG v1.4.9 (GNU/Linux)
+
+       mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu
+       twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99
+       dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC
+       JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W
+       ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I
+       XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe
+       DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm
+       sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO
+       Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ
+       YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG
+       CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K
+       +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu5Ag0ESmkLtBAIAIO2SwlR
+       lU5i6gTOp42RHWW7/pmW78CwUqJnYqnXROrt3h9F9xrsGkH0Fh1FRtsnncgzIhvh
+       DLQnRHnkXm0ws0jV0PF74ttoUT6BLAUsFi2SPP1zYNJ9H9fhhK/pjijtAcQwdgxu
+       wwNJ5xCEscBZCjhSRXm0d30bK1o49Cow8ZIbHtnXVP41c9QWOzX/LaGZsKQZnaMx
+       EzDk8dyyctR2f03vRSVyTFGgdpUcpbr9eTFVgikCa6ODEBv+0BnCH6yGTXwBid9g
+       w0o1e/2DviKUWCC+AlAUOubLmOIGFBuI4UR+rux9affbHcLIOTiKQXv79lW3P7W8
+       AAfniSQKfPWXrrcAAwUH/2XBqD4Uxhbs25HDUUiM/m6Gnlj6EsStg8n0nMggLhuN
+       QmPfoNByMPUqvA7sULyfr6xCYzbzRNxABHSpf85FzGQ29RF4xsA4vOOU8RDIYQ9X
+       Q8NqqR6pydprRFqWe47hsAN7BoYuhWqTtOLSBmnAnzTR5pURoqcquWYiiEavZixJ
+       3ZRAq/HMGioJEtMFrvsZjGXuzef7f0ytfR1zYeLVWnL9Bd32CueBlI7dhYwkFe+V
+       Ep5jWOCj02C1wHcwt+uIRDJV6TdtbIiBYAdOMPk15+VBdweBXwMuYXr76+A7VeDL
+       zIhi7tKFo6WiwjKZq0dzctsJJjtIfr4K4vbiD9Ojg1iISQQYEQIACQUCSmkLtAIb
+       DAAKCRApQKupg++CauISAJ9CxYPOKhOxalBnVTLeNUkAHGg2gACeIsbobtaD4ZHG
+       0GLl8EkfA8uhluM=
+       =zKAm
+       -----END PGP PUBLIC KEY BLOCK-----
 
 chef:
 
diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt
index 3cc9c05..190029e 100644
--- a/doc/examples/cloud-config.txt
+++ b/doc/examples/cloud-config.txt
@@ -18,256 +18,7 @@ package_upgrade: true
 # Aliases: apt_reboot_if_required
 package_reboot_if_required: true
 
-# Add apt repositories
-#
-# Default: auto select based on cloud metadata
-#  in ec2, the default is <region>.archive.ubuntu.com
-# apt_mirror:
-#   use the provided mirror
-# apt_mirror_search:
-#   search the list for the first mirror.
-#   this is currently very limited, only verifying that
-#   the mirror is dns resolvable or an IP address
-#
-# if neither apt_mirror nor apt_mirror search is set (the default)
-# then use the mirror provided by the DataSource found.
-# In EC2, that means using <region>.ec2.archive.ubuntu.com
-# 
-# if no mirror is provided by the DataSource, and 'apt_mirror_search_dns' is
-# true, then search for dns names '<distro>-mirror' in each of
-# - fqdn of this host per cloud metadata
-# - localdomain
-# - no domain (which would search domains listed in /etc/resolv.conf)
-# If there is a dns entry for <distro>-mirror, then it is assumed that there
-# is a distro mirror at http://<distro>-mirror.<domain>/<distro>
-#
-# That gives the cloud provider the opportunity to set mirrors of a distro
-# up and expose them only by creating dns entries.
-#
-# if none of that is found, then the default distro mirror is used
-apt_mirror: http://us.archive.ubuntu.com/ubuntu/
-apt_mirror_search: 
- - http://local-mirror.mydomain
- - http://archive.ubuntu.com
-
-apt_mirror_search_dns: False
-
-# apt_proxy (configure Acquire::HTTP::Proxy)
-# 'apt_http_proxy' is an alias for 'apt_proxy'.
-# Also, available are 'apt_ftp_proxy' and 'apt_https_proxy'.
-# These affect Acquire::FTP::Proxy and Acquire::HTTPS::Proxy respectively
-apt_proxy: http://my.apt.proxy:3128
-
-# apt_pipelining (configure Acquire::http::Pipeline-Depth)
-# Default: disables HTTP pipelining. Certain web servers, such
-# as S3 do not pipeline properly (LP: #948461).
-# Valid options:
-#   False/default: Disables pipelining for APT
-#   None/Unchanged: Use OS default
-#   Number: Set pipelining to some number (not recommended)
-apt_pipelining: False
-
-# Preserve existing /etc/apt/sources.list
-# Default: overwrite sources_list with mirror.  If this is true
-# then apt_mirror above will have no effect
-apt_preserve_sources_list: true
-
-# Provide a custom template for rendering sources.list
-# Default: a default template for Ubuntu/Debain will be used as packaged in
-#  Ubuntu: /etc/cloud/templates/sources.list.ubuntu.tmpl
-#  Debian: /etc/cloud/templates/sources.list.debian.tmpl
-#  Others: n/a
-# This will follow the normal mirror/codename replacement rules before
-# being written to disk.
-apt_custom_sources_list: |
-    ## template:jinja
-    ## Note, this file is written by cloud-init on first boot of an instance
-    ## modifications made here will not survive a re-bundle.
-    ## if you wish to make changes you can:
-    ## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg
-    ##     or do the same in user-data
-    ## b.) add sources in /etc/apt/sources.list.d
-    ## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl
-    deb {{mirror}} {{codename}} main restricted
-    deb-src {{mirror}} {{codename}} main restricted
-
-    # could drop some of the usually used entries
-
-    # could refer to other mirrors
-    deb http://ddebs.ubuntu.com {{codename}} main restricted universe multiverse
-    deb http://ddebs.ubuntu.com {{codename}}-updates main restricted universe multiverse
-    deb http://ddebs.ubuntu.com {{codename}}-proposed main restricted universe multiverse
-
-    # or even more uncommon examples like local or NFS mounted repos,
-    # eventually whatever is compatible with sources.list syntax
-    deb file:/home/apt/debian unstable main contrib non-free
-
-# 'source' entries in apt-sources that match this python regex
-# expression will be passed to add-apt-repository
-add_apt_repo_match: '^[\w-]+:\w'
-
-# 'apt_sources' is a dictionary
-# The key is the filename and will be prepended by /etc/apt/sources.list.d/ if
-# it doesn't start with a '/'.
-# There are certain cases - where no content is written into a source.list file
-# where the filename will be ignored - yet it can still be used as index for
-# merging.
-# The value it maps to is a dictionary with the following optional entries:
-#   source: a sources.list entry (some variable replacements apply)
-#   keyid: providing a key to import via shortid or fingerprint
-#   key: providing a raw PGP key
-#   keyserver: keyserver to fetch keys from, default is keyserver.ubuntu.com
-#   filename: for compatibility with the older format (now the key to this
-#               dictionary is the filename). If specified this overwrites the
-#               filename given as key.
-
-# the new "filename: {specification-dictionary}, filename2: ..." format allows
-# better merging between multiple input files than a list like:
-# cloud-config1
-# sources:
-#    s1: {'key': 'key1', 'source': 'source1'}
-# cloud-config2
-# sources:
-#    s2: {'key': 'key2'}
-#    s1: {filename: 'foo'}
-# this would be merged to
-#sources:
-#    s1:
-#        filename: foo
-#        key: key1
-#        source: source1
-#    s2:
-#        key: key2
-# Be aware that this style of merging is not the default (for backward
-# compatibility reasons). You should specify the following merge_how to get
-# this more complete and modern merging behaviour:
-#   merge_how: "list()+dict()+str()"
-# This would then also be equivalent to the config merging used in curtin
-# (https://launchpad.net/curtin).
-
-# for more details see below in the various examples
-
-apt_sources:
- byobu-ppa.list:
-   source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main"
-   keyid: F430BBA5 # GPG key ID published on a key server
- # adding a source.list line, importing a gpg key for a given key id and
- # storing it in the file /etc/apt/sources.list.d/byobu-ppa.list
-
- # PPA shortcut:
- #  * Setup correct apt sources.list line
- #  * Import the signing key from LP
- #
- #  See https://help.launchpad.net/Packaging/PPA for more information
- #  this requires 'add-apt-repository'
- #  due to that the filename key is ignored in this case
- ignored1:
-   source: "ppa:smoser/ppa"    # Quote the string
-
- # Custom apt repository:
- #  * all that is required is 'source'
- #  * Creates a file in /etc/apt/sources.list.d/ for the sources list entry
- #  * [optional] Import the apt signing key from the keyserver 
- #  * Defaults:
- #    + keyserver: keyserver.ubuntu.com
- #
- #    See sources.list man page for more information about the format
- my-repo.list:
-   source: deb http://archive.ubuntu.com/ubuntu karmic-backports main universe multiverse restricted
-
- # sources can use $MIRROR and $RELEASE and they will be replaced
- # with the local mirror for this cloud, and the running release
- # the entry below would be possibly turned into:
- # source: deb http://us-east-1.ec2.archive.ubuntu.com/ubuntu natty multiverse
- my-repo.list:
-   source: deb $MIRROR $RELEASE multiverse
-
- # this would have the same end effect as 'ppa:byobu/ppa'
- my-repo.list:
-   source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main"
-   keyid: F430BBA5 # GPG key ID published on a key server
-   filename: byobu-ppa.list
-
- # this would only import the key without adding a ppa or other source spec
- # since this doesn't generate a source.list file the filename key is ignored
- ignored2:
-   keyid: F430BBA5 # GPG key ID published on a key server
-
- # In general keyid's can also be specified via their long fingerprints
- # since this doesn't generate a source.list file the filename key is ignored
- ignored3:
-   keyid: B59D 5F15 97A5 04B7 E230  6DCA 0620 BBCF 0368 3F77
-
- # Custom apt repository:
- #  * The apt signing key can also be specified
- #    by providing a pgp public key block
- #  * Providing the PGP key here is the most robust method for
- #    specifying a key, as it removes dependency on a remote key server
- my-repo.list:
-   source: deb http://ppa.launchpad.net/alestic/ppa/ubuntu karmic main 
-   key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK-----
-      -----BEGIN PGP PUBLIC KEY BLOCK-----
-      Version: SKS 1.0.10
-
-      mI0ESpA3UQEEALdZKVIMq0j6qWAXAyxSlF63SvPVIgxHPb9Nk0DZUixn+akqytxG4zKCONz6
-      qLjoBBfHnynyVLfT4ihg9an1PqxRnTO+JKQxl8NgKGz6Pon569GtAOdWNKw15XKinJTDLjnj
-      9y96ljJqRcpV9t/WsIcdJPcKFR5voHTEoABE2aEXABEBAAG0GUxhdW5jaHBhZCBQUEEgZm9y
-      IEFsZXN0aWOItgQTAQIAIAUCSpA3UQIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEA7H
-      5Qi+CcVxWZ8D/1MyYvfj3FJPZUm2Yo1zZsQ657vHI9+pPouqflWOayRR9jbiyUFIn0VdQBrP
-      t0FwvnOFArUovUWoKAEdqR8hPy3M3APUZjl5K4cMZR/xaMQeQRZ5CHpS4DBKURKAHC0ltS5o
-      uBJKQOZm5iltJp15cgyIkBkGe8Mx18VFyVglAZey
-      =Y2oI
-      -----END PGP PUBLIC KEY BLOCK-----
-
- # Custom gpg key:
- #  * As with keyid, a key may also be specified without a related source.
- #  * all other facts mentioned above still apply
- # since this doesn't generate a source.list file the filename key is ignored
- ignored4:
-   key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK-----
-      -----BEGIN PGP PUBLIC KEY BLOCK-----
-      Version: SKS 1.0.10
-
-      mI0ESpA3UQEEALdZKVIMq0j6qWAXAyxSlF63SvPVIgxHPb9Nk0DZUixn+akqytxG4zKCONz6
-      qLjoBBfHnynyVLfT4ihg9an1PqxRnTO+JKQxl8NgKGz6Pon569GtAOdWNKw15XKinJTDLjnj
-      9y96ljJqRcpV9t/WsIcdJPcKFR5voHTEoABE2aEXABEBAAG0GUxhdW5jaHBhZCBQUEEgZm9y
-      IEFsZXN0aWOItgQTAQIAIAUCSpA3UQIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEA7H
-      5Qi+CcVxWZ8D/1MyYvfj3FJPZUm2Yo1zZsQ657vHI9+pPouqflWOayRR9jbiyUFIn0VdQBrP
-      t0FwvnOFArUovUWoKAEdqR8hPy3M3APUZjl5K4cMZR/xaMQeQRZ5CHpS4DBKURKAHC0ltS5o
-      uBJKQOZm5iltJp15cgyIkBkGe8Mx18VFyVglAZey
-      =Y2oI
-      -----END PGP PUBLIC KEY BLOCK-----
-
-
-## apt config via system_info:
-# under the 'system_info', you can further customize cloud-init's interaction
-# with apt. 
-#   system_info:
-#    apt_get_command: [command, argument, argument]
-#    apt_get_upgrade_subcommand: dist-upgrade
-#
-# apt_get_command:
-#  To specify a different 'apt-get' command, set 'apt_get_command'.
-#  This must be a list, and the subcommand (update, upgrade) is appended to it.
-#  default is:
-#    ['apt-get', '--option=Dpkg::Options::=--force-confold',
-#     '--option=Dpkg::options::=--force-unsafe-io', '--assume-yes', '--quiet']
-#
-# apt_get_upgrade_subcommand:
-#  Specify a different subcommand for 'upgrade. The default is 'dist-upgrade'.
-#  This is the subcommand that is invoked if package_upgrade is set to true above.
-#
-# apt_get_wrapper:
-#   command: eatmydata
-#   enabled: [True, False, "auto"]
-#  
-
-# Install additional packages on first boot
-#
-# Default: none
-#
-# if packages are specified, this apt_update will be set to true
-#
+# For 'apt' specific config, see cloud-config-apt.txt
 packages:
  - pwgen
  - pastebinit
diff --git a/tests/configs/sample1.yaml b/tests/configs/sample1.yaml
index 6231f29..ae935cc 100644
--- a/tests/configs/sample1.yaml
+++ b/tests/configs/sample1.yaml
@@ -3,9 +3,6 @@
 #apt_upgrade: true
 packages: [ bzr, pastebinit, ubuntu-dev-tools, ccache, bzr-builddeb, vim-nox, git-core, lftp ]
 
-#apt_sources:
-# - source: ppa:smoser/ppa
-
 #disable_root: False
 
 # mounts:
diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py
index 96fa081..24ad115 100644
--- a/tests/unittests/test_distros/test_generic.py
+++ b/tests/unittests/test_distros/test_generic.py
@@ -226,8 +226,5 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase):
         os.symlink('/', '/run/systemd/system')
         self.assertFalse(d.uses_systemd())
 
-# def _get_package_mirror_info(mirror_info, availability_zone=None,
-#                             mirror_filter=util.search_for_mirror):
-
 
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_handler_apt_conf_v1.py b/tests/unittests/test_handler/test_handler_apt_conf_v1.py
new file mode 100644
index 0000000..95fd1da
--- /dev/null
+++ b/tests/unittests/test_handler/test_handler_apt_conf_v1.py
@@ -0,0 +1,109 @@
+from cloudinit.config import cc_apt_configure
+from cloudinit import util
+
+from ..helpers import TestCase
+
+import os
+import re
+import shutil
+import tempfile
+
+
+def load_tfile_or_url(*args, **kwargs):
+    return(util.decode_binary(util.read_file_or_url(*args, **kwargs).contents))
+
+
+class TestAptProxyConfig(TestCase):
+    def setUp(self):
+        super(TestAptProxyConfig, self).setUp()
+        self.tmp = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.tmp)
+        self.pfile = os.path.join(self.tmp, "proxy.cfg")
+        self.cfile = os.path.join(self.tmp, "config.cfg")
+
+    def _search_apt_config(self, contents, ptype, value):
+        return re.search(
+            r"acquire::%s::proxy\s+[\"']%s[\"'];\n" % (ptype, value),
+            contents, flags=re.IGNORECASE)
+
+    def test_apt_proxy_written(self):
+        cfg = {'proxy': 'myproxy'}
+        cc_apt_configure.apply_apt_config(cfg, self.pfile, self.cfile)
+
+        self.assertTrue(os.path.isfile(self.pfile))
+        self.assertFalse(os.path.isfile(self.cfile))
+
+        contents = load_tfile_or_url(self.pfile)
+        self.assertTrue(self._search_apt_config(contents, "http", "myproxy"))
+
+    def test_apt_http_proxy_written(self):
+        cfg = {'http_proxy': 'myproxy'}
+        cc_apt_configure.apply_apt_config(cfg, self.pfile, self.cfile)
+
+        self.assertTrue(os.path.isfile(self.pfile))
+        self.assertFalse(os.path.isfile(self.cfile))
+
+        contents = load_tfile_or_url(self.pfile)
+        self.assertTrue(self._search_apt_config(contents, "http", "myproxy"))
+
+    def test_apt_all_proxy_written(self):
+        cfg = {'http_proxy': 'myproxy_http_proxy',
+               'https_proxy': 'myproxy_https_proxy',
+               'ftp_proxy': 'myproxy_ftp_proxy'}
+
+        values = {'http': cfg['http_proxy'],
+                  'https': cfg['https_proxy'],
+                  'ftp': cfg['ftp_proxy'],
+                  }
+
+        cc_apt_configure.apply_apt_config(cfg, self.pfile, self.cfile)
+
+        self.assertTrue(os.path.isfile(self.pfile))
+        self.assertFalse(os.path.isfile(self.cfile))
+
+        contents = load_tfile_or_url(self.pfile)
+
+        for ptype, pval in values.items():
+            self.assertTrue(self._search_apt_config(contents, ptype, pval))
+
+    def test_proxy_deleted(self):
+        util.write_file(self.cfile, "content doesnt matter")
+        cc_apt_configure.apply_apt_config({}, self.pfile, self.cfile)
+        self.assertFalse(os.path.isfile(self.pfile))
+        self.assertFalse(os.path.isfile(self.cfile))
+
+    def test_proxy_replaced(self):
+        util.write_file(self.cfile, "content doesnt matter")
+        cc_apt_configure.apply_apt_config({'proxy': "foo"},
+                                          self.pfile, self.cfile)
+        self.assertTrue(os.path.isfile(self.pfile))
+        contents = load_tfile_or_url(self.pfile)
+        self.assertTrue(self._search_apt_config(contents, "http", "foo"))
+
+    def test_config_written(self):
+        payload = 'this is my apt config'
+        cfg = {'conf': payload}
+
+        cc_apt_configure.apply_apt_config(cfg, self.pfile, self.cfile)
+
+        self.assertTrue(os.path.isfile(self.cfile))
+        self.assertFalse(os.path.isfile(self.pfile))
+
+        self.assertEqual(load_tfile_or_url(self.cfile), payload)
+
+    def test_config_replaced(self):
+        util.write_file(self.pfile, "content doesnt matter")
+        cc_apt_configure.apply_apt_config({'conf': "foo"},
+                                          self.pfile, self.cfile)
+        self.assertTrue(os.path.isfile(self.cfile))
+        self.assertEqual(load_tfile_or_url(self.cfile), "foo")
+
+    def test_config_deleted(self):
+        # if no 'conf' is provided, delete any previously written file
+        util.write_file(self.pfile, "content doesnt matter")
+        cc_apt_configure.apply_apt_config({}, self.pfile, self.cfile)
+        self.assertFalse(os.path.isfile(self.pfile))
+        self.assertFalse(os.path.isfile(self.cfile))
+
+
+# vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_handler_apt_configure.py b/tests/unittests/test_handler/test_handler_apt_configure.py
deleted file mode 100644
index d1dca2c..0000000
--- a/tests/unittests/test_handler/test_handler_apt_configure.py
+++ /dev/null
@@ -1,109 +0,0 @@
-from cloudinit.config import cc_apt_configure
-from cloudinit import util
-
-from ..helpers import TestCase
-
-import os
-import re
-import shutil
-import tempfile
-
-
-def load_tfile_or_url(*args, **kwargs):
-    return(util.decode_binary(util.read_file_or_url(*args, **kwargs).contents))
-
-
-class TestAptProxyConfig(TestCase):
-    def setUp(self):
-        super(TestAptProxyConfig, self).setUp()
-        self.tmp = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, self.tmp)
-        self.pfile = os.path.join(self.tmp, "proxy.cfg")
-        self.cfile = os.path.join(self.tmp, "config.cfg")
-
-    def _search_apt_config(self, contents, ptype, value):
-        return re.search(
-            r"acquire::%s::proxy\s+[\"']%s[\"'];\n" % (ptype, value),
-            contents, flags=re.IGNORECASE)
-
-    def test_apt_proxy_written(self):
-        cfg = {'apt_proxy': 'myproxy'}
-        cc_apt_configure.apply_apt_config(cfg, self.pfile, self.cfile)
-
-        self.assertTrue(os.path.isfile(self.pfile))
-        self.assertFalse(os.path.isfile(self.cfile))
-
-        contents = load_tfile_or_url(self.pfile)
-        self.assertTrue(self._search_apt_config(contents, "http", "myproxy"))
-
-    def test_apt_http_proxy_written(self):
-        cfg = {'apt_http_proxy': 'myproxy'}
-        cc_apt_configure.apply_apt_config(cfg, self.pfile, self.cfile)
-
-        self.assertTrue(os.path.isfile(self.pfile))
-        self.assertFalse(os.path.isfile(self.cfile))
-
-        contents = load_tfile_or_url(self.pfile)
-        self.assertTrue(self._search_apt_config(contents, "http", "myproxy"))
-
-    def test_apt_all_proxy_written(self):
-        cfg = {'apt_http_proxy': 'myproxy_http_proxy',
-               'apt_https_proxy': 'myproxy_https_proxy',
-               'apt_ftp_proxy': 'myproxy_ftp_proxy'}
-
-        values = {'http': cfg['apt_http_proxy'],
-                  'https': cfg['apt_https_proxy'],
-                  'ftp': cfg['apt_ftp_proxy'],
-                  }
-
-        cc_apt_configure.apply_apt_config(cfg, self.pfile, self.cfile)
-
-        self.assertTrue(os.path.isfile(self.pfile))
-        self.assertFalse(os.path.isfile(self.cfile))
-
-        contents = load_tfile_or_url(self.pfile)
-
-        for ptype, pval in values.items():
-            self.assertTrue(self._search_apt_config(contents, ptype, pval))
-
-    def test_proxy_deleted(self):
-        util.write_file(self.cfile, "content doesnt matter")
-        cc_apt_configure.apply_apt_config({}, self.pfile, self.cfile)
-        self.assertFalse(os.path.isfile(self.pfile))
-        self.assertFalse(os.path.isfile(self.cfile))
-
-    def test_proxy_replaced(self):
-        util.write_file(self.cfile, "content doesnt matter")
-        cc_apt_configure.apply_apt_config({'apt_proxy': "foo"},
-                                          self.pfile, self.cfile)
-        self.assertTrue(os.path.isfile(self.pfile))
-        contents = load_tfile_or_url(self.pfile)
-        self.assertTrue(self._search_apt_config(contents, "http", "foo"))
-
-    def test_config_written(self):
-        payload = 'this is my apt config'
-        cfg = {'apt_config': payload}
-
-        cc_apt_configure.apply_apt_config(cfg, self.pfile, self.cfile)
-
-        self.assertTrue(os.path.isfile(self.cfile))
-        self.assertFalse(os.path.isfile(self.pfile))
-
-        self.assertEqual(load_tfile_or_url(self.cfile), payload)
-
-    def test_config_replaced(self):
-        util.write_file(self.pfile, "content doesnt matter")
-        cc_apt_configure.apply_apt_config({'apt_config': "foo"},
-                                          self.pfile, self.cfile)
-        self.assertTrue(os.path.isfile(self.cfile))
-        self.assertEqual(load_tfile_or_url(self.cfile), "foo")
-
-    def test_config_deleted(self):
-        # if no 'apt_config' is provided, delete any previously written file
-        util.write_file(self.pfile, "content doesnt matter")
-        cc_apt_configure.apply_apt_config({}, self.pfile, self.cfile)
-        self.assertFalse(os.path.isfile(self.pfile))
-        self.assertFalse(os.path.isfile(self.cfile))
-
-
-# vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py
deleted file mode 100644
index acde086..0000000
--- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py
+++ /dev/null
@@ -1,180 +0,0 @@
-""" test_handler_apt_configure_sources_list
-Test templating of sources list
-"""
-import logging
-import os
-import shutil
-import tempfile
-
-try:
-    from unittest import mock
-except ImportError:
-    import mock
-
-from cloudinit import cloud
-from cloudinit import distros
-from cloudinit import helpers
-from cloudinit import templater
-from cloudinit import util
-
-from cloudinit.config import cc_apt_configure
-from cloudinit.sources import DataSourceNone
-
-from cloudinit.distros.debian import Distro
-
-from .. import helpers as t_help
-
-LOG = logging.getLogger(__name__)
-
-YAML_TEXT_CUSTOM_SL = """
-apt_mirror: http://archive.ubuntu.com/ubuntu/
-apt_custom_sources_list: |
-    ## template:jinja
-    ## Note, this file is written by cloud-init on first boot of an instance
-    ## modifications made here will not survive a re-bundle.
-    ## if you wish to make changes you can:
-    ## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg
-    ##     or do the same in user-data
-    ## b.) add sources in /etc/apt/sources.list.d
-    ## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl
-
-    # See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to
-    # newer versions of the distribution.
-    deb {{mirror}} {{codename}} main restricted
-    deb-src {{mirror}} {{codename}} main restricted
-    # FIND_SOMETHING_SPECIAL
-"""
-
-EXPECTED_CONVERTED_CONTENT = (
-    """## Note, this file is written by cloud-init on first boot of an instance
-## modifications made here will not survive a re-bundle.
-## if you wish to make changes you can:
-## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg
-##     or do the same in user-data
-## b.) add sources in /etc/apt/sources.list.d
-## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl
-
-# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to
-# newer versions of the distribution.
-deb http://archive.ubuntu.com/ubuntu/ fakerelease main restricted
-deb-src http://archive.ubuntu.com/ubuntu/ fakerelease main restricted
-# FIND_SOMETHING_SPECIAL
-""")
-
-
-def load_tfile_or_url(*args, **kwargs):
-    """load_tfile_or_url
-    load file and return content after decoding
-    """
-    return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents)
-
-
-class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase):
-    """TestAptSourceConfigSourceList
-    Main Class to test sources list rendering
-    """
-    def setUp(self):
-        super(TestAptSourceConfigSourceList, self).setUp()
-        self.subp = util.subp
-        self.new_root = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, self.new_root)
-
-    def _get_cloud(self, distro, metadata=None):
-        self.patchUtils(self.new_root)
-        paths = helpers.Paths({})
-        cls = distros.fetch(distro)
-        mydist = cls(distro, {}, paths)
-        myds = DataSourceNone.DataSourceNone({}, mydist, paths)
-        if metadata:
-            myds.metadata.update(metadata)
-        return cloud.Cloud(myds, paths, {}, mydist, None)
-
-    def apt_source_list(self, distro, mirror, mirrorcheck=None):
-        """apt_source_list
-        Test rendering of a source.list from template for a given distro
-        """
-        if mirrorcheck is None:
-            mirrorcheck = mirror
-
-        if isinstance(mirror, list):
-            cfg = {'apt_mirror_search': mirror}
-        else:
-            cfg = {'apt_mirror': mirror}
-        mycloud = self._get_cloud(distro)
-
-        with mock.patch.object(templater, 'render_to_file') as mocktmpl:
-            with mock.patch.object(os.path, 'isfile',
-                                   return_value=True) as mockisfile:
-                with mock.patch.object(util, 'rename'):
-                    cc_apt_configure.handle("notimportant", cfg, mycloud,
-                                            LOG, None)
-
-        mockisfile.assert_any_call(
-            ('/etc/cloud/templates/sources.list.%s.tmpl' % distro))
-        mocktmpl.assert_called_once_with(
-            ('/etc/cloud/templates/sources.list.%s.tmpl' % distro),
-            '/etc/apt/sources.list',
-            {'codename': '', 'primary': mirrorcheck, 'mirror': mirrorcheck})
-
-    def test_apt_source_list_debian(self):
-        """Test rendering of a source.list from template for debian"""
-        self.apt_source_list('debian', 'http://httpredir.debian.org/debian')
-
-    def test_apt_source_list_ubuntu(self):
-        """Test rendering of a source.list from template for ubuntu"""
-        self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/')
-
-    @staticmethod
-    def myresolve(name):
-        """Fake util.is_resolvable for mirrorfail tests"""
-        if name == "does.not.exist":
-            print("Faking FAIL for '%s'" % name)
-            return False
-        else:
-            print("Faking SUCCESS for '%s'" % name)
-            return True
-
-    def test_apt_srcl_debian_mirrorfail(self):
-        """Test rendering of a source.list from template for debian"""
-        with mock.patch.object(util, 'is_resolvable',
-                               side_effect=self.myresolve) as mockresolve:
-            self.apt_source_list('debian',
-                                 ['http://does.not.exist',
-                                  'http://httpredir.debian.org/debian'],
-                                 'http://httpredir.debian.org/debian')
-        mockresolve.assert_any_call("does.not.exist")
-        mockresolve.assert_any_call("httpredir.debian.org")
-
-    def test_apt_srcl_ubuntu_mirrorfail(self):
-        """Test rendering of a source.list from template for ubuntu"""
-        with mock.patch.object(util, 'is_resolvable',
-                               side_effect=self.myresolve) as mockresolve:
-            self.apt_source_list('ubuntu',
-                                 ['http://does.not.exist',
-                                  'http://archive.ubuntu.com/ubuntu/'],
-                                 'http://archive.ubuntu.com/ubuntu/')
-        mockresolve.assert_any_call("does.not.exist")
-        mockresolve.assert_any_call("archive.ubuntu.com")
-
-    def test_apt_srcl_custom(self):
-        """Test rendering from a custom source.list template"""
-        cfg = util.load_yaml(YAML_TEXT_CUSTOM_SL)
-        mycloud = self._get_cloud('ubuntu')
-
-        # the second mock restores the original subp
-        with mock.patch.object(util, 'write_file') as mockwrite:
-            with mock.patch.object(util, 'subp', self.subp):
-                with mock.patch.object(cc_apt_configure, 'get_release',
-                                       return_value='fakerelease'):
-                    with mock.patch.object(Distro, 'get_primary_arch',
-                                           return_value='amd64'):
-                        cc_apt_configure.handle("notimportant", cfg, mycloud,
-                                                LOG, None)
-
-        mockwrite.assert_called_once_with(
-            '/etc/apt/sources.list',
-            EXPECTED_CONVERTED_CONTENT,
-            mode=420)
-
-
-# vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py
new file mode 100644
index 0000000..f441186
--- /dev/null
+++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py
@@ -0,0 +1,200 @@
+""" test_handler_apt_configure_sources_list
+Test templating of sources list
+"""
+import logging
+import os
+import shutil
+import tempfile
+
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+
+from cloudinit import cloud
+from cloudinit import distros
+from cloudinit import helpers
+from cloudinit import templater
+from cloudinit import util
+
+from cloudinit.config import cc_apt_configure
+from cloudinit.sources import DataSourceNone
+
+from cloudinit.distros.debian import Distro
+
+from .. import helpers as t_help
+
+LOG = logging.getLogger(__name__)
+
+YAML_TEXT_CUSTOM_SL = """
+apt_mirror: http://archive.ubuntu.com/ubuntu/
+apt_custom_sources_list: |
+    ## template:jinja
+    ## Note, this file is written by cloud-init on first boot of an instance
+    ## modifications made here will not survive a re-bundle.
+    ## if you wish to make changes you can:
+    ## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg
+    ##     or do the same in user-data
+    ## b.) add sources in /etc/apt/sources.list.d
+    ## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl
+
+    # See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to
+    # newer versions of the distribution.
+    deb {{mirror}} {{codename}} main restricted
+    deb-src {{mirror}} {{codename}} main restricted
+    # FIND_SOMETHING_SPECIAL
+"""
+
+EXPECTED_CONVERTED_CONTENT = (
+    """## Note, this file is written by cloud-init on first boot of an instance
+## modifications made here will not survive a re-bundle.
+## if you wish to make changes you can:
+## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg
+##     or do the same in user-data
+## b.) add sources in /etc/apt/sources.list.d
+## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl
+
+# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to
+# newer versions of the distribution.
+deb http://archive.ubuntu.com/ubuntu/ fakerelease main restricted
+deb-src http://archive.ubuntu.com/ubuntu/ fakerelease main restricted
+# FIND_SOMETHING_SPECIAL
+""")
+
+
+def load_tfile_or_url(*args, **kwargs):
+    """load_tfile_or_url
+    load file and return content after decoding
+    """
+    return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents)
+
+
+class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase):
+    """TestAptSourceConfigSourceList
+    Main Class to test sources list rendering
+    """
+    def setUp(self):
+        super(TestAptSourceConfigSourceList, self).setUp()
+        self.subp = util.subp
+        self.new_root = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.new_root)
+
+        rpatcher = mock.patch("cloudinit.util.lsb_release")
+        get_rel = rpatcher.start()
+        get_rel.return_value = {'codename': "fakerelease"}
+        self.addCleanup(rpatcher.stop)
+        apatcher = mock.patch("cloudinit.util.get_architecture")
+        get_arch = apatcher.start()
+        get_arch.return_value = 'amd64'
+        self.addCleanup(apatcher.stop)
+
+    def _get_cloud(self, distro, metadata=None):
+        self.patchUtils(self.new_root)
+        paths = helpers.Paths({})
+        cls = distros.fetch(distro)
+        mydist = cls(distro, {}, paths)
+        myds = DataSourceNone.DataSourceNone({}, mydist, paths)
+        if metadata:
+            myds.metadata.update(metadata)
+        return cloud.Cloud(myds, paths, {}, mydist, None)
+
+    def apt_source_list(self, distro, mirror, mirrorcheck=None):
+        """apt_source_list
+        Test rendering of a source.list from template for a given distro
+        """
+        if mirrorcheck is None:
+            mirrorcheck = mirror
+
+        if isinstance(mirror, list):
+            cfg = {'apt_mirror_search': mirror}
+        else:
+            cfg = {'apt_mirror': mirror}
+        mycloud = self._get_cloud(distro)
+
+        with mock.patch.object(util, 'write_file') as mockwf:
+            with mock.patch.object(util, 'load_file',
+                                   return_value="faketmpl") as mocklf:
+                with mock.patch.object(os.path, 'isfile',
+                                       return_value=True) as mockisfile:
+                    with mock.patch.object(templater, 'render_string',
+                                           return_value="fake") as mockrnd:
+                        with mock.patch.object(util, 'rename'):
+                            cc_apt_configure.handle("test", cfg, mycloud,
+                                                    LOG, None)
+
+        mockisfile.assert_any_call(
+            ('/etc/cloud/templates/sources.list.%s.tmpl' % distro))
+        mocklf.assert_any_call(
+            ('/etc/cloud/templates/sources.list.%s.tmpl' % distro))
+        mockrnd.assert_called_once_with('faketmpl',
+                                        {'RELEASE': 'fakerelease',
+                                         'PRIMARY': mirrorcheck,
+                                         'MIRROR': mirrorcheck,
+                                         'SECURITY': mirrorcheck,
+                                         'codename': 'fakerelease',
+                                         'primary': mirrorcheck,
+                                         'mirror': mirrorcheck,
+                                         'security': mirrorcheck})
+        mockwf.assert_called_once_with('/etc/apt/sources.list', 'fake',
+                                       mode=0o644)
+
+    def test_apt_v1_source_list_debian(self):
+        """Test rendering of a source.list from template for debian"""
+        self.apt_source_list('debian', 'http://httpredir.debian.org/debian')
+
+    def test_apt_v1_source_list_ubuntu(self):
+        """Test rendering of a source.list from template for ubuntu"""
+        self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/')
+
+    @staticmethod
+    def myresolve(name):
+        """Fake util.is_resolvable for mirrorfail tests"""
+        if name == "does.not.exist":
+            print("Faking FAIL for '%s'" % name)
+            return False
+        else:
+            print("Faking SUCCESS for '%s'" % name)
+            return True
+
+    def test_apt_v1_srcl_debian_mirrorfail(self):
+        """Test rendering of a source.list from template for debian"""
+        with mock.patch.object(util, 'is_resolvable',
+                               side_effect=self.myresolve) as mockresolve:
+            self.apt_source_list('debian',
+                                 ['http://does.not.exist',
+                                  'http://httpredir.debian.org/debian'],
+                                 'http://httpredir.debian.org/debian')
+        mockresolve.assert_any_call("does.not.exist")
+        mockresolve.assert_any_call("httpredir.debian.org")
+
+    def test_apt_v1_srcl_ubuntu_mirrorfail(self):
+        """Test rendering of a source.list from template for ubuntu"""
+        with mock.patch.object(util, 'is_resolvable',
+                               side_effect=self.myresolve) as mockresolve:
+            self.apt_source_list('ubuntu',
+                                 ['http://does.not.exist',
+                                  'http://archive.ubuntu.com/ubuntu/'],
+                                 'http://archive.ubuntu.com/ubuntu/')
+        mockresolve.assert_any_call("does.not.exist")
+        mockresolve.assert_any_call("archive.ubuntu.com")
+
+    def test_apt_v1_srcl_custom(self):
+        """Test rendering from a custom source.list template"""
+        cfg = util.load_yaml(YAML_TEXT_CUSTOM_SL)
+        mycloud = self._get_cloud('ubuntu')
+
+        # the second mock restores the original subp
+        with mock.patch.object(util, 'write_file') as mockwrite:
+            with mock.patch.object(util, 'subp', self.subp):
+                with mock.patch.object(Distro, 'get_primary_arch',
+                                       return_value='amd64'):
+                    cc_apt_configure.handle("notimportant", cfg, mycloud,
+                                            LOG, None)
+
+        mockwrite.assert_called_once_with(
+            '/etc/apt/sources.list',
+            EXPECTED_CONVERTED_CONTENT,
+            mode=420)
+
+
+# vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py
new file mode 100644
index 0000000..e53b045
--- /dev/null
+++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py
@@ -0,0 +1,187 @@
+""" test_apt_custom_sources_list
+Test templating of custom sources list
+"""
+import logging
+import os
+import shutil
+import tempfile
+
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+from mock import call
+
+from cloudinit import cloud
+from cloudinit import distros
+from cloudinit import helpers
+from cloudinit import util
+
+from cloudinit.config import cc_apt_configure
+from cloudinit.sources import DataSourceNone
+
+from cloudinit.distros.debian import Distro
+
+from .. import helpers as t_help
+
+LOG = logging.getLogger(__name__)
+
+TARGET = "/"
+
+# Input and expected output for the custom template
+YAML_TEXT_CUSTOM_SL = """
+apt:
+  primary:
+    - arches: [default]
+      uri: http://test.ubuntu.com/ubuntu/
+  security:
+    - arches: [default]
+      uri: http://testsec.ubuntu.com/ubuntu/
+  sources_list: |
+
+      # Note, this file is written by cloud-init at install time. It should not
+      # end up on the installed system itself.
+      # See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to
+      # newer versions of the distribution.
+      deb $MIRROR $RELEASE main restricted
+      deb-src $MIRROR $RELEASE main restricted
+      deb $PRIMARY $RELEASE universe restricted
+      deb $SECURITY $RELEASE-security multiverse
+      # FIND_SOMETHING_SPECIAL
+"""
+
+EXPECTED_CONVERTED_CONTENT = """
+# Note, this file is written by cloud-init at install time. It should not
+# end up on the installed system itself.
+# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to
+# newer versions of the distribution.
+deb http://test.ubuntu.com/ubuntu/ fakerel main restricted
+deb-src http://test.ubuntu.com/ubuntu/ fakerel main restricted
+deb http://test.ubuntu.com/ubuntu/ fakerel universe restricted
+deb http://testsec.ubuntu.com/ubuntu/ fakerel-security multiverse
+# FIND_SOMETHING_SPECIAL
+"""
+
+# mocked to be independent to the unittest system
+MOCKED_APT_SRC_LIST = """
+deb http://test.ubuntu.com/ubuntu/ notouched main restricted
+deb-src http://test.ubuntu.com/ubuntu/ notouched main restricted
+deb http://test.ubuntu.com/ubuntu/ notouched-updates main restricted
+deb http://testsec.ubuntu.com/ubuntu/ notouched-security main restricted
+"""
+
+EXPECTED_BASE_CONTENT = ("""
+deb http://test.ubuntu.com/ubuntu/ notouched main restricted
+deb-src http://test.ubuntu.com/ubuntu/ notouched main restricted
+deb http://test.ubuntu.com/ubuntu/ notouched-updates main restricted
+deb http://testsec.ubuntu.com/ubuntu/ notouched-security main restricted
+""")
+
+EXPECTED_MIRROR_CONTENT = ("""
+deb http://test.ubuntu.com/ubuntu/ notouched main restricted
+deb-src http://test.ubuntu.com/ubuntu/ notouched main restricted
+deb http://test.ubuntu.com/ubuntu/ notouched-updates main restricted
+deb http://test.ubuntu.com/ubuntu/ notouched-security main restricted
+""")
+
+EXPECTED_PRIMSEC_CONTENT = ("""
+deb http://test.ubuntu.com/ubuntu/ notouched main restricted
+deb-src http://test.ubuntu.com/ubuntu/ notouched main restricted
+deb http://test.ubuntu.com/ubuntu/ notouched-updates main restricted
+deb http://testsec.ubuntu.com/ubuntu/ notouched-security main restricted
+""")
+
+
+class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase):
+    """TestAptSourceConfigSourceList - Class to test sources list rendering"""
+    def setUp(self):
+        super(TestAptSourceConfigSourceList, self).setUp()
+        self.subp = util.subp
+        self.new_root = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.new_root)
+
+        rpatcher = mock.patch("cloudinit.util.lsb_release")
+        get_rel = rpatcher.start()
+        get_rel.return_value = {'codename': "fakerel"}
+        self.addCleanup(rpatcher.stop)
+        apatcher = mock.patch("cloudinit.util.get_architecture")
+        get_arch = apatcher.start()
+        get_arch.return_value = 'amd64'
+        self.addCleanup(apatcher.stop)
+
+    def _get_cloud(self, distro, metadata=None):
+        self.patchUtils(self.new_root)
+        paths = helpers.Paths({})
+        cls = distros.fetch(distro)
+        mydist = cls(distro, {}, paths)
+        myds = DataSourceNone.DataSourceNone({}, mydist, paths)
+        if metadata:
+            myds.metadata.update(metadata)
+        return cloud.Cloud(myds, paths, {}, mydist, None)
+
+    def _apt_source_list(self, cfg, expected, distro):
+        "_apt_source_list - Test rendering from template (generic)"
+
+        # entry at top level now, wrap in 'apt' key
+        cfg = {'apt': cfg}
+        mycloud = self._get_cloud(distro)
+        with mock.patch.object(util, 'write_file') as mockwf:
+            with mock.patch.object(util, 'load_file',
+                                   return_value=MOCKED_APT_SRC_LIST) as mocklf:
+                with mock.patch.object(os.path, 'isfile',
+                                       return_value=True) as mockisfile:
+                    with mock.patch.object(util, 'rename'):
+                        cc_apt_configure.handle("test", cfg, mycloud,
+                                                LOG, None)
+
+        # check if it would have loaded the distro template
+        mockisfile.assert_any_call(
+            ('/etc/cloud/templates/sources.list.%s.tmpl' % distro))
+        mocklf.assert_any_call(
+            ('/etc/cloud/templates/sources.list.%s.tmpl' % distro))
+        # check expected content in result
+        mockwf.assert_called_once_with('/etc/apt/sources.list', expected,
+                                       mode=0o644)
+
+    def test_apt_v3_source_list_debian(self):
+        """test_apt_v3_source_list_debian - without custom sources or parms"""
+        cfg = {}
+        self._apt_source_list(cfg, EXPECTED_BASE_CONTENT, 'debian')
+
+    def test_apt_v3_source_list_ubuntu(self):
+        """test_apt_v3_source_list_ubuntu - without custom sources or parms"""
+        cfg = {}
+        self._apt_source_list(cfg, EXPECTED_BASE_CONTENT, 'ubuntu')
+
+    def test_apt_v3_source_list_psm(self):
+        """test_apt_v3_source_list_psm - Test specifying prim+sec mirrors"""
+        pm = 'http://test.ubuntu.com/ubuntu/'
+        sm = 'http://testsec.ubuntu.com/ubuntu/'
+        cfg = {'preserve_sources_list': False,
+               'primary': [{'arches': ["default"],
+                            'uri': pm}],
+               'security': [{'arches': ["default"],
+                             'uri': sm}]}
+
+        self._apt_source_list(cfg, EXPECTED_PRIMSEC_CONTENT, 'ubuntu')
+
+    def test_apt_v3_srcl_custom(self):
+        """test_apt_v3_srcl_custom - Test rendering a custom source template"""
+        cfg = util.load_yaml(YAML_TEXT_CUSTOM_SL)
+        mycloud = self._get_cloud('ubuntu')
+
+        # the second mock restores the original subp
+        with mock.patch.object(util, 'write_file') as mockwrite:
+            with mock.patch.object(util, 'subp', self.subp):
+                with mock.patch.object(Distro, 'get_primary_arch',
+                                       return_value='amd64'):
+                    cc_apt_configure.handle("notimportant", cfg, mycloud,
+                                            LOG, None)
+
+        calls = [call('/etc/apt/sources.list',
+                      EXPECTED_CONVERTED_CONTENT,
+                      mode=0o644)]
+        mockwrite.assert_has_calls(calls)
+
+
+# vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py
deleted file mode 100644
index 99a4d86..0000000
--- a/tests/unittests/test_handler/test_handler_apt_source.py
+++ /dev/null
@@ -1,516 +0,0 @@
-""" test_handler_apt_source
-Testing various config variations of the apt_source config
-"""
-import os
-import re
-import shutil
-import tempfile
-
-try:
-    from unittest import mock
-except ImportError:
-    import mock
-from mock import call
-
-from cloudinit.config import cc_apt_configure
-from cloudinit import gpg
-from cloudinit import util
-
-from ..helpers import TestCase
-
-EXPECTEDKEY = """-----BEGIN PGP PUBLIC KEY BLOCK-----
-Version: GnuPG v1
-
-mI0ESuZLUgEEAKkqq3idtFP7g9hzOu1a8+v8ImawQN4TrvlygfScMU1TIS1eC7UQ
-NUA8Qqgr9iUaGnejb0VciqftLrU9D6WYHSKz+EITefgdyJ6SoQxjoJdsCpJ7o9Jy
-8PQnpRttiFm4qHu6BVnKnBNxw/z3ST9YMqW5kbMQpfxbGe+obRox59NpABEBAAG0
-HUxhdW5jaHBhZCBQUEEgZm9yIFNjb3R0IE1vc2VyiLYEEwECACAFAkrmS1ICGwMG
-CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRAGILvPA2g/d3aEA/9tVjc10HOZwV29
-OatVuTeERjjrIbxflO586GLA8cp0C9RQCwgod/R+cKYdQcHjbqVcP0HqxveLg0RZ
-FJpWLmWKamwkABErwQLGlM/Hwhjfade8VvEQutH5/0JgKHmzRsoqfR+LMO6OS+Sm
-S0ORP6HXET3+jC8BMG4tBWCTK/XEZw==
-=ACB2
------END PGP PUBLIC KEY BLOCK-----"""
-
-
-def load_tfile_or_url(*args, **kwargs):
-    """load_tfile_or_url
-    load file and return content after decoding
-    """
-    return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents)
-
-
-class TestAptSourceConfig(TestCase):
-    """TestAptSourceConfig
-    Main Class to test apt_source configs
-    """
-    release = "fantastic"
-
-    def setUp(self):
-        super(TestAptSourceConfig, self).setUp()
-        self.tmp = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, self.tmp)
-        self.aptlistfile = os.path.join(self.tmp, "single-deb.list")
-        self.aptlistfile2 = os.path.join(self.tmp, "single-deb2.list")
-        self.aptlistfile3 = os.path.join(self.tmp, "single-deb3.list")
-        self.join = os.path.join
-        # mock fallback filename into writable tmp dir
-        self.fallbackfn = os.path.join(self.tmp, "etc/apt/sources.list.d/",
-                                       "cloud_config_sources.list")
-
-        patcher = mock.patch("cloudinit.config.cc_apt_configure.get_release")
-        get_rel = patcher.start()
-        get_rel.return_value = self.release
-        self.addCleanup(patcher.stop)
-
-    @staticmethod
-    def _get_default_params():
-        """get_default_params
-        Get the most basic default mrror and release info to be used in tests
-        """
-        params = {}
-        params['RELEASE'] = cc_apt_configure.get_release()
-        params['MIRROR'] = "http://archive.ubuntu.com/ubuntu"
-        return params
-
-    def myjoin(self, *args, **kwargs):
-        """myjoin - redir into writable tmpdir"""
-        if (args[0] == "/etc/apt/sources.list.d/" and
-                args[1] == "cloud_config_sources.list" and
-                len(args) == 2):
-            return self.join(self.tmp, args[0].lstrip("/"), args[1])
-        else:
-            return self.join(*args, **kwargs)
-
-    def apt_src_basic(self, filename, cfg):
-        """apt_src_basic
-        Test Fix deb source string, has to overwrite mirror conf in params
-        """
-        params = self._get_default_params()
-
-        cc_apt_configure.add_apt_sources(cfg, params)
-
-        self.assertTrue(os.path.isfile(filename))
-
-        contents = load_tfile_or_url(filename)
-        self.assertTrue(re.search(r"%s %s %s %s\n" %
-                                  ("deb", "http://archive.ubuntu.com/ubuntu",
-                                   "karmic-backports",
-                                   "main universe multiverse restricted"),
-                                  contents, flags=re.IGNORECASE))
-
-    def test_apt_src_basic(self):
-        """Test deb source string, overwrite mirror and filename"""
-        cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu'
-                          ' karmic-backports'
-                          ' main universe multiverse restricted'),
-               'filename': self.aptlistfile}
-        self.apt_src_basic(self.aptlistfile, [cfg])
-
-    def test_apt_src_basic_dict(self):
-        """Test deb source string, overwrite mirror and filename (dict)"""
-        cfg = {self.aptlistfile: {'source':
-                                  ('deb http://archive.ubuntu.com/ubuntu'
-                                   ' karmic-backports'
-                                   ' main universe multiverse restricted')}}
-        self.apt_src_basic(self.aptlistfile, cfg)
-
-    def apt_src_basic_tri(self, cfg):
-        """apt_src_basic_tri
-        Test Fix three deb source string, has to overwrite mirror conf in
-        params. Test with filenames provided in config.
-        generic part to check three files with different content
-        """
-        self.apt_src_basic(self.aptlistfile, cfg)
-
-        # extra verify on two extra files of this test
-        contents = load_tfile_or_url(self.aptlistfile2)
-        self.assertTrue(re.search(r"%s %s %s %s\n" %
-                                  ("deb", "http://archive.ubuntu.com/ubuntu",
-                                   "precise-backports",
-                                   "main universe multiverse restricted"),
-                                  contents, flags=re.IGNORECASE))
-        contents = load_tfile_or_url(self.aptlistfile3)
-        self.assertTrue(re.search(r"%s %s %s %s\n" %
-                                  ("deb", "http://archive.ubuntu.com/ubuntu",
-                                   "lucid-backports",
-                                   "main universe multiverse restricted"),
-                                  contents, flags=re.IGNORECASE))
-
-    def test_apt_src_basic_tri(self):
-        """Test Fix three deb source string with filenames"""
-        cfg1 = {'source': ('deb http://archive.ubuntu.com/ubuntu'
-                           ' karmic-backports'
-                           ' main universe multiverse restricted'),
-                'filename': self.aptlistfile}
-        cfg2 = {'source': ('deb http://archive.ubuntu.com/ubuntu'
-                           ' precise-backports'
-                           ' main universe multiverse restricted'),
-                'filename': self.aptlistfile2}
-        cfg3 = {'source': ('deb http://archive.ubuntu.com/ubuntu'
-                           ' lucid-backports'
-                           ' main universe multiverse restricted'),
-                'filename': self.aptlistfile3}
-        self.apt_src_basic_tri([cfg1, cfg2, cfg3])
-
-    def test_apt_src_basic_dict_tri(self):
-        """Test Fix three deb source string with filenames (dict)"""
-        cfg = {self.aptlistfile: {'source':
-                                  ('deb http://archive.ubuntu.com/ubuntu'
-                                   ' karmic-backports'
-                                   ' main universe multiverse restricted')},
-               self.aptlistfile2: {'source':
-                                   ('deb http://archive.ubuntu.com/ubuntu'
-                                    ' precise-backports'
-                                    ' main universe multiverse restricted')},
-               self.aptlistfile3: {'source':
-                                   ('deb http://archive.ubuntu.com/ubuntu'
-                                    ' lucid-backports'
-                                    ' main universe multiverse restricted')}}
-        self.apt_src_basic_tri(cfg)
-
-    def test_apt_src_basic_nofn(self):
-        """Test Fix three deb source string without filenames (dict)"""
-        cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu'
-                          ' karmic-backports'
-                          ' main universe multiverse restricted')}
-        with mock.patch.object(os.path, 'join', side_effect=self.myjoin):
-            self.apt_src_basic(self.fallbackfn, [cfg])
-
-    def apt_src_replacement(self, filename, cfg):
-        """apt_src_replace
-        Test Autoreplacement of MIRROR and RELEASE in source specs
-        """
-        params = self._get_default_params()
-        cc_apt_configure.add_apt_sources(cfg, params)
-
-        self.assertTrue(os.path.isfile(filename))
-
-        contents = load_tfile_or_url(filename)
-        self.assertTrue(re.search(r"%s %s %s %s\n" %
-                                  ("deb", params['MIRROR'], params['RELEASE'],
-                                   "multiverse"),
-                                  contents, flags=re.IGNORECASE))
-
-    def test_apt_src_replace(self):
-        """Test Autoreplacement of MIRROR and RELEASE in source specs"""
-        cfg = {'source': 'deb $MIRROR $RELEASE multiverse',
-               'filename': self.aptlistfile}
-        self.apt_src_replacement(self.aptlistfile, [cfg])
-
-    def apt_src_replace_tri(self, cfg):
-        """apt_src_replace_tri
-        Test three autoreplacements of MIRROR and RELEASE in source specs with
-        generic part
-        """
-        self.apt_src_replacement(self.aptlistfile, cfg)
-
-        # extra verify on two extra files of this test
-        params = self._get_default_params()
-        contents = load_tfile_or_url(self.aptlistfile2)
-        self.assertTrue(re.search(r"%s %s %s %s\n" %
-                                  ("deb", params['MIRROR'], params['RELEASE'],
-                                   "main"),
-                                  contents, flags=re.IGNORECASE))
-        contents = load_tfile_or_url(self.aptlistfile3)
-        self.assertTrue(re.search(r"%s %s %s %s\n" %
-                                  ("deb", params['MIRROR'], params['RELEASE'],
-                                   "universe"),
-                                  contents, flags=re.IGNORECASE))
-
-    def test_apt_src_replace_tri(self):
-        """Test triple Autoreplacement of MIRROR and RELEASE in source specs"""
-        cfg1 = {'source': 'deb $MIRROR $RELEASE multiverse',
-                'filename': self.aptlistfile}
-        cfg2 = {'source': 'deb $MIRROR $RELEASE main',
-                'filename': self.aptlistfile2}
-        cfg3 = {'source': 'deb $MIRROR $RELEASE universe',
-                'filename': self.aptlistfile3}
-        self.apt_src_replace_tri([cfg1, cfg2, cfg3])
-
-    def test_apt_src_replace_dict_tri(self):
-        """Test triple Autoreplacement in source specs (dict)"""
-        cfg = {self.aptlistfile: {'source': 'deb $MIRROR $RELEASE multiverse'},
-               'notused': {'source': 'deb $MIRROR $RELEASE main',
-                           'filename': self.aptlistfile2},
-               self.aptlistfile3: {'source': 'deb $MIRROR $RELEASE universe'}}
-        self.apt_src_replace_tri(cfg)
-
-    def test_apt_src_replace_nofn(self):
-        """Test Autoreplacement of MIRROR and RELEASE in source specs nofile"""
-        cfg = {'source': 'deb $MIRROR $RELEASE multiverse'}
-        with mock.patch.object(os.path, 'join', side_effect=self.myjoin):
-            self.apt_src_replacement(self.fallbackfn, [cfg])
-
-    def apt_src_keyid(self, filename, cfg, keynum):
-        """apt_src_keyid
-        Test specification of a source + keyid
-        """
-        params = self._get_default_params()
-
-        with mock.patch.object(util, 'subp',
-                               return_value=('fakekey 1234', '')) as mockobj:
-            cc_apt_configure.add_apt_sources(cfg, params)
-
-        # check if it added the right ammount of keys
-        calls = []
-        for _ in range(keynum):
-            calls.append(call(('apt-key', 'add', '-'), 'fakekey 1234'))
-        mockobj.assert_has_calls(calls, any_order=True)
-
-        self.assertTrue(os.path.isfile(filename))
-
-        contents = load_tfile_or_url(filename)
-        self.assertTrue(re.search(r"%s %s %s %s\n" %
-                                  ("deb",
-                                   ('http://ppa.launchpad.net/smoser/'
-                                    'cloud-init-test/ubuntu'),
-                                   "xenial", "main"),
-                                  contents, flags=re.IGNORECASE))
-
-    def test_apt_src_keyid(self):
-        """Test specification of a source + keyid with filename being set"""
-        cfg = {'source': ('deb '
-                          'http://ppa.launchpad.net/'
-                          'smoser/cloud-init-test/ubuntu'
-                          ' xenial main'),
-               'keyid': "03683F77",
-               'filename': self.aptlistfile}
-        self.apt_src_keyid(self.aptlistfile, [cfg], 1)
-
-    def test_apt_src_keyid_tri(self):
-        """Test 3x specification of a source + keyid with filename being set"""
-        cfg1 = {'source': ('deb '
-                           'http://ppa.launchpad.net/'
-                           'smoser/cloud-init-test/ubuntu'
-                           ' xenial main'),
-                'keyid': "03683F77",
-                'filename': self.aptlistfile}
-        cfg2 = {'source': ('deb '
-                           'http://ppa.launchpad.net/'
-                           'smoser/cloud-init-test/ubuntu'
-                           ' xenial universe'),
-                'keyid': "03683F77",
-                'filename': self.aptlistfile2}
-        cfg3 = {'source': ('deb '
-                           'http://ppa.launchpad.net/'
-                           'smoser/cloud-init-test/ubuntu'
-                           ' xenial multiverse'),
-                'keyid': "03683F77",
-                'filename': self.aptlistfile3}
-
-        self.apt_src_keyid(self.aptlistfile, [cfg1, cfg2, cfg3], 3)
-        contents = load_tfile_or_url(self.aptlistfile2)
-        self.assertTrue(re.search(r"%s %s %s %s\n" %
-                                  ("deb",
-                                   ('http://ppa.launchpad.net/smoser/'
-                                    'cloud-init-test/ubuntu'),
-                                   "xenial", "universe"),
-                                  contents, flags=re.IGNORECASE))
-        contents = load_tfile_or_url(self.aptlistfile3)
-        self.assertTrue(re.search(r"%s %s %s %s\n" %
-                                  ("deb",
-                                   ('http://ppa.launchpad.net/smoser/'
-                                    'cloud-init-test/ubuntu'),
-                                   "xenial", "multiverse"),
-                                  contents, flags=re.IGNORECASE))
-
-    def test_apt_src_keyid_nofn(self):
-        """Test specification of a source + keyid without filename being set"""
-        cfg = {'source': ('deb '
-                          'http://ppa.launchpad.net/'
-                          'smoser/cloud-init-test/ubuntu'
-                          ' xenial main'),
-               'keyid': "03683F77"}
-        with mock.patch.object(os.path, 'join', side_effect=self.myjoin):
-            self.apt_src_keyid(self.fallbackfn, [cfg], 1)
-
-    def apt_src_key(self, filename, cfg):
-        """apt_src_key
-        Test specification of a source + key
-        """
-        params = self._get_default_params()
-
-        with mock.patch.object(util, 'subp') as mockobj:
-            cc_apt_configure.add_apt_sources([cfg], params)
-
-        mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 4321')
-
-        self.assertTrue(os.path.isfile(filename))
-
-        contents = load_tfile_or_url(filename)
-        self.assertTrue(re.search(r"%s %s %s %s\n" %
-                                  ("deb",
-                                   ('http://ppa.launchpad.net/smoser/'
-                                    'cloud-init-test/ubuntu'),
-                                   "xenial", "main"),
-                                  contents, flags=re.IGNORECASE))
-
-    def test_apt_src_key(self):
-        """Test specification of a source + key with filename being set"""
-        cfg = {'source': ('deb '
-                          'http://ppa.launchpad.net/'
-                          'smoser/cloud-init-test/ubuntu'
-                          ' xenial main'),
-               'key': "fakekey 4321",
-               'filename': self.aptlistfile}
-        self.apt_src_key(self.aptlistfile, cfg)
-
-    def test_apt_src_key_nofn(self):
-        """Test specification of a source + key without filename being set"""
-        cfg = {'source': ('deb '
-                          'http://ppa.launchpad.net/'
-                          'smoser/cloud-init-test/ubuntu'
-                          ' xenial main'),
-               'key': "fakekey 4321"}
-        with mock.patch.object(os.path, 'join', side_effect=self.myjoin):
-            self.apt_src_key(self.fallbackfn, cfg)
-
-    def test_apt_src_keyonly(self):
-        """Test specifying key without source"""
-        params = self._get_default_params()
-        cfg = {'key': "fakekey 4242",
-               'filename': self.aptlistfile}
-
-        with mock.patch.object(util, 'subp') as mockobj:
-            cc_apt_configure.add_apt_sources([cfg], params)
-
-        mockobj.assert_called_once_with(('apt-key', 'add', '-'),
-                                        'fakekey 4242')
-
-        # filename should be ignored on key only
-        self.assertFalse(os.path.isfile(self.aptlistfile))
-
-    def test_apt_src_keyidonly(self):
-        """Test specification of a keyid without source"""
-        params = self._get_default_params()
-        cfg = {'keyid': "03683F77",
-               'filename': self.aptlistfile}
-
-        with mock.patch.object(util, 'subp',
-                               return_value=('fakekey 1212', '')) as mockobj:
-            cc_apt_configure.add_apt_sources([cfg], params)
-
-        mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 1212')
-
-        # filename should be ignored on key only
-        self.assertFalse(os.path.isfile(self.aptlistfile))
-
-    def apt_src_keyid_real(self, cfg, expectedkey):
-        """apt_src_keyid_real
-        Test specification of a keyid without source including
-        up to addition of the key (add_apt_key_raw mocked to keep the
-        environment as is)
-        """
-        params = self._get_default_params()
-
-        with mock.patch.object(cc_apt_configure, 'add_apt_key_raw') as mockkey:
-            with mock.patch.object(gpg, 'get_key_by_id',
-                                   return_value=expectedkey) as mockgetkey:
-                cc_apt_configure.add_apt_sources([cfg], params)
-
-        mockgetkey.assert_called_with(cfg['keyid'],
-                                      cfg.get('keyserver',
-                                              'keyserver.ubuntu.com'))
-        mockkey.assert_called_with(expectedkey)
-
-        # filename should be ignored on key only
-        self.assertFalse(os.path.isfile(self.aptlistfile))
-
-    def test_apt_src_keyid_real(self):
-        """test_apt_src_keyid_real - Test keyid including key add"""
-        keyid = "03683F77"
-        cfg = {'keyid': keyid,
-               'filename': self.aptlistfile}
-
-        self.apt_src_keyid_real(cfg, EXPECTEDKEY)
-
-    def test_apt_src_longkeyid_real(self):
-        """test_apt_src_longkeyid_real - Test long keyid including key add"""
-        keyid = "B59D 5F15 97A5 04B7 E230  6DCA 0620 BBCF 0368 3F77"
-        cfg = {'keyid': keyid,
-               'filename': self.aptlistfile}
-
-        self.apt_src_keyid_real(cfg, EXPECTEDKEY)
-
-    def test_apt_src_longkeyid_ks_real(self):
-        """test_apt_src_longkeyid_ks_real - Test long keyid from other ks"""
-        keyid = "B59D 5F15 97A5 04B7 E230  6DCA 0620 BBCF 0368 3F77"
-        cfg = {'keyid': keyid,
-               'keyserver': 'keys.gnupg.net',
-               'filename': self.aptlistfile}
-
-        self.apt_src_keyid_real(cfg, EXPECTEDKEY)
-
-    def test_apt_src_ppa(self):
-        """Test adding a ppa"""
-        params = self._get_default_params()
-        cfg = {'source': 'ppa:smoser/cloud-init-test',
-               'filename': self.aptlistfile}
-
-        # default matcher needed for ppa
-        matcher = re.compile(r'^[\w-]+:\w').search
-
-        with mock.patch.object(util, 'subp') as mockobj:
-            cc_apt_configure.add_apt_sources([cfg], params,
-                                             aa_repo_match=matcher)
-        mockobj.assert_called_once_with(['add-apt-repository',
-                                         'ppa:smoser/cloud-init-test'])
-
-        # adding ppa should ignore filename (uses add-apt-repository)
-        self.assertFalse(os.path.isfile(self.aptlistfile))
-
-    def test_apt_src_ppa_tri(self):
-        """Test adding three ppa's"""
-        params = self._get_default_params()
-        cfg1 = {'source': 'ppa:smoser/cloud-init-test',
-                'filename': self.aptlistfile}
-        cfg2 = {'source': 'ppa:smoser/cloud-init-test2',
-                'filename': self.aptlistfile2}
-        cfg3 = {'source': 'ppa:smoser/cloud-init-test3',
-                'filename': self.aptlistfile3}
-
-        # default matcher needed for ppa
-        matcher = re.compile(r'^[\w-]+:\w').search
-
-        with mock.patch.object(util, 'subp') as mockobj:
-            cc_apt_configure.add_apt_sources([cfg1, cfg2, cfg3], params,
-                                             aa_repo_match=matcher)
-        calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test']),
-                 call(['add-apt-repository', 'ppa:smoser/cloud-init-test2']),
-                 call(['add-apt-repository', 'ppa:smoser/cloud-init-test3'])]
-        mockobj.assert_has_calls(calls, any_order=True)
-
-        # adding ppa should ignore all filenames (uses add-apt-repository)
-        self.assertFalse(os.path.isfile(self.aptlistfile))
-        self.assertFalse(os.path.isfile(self.aptlistfile2))
-        self.assertFalse(os.path.isfile(self.aptlistfile3))
-
-    def test_convert_to_new_format(self):
-        """Test the conversion of old to new format"""
-        cfg1 = {'source': 'deb $MIRROR $RELEASE multiverse',
-                'filename': self.aptlistfile}
-        cfg2 = {'source': 'deb $MIRROR $RELEASE main',
-                'filename': self.aptlistfile2}
-        cfg3 = {'source': 'deb $MIRROR $RELEASE universe',
-                'filename': self.aptlistfile3}
-        checkcfg = {self.aptlistfile: {'filename': self.aptlistfile,
-                                       'source': 'deb $MIRROR $RELEASE '
-                                                 'multiverse'},
-                    self.aptlistfile2: {'filename': self.aptlistfile2,
-                                        'source': 'deb $MIRROR $RELEASE main'},
-                    self.aptlistfile3: {'filename': self.aptlistfile3,
-                                        'source': 'deb $MIRROR $RELEASE '
-                                                  'universe'}}
-
-        newcfg = cc_apt_configure.convert_to_new_format([cfg1, cfg2, cfg3])
-        self.assertEqual(newcfg, checkcfg)
-
-        newcfg2 = cc_apt_configure.convert_to_new_format(newcfg)
-        self.assertEqual(newcfg2, checkcfg)
-
-        with self.assertRaises(ValueError):
-            cc_apt_configure.convert_to_new_format(5)
-
-
-# vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_handler_apt_source_v1.py b/tests/unittests/test_handler/test_handler_apt_source_v1.py
new file mode 100644
index 0000000..d96779c
--- /dev/null
+++ b/tests/unittests/test_handler/test_handler_apt_source_v1.py
@@ -0,0 +1,551 @@
+""" test_handler_apt_source_v1
+Testing various config variations of the apt_source config
+This calls all things with v1 format to stress the conversion code on top of
+the actually tested code.
+"""
+import os
+import re
+import shutil
+import tempfile
+
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+from mock import call
+
+from cloudinit.config import cc_apt_configure
+from cloudinit import gpg
+from cloudinit import util
+
+from ..helpers import TestCase
+
+EXPECTEDKEY = """-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mI0ESuZLUgEEAKkqq3idtFP7g9hzOu1a8+v8ImawQN4TrvlygfScMU1TIS1eC7UQ
+NUA8Qqgr9iUaGnejb0VciqftLrU9D6WYHSKz+EITefgdyJ6SoQxjoJdsCpJ7o9Jy
+8PQnpRttiFm4qHu6BVnKnBNxw/z3ST9YMqW5kbMQpfxbGe+obRox59NpABEBAAG0
+HUxhdW5jaHBhZCBQUEEgZm9yIFNjb3R0IE1vc2VyiLYEEwECACAFAkrmS1ICGwMG
+CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRAGILvPA2g/d3aEA/9tVjc10HOZwV29
+OatVuTeERjjrIbxflO586GLA8cp0C9RQCwgod/R+cKYdQcHjbqVcP0HqxveLg0RZ
+FJpWLmWKamwkABErwQLGlM/Hwhjfade8VvEQutH5/0JgKHmzRsoqfR+LMO6OS+Sm
+S0ORP6HXET3+jC8BMG4tBWCTK/XEZw==
+=ACB2
+-----END PGP PUBLIC KEY BLOCK-----"""
+
+ADD_APT_REPO_MATCH = r"^[\w-]+:\w"
+
+
+def load_tfile_or_url(*args, **kwargs):
+    """load_tfile_or_url
+    load file and return content after decoding
+    """
+    return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents)
+
+
+class FakeDistro(object):
+    """Fake Distro helper object"""
+    def update_package_sources(self):
+        """Fake update_package_sources helper method"""
+        return
+
+
+class FakeCloud(object):
+    """Fake Cloud helper object"""
+    def __init__(self):
+        self.distro = FakeDistro()
+
+
+class TestAptSourceConfig(TestCase):
+    """TestAptSourceConfig
+    Main Class to test apt_source configs
+    """
+    release = "fantastic"
+
+    def setUp(self):
+        super(TestAptSourceConfig, self).setUp()
+        self.tmp = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.tmp)
+        self.aptlistfile = os.path.join(self.tmp, "single-deb.list")
+        self.aptlistfile2 = os.path.join(self.tmp, "single-deb2.list")
+        self.aptlistfile3 = os.path.join(self.tmp, "single-deb3.list")
+        self.join = os.path.join
+        self.matcher = re.compile(ADD_APT_REPO_MATCH).search
+        # mock fallback filename into writable tmp dir
+        self.fallbackfn = os.path.join(self.tmp, "etc/apt/sources.list.d/",
+                                       "cloud_config_sources.list")
+
+        self.fakecloud = FakeCloud()
+
+        rpatcher = mock.patch("cloudinit.util.lsb_release")
+        get_rel = rpatcher.start()
+        get_rel.return_value = {'codename': self.release}
+        self.addCleanup(rpatcher.stop)
+        apatcher = mock.patch("cloudinit.util.get_architecture")
+        get_arch = apatcher.start()
+        get_arch.return_value = 'amd64'
+        self.addCleanup(apatcher.stop)
+
+    def _get_default_params(self):
+        """get_default_params
+        Get the most basic default mrror and release info to be used in tests
+        """
+        params = {}
+        params['RELEASE'] = self.release
+        params['MIRROR'] = "http://archive.ubuntu.com/ubuntu"
+        return params
+
+    def wrapv1conf(self, cfg):
+        params = self._get_default_params()
+        # old v1 list format under old keys, but callabe to main handler
+        # disable source.list rendering and set mirror to avoid other code
+        return {'apt_preserve_sources_list': True,
+                'apt_mirror': params['MIRROR'],
+                'apt_sources': cfg}
+
+    def myjoin(self, *args, **kwargs):
+        """myjoin - redir into writable tmpdir"""
+        if (args[0] == "/etc/apt/sources.list.d/" and
+                args[1] == "cloud_config_sources.list" and
+                len(args) == 2):
+            return self.join(self.tmp, args[0].lstrip("/"), args[1])
+        else:
+            return self.join(*args, **kwargs)
+
+    def apt_src_basic(self, filename, cfg):
+        """apt_src_basic
+        Test Fix deb source string, has to overwrite mirror conf in params
+        """
+        cfg = self.wrapv1conf(cfg)
+
+        cc_apt_configure.handle("test", cfg, self.fakecloud, None, None)
+
+        self.assertTrue(os.path.isfile(filename))
+
+        contents = load_tfile_or_url(filename)
+        self.assertTrue(re.search(r"%s %s %s %s\n" %
+                                  ("deb", "http://archive.ubuntu.com/ubuntu",
+                                   "karmic-backports",
+                                   "main universe multiverse restricted"),
+                                  contents, flags=re.IGNORECASE))
+
+    def test_apt_src_basic(self):
+        """Test deb source string, overwrite mirror and filename"""
+        cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu'
+                          ' karmic-backports'
+                          ' main universe multiverse restricted'),
+               'filename': self.aptlistfile}
+        self.apt_src_basic(self.aptlistfile, [cfg])
+
+    def test_apt_src_basic_dict(self):
+        """Test deb source string, overwrite mirror and filename (dict)"""
+        cfg = {self.aptlistfile: {'source':
+                                  ('deb http://archive.ubuntu.com/ubuntu'
+                                   ' karmic-backports'
+                                   ' main universe multiverse restricted')}}
+        self.apt_src_basic(self.aptlistfile, cfg)
+
+    def apt_src_basic_tri(self, cfg):
+        """apt_src_basic_tri
+        Test Fix three deb source string, has to overwrite mirror conf in
+        params. Test with filenames provided in config.
+        generic part to check three files with different content
+        """
+        self.apt_src_basic(self.aptlistfile, cfg)
+
+        # extra verify on two extra files of this test
+        contents = load_tfile_or_url(self.aptlistfile2)
+        self.assertTrue(re.search(r"%s %s %s %s\n" %
+                                  ("deb", "http://archive.ubuntu.com/ubuntu",
+                                   "precise-backports",
+                                   "main universe multiverse restricted"),
+                                  contents, flags=re.IGNORECASE))
+        contents = load_tfile_or_url(self.aptlistfile3)
+        self.assertTrue(re.search(r"%s %s %s %s\n" %
+                                  ("deb", "http://archive.ubuntu.com/ubuntu",
+                                   "lucid-backports",
+                                   "main universe multiverse restricted"),
+                                  contents, flags=re.IGNORECASE))
+
+    def test_apt_src_basic_tri(self):
+        """Test Fix three deb source string with filenames"""
+        cfg1 = {'source': ('deb http://archive.ubuntu.com/ubuntu'
+                           ' karmic-backports'
+                           ' main universe multiverse restricted'),
+                'filename': self.aptlistfile}
+        cfg2 = {'source': ('deb http://archive.ubuntu.com/ubuntu'
+                           ' precise-backports'
+                           ' main universe multiverse restricted'),
+                'filename': self.aptlistfile2}
+        cfg3 = {'source': ('deb http://archive.ubuntu.com/ubuntu'
+                           ' lucid-backports'
+                           ' main universe multiverse restricted'),
+                'filename': self.aptlistfile3}
+        self.apt_src_basic_tri([cfg1, cfg2, cfg3])
+
+    def test_apt_src_basic_dict_tri(self):
+        """Test Fix three deb source string with filenames (dict)"""
+        cfg = {self.aptlistfile: {'source':
+                                  ('deb http://archive.ubuntu.com/ubuntu'
+                                   ' karmic-backports'
+                                   ' main universe multiverse restricted')},
+               self.aptlistfile2: {'source':
+                                   ('deb http://archive.ubuntu.com/ubuntu'
+                                    ' precise-backports'
+                                    ' main universe multiverse restricted')},
+               self.aptlistfile3: {'source':
+                                   ('deb http://archive.ubuntu.com/ubuntu'
+                                    ' lucid-backports'
+                                    ' main universe multiverse restricted')}}
+        self.apt_src_basic_tri(cfg)
+
+    def test_apt_src_basic_nofn(self):
+        """Test Fix three deb source string without filenames (dict)"""
+        cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu'
+                          ' karmic-backports'
+                          ' main universe multiverse restricted')}
+        with mock.patch.object(os.path, 'join', side_effect=self.myjoin):
+            self.apt_src_basic(self.fallbackfn, [cfg])
+
+    def apt_src_replacement(self, filename, cfg):
+        """apt_src_replace
+        Test Autoreplacement of MIRROR and RELEASE in source specs
+        """
+        cfg = self.wrapv1conf(cfg)
+        params = self._get_default_params()
+        cc_apt_configure.handle("test", cfg, self.fakecloud, None, None)
+
+        self.assertTrue(os.path.isfile(filename))
+
+        contents = load_tfile_or_url(filename)
+        self.assertTrue(re.search(r"%s %s %s %s\n" %
+                                  ("deb", params['MIRROR'], params['RELEASE'],
+                                   "multiverse"),
+                                  contents, flags=re.IGNORECASE))
+
+    def test_apt_src_replace(self):
+        """Test Autoreplacement of MIRROR and RELEASE in source specs"""
+        cfg = {'source': 'deb $MIRROR $RELEASE multiverse',
+               'filename': self.aptlistfile}
+        self.apt_src_replacement(self.aptlistfile, [cfg])
+
+    def apt_src_replace_tri(self, cfg):
+        """apt_src_replace_tri
+        Test three autoreplacements of MIRROR and RELEASE in source specs with
+        generic part
+        """
+        self.apt_src_replacement(self.aptlistfile, cfg)
+
+        # extra verify on two extra files of this test
+        params = self._get_default_params()
+        contents = load_tfile_or_url(self.aptlistfile2)
+        self.assertTrue(re.search(r"%s %s %s %s\n" %
+                                  ("deb", params['MIRROR'], params['RELEASE'],
+                                   "main"),
+                                  contents, flags=re.IGNORECASE))
+        contents = load_tfile_or_url(self.aptlistfile3)
+        self.assertTrue(re.search(r"%s %s %s %s\n" %
+                                  ("deb", params['MIRROR'], params['RELEASE'],
+                                   "universe"),
+                                  contents, flags=re.IGNORECASE))
+
+    def test_apt_src_replace_tri(self):
+        """Test triple Autoreplacement of MIRROR and RELEASE in source specs"""
+        cfg1 = {'source': 'deb $MIRROR $RELEASE multiverse',
+                'filename': self.aptlistfile}
+        cfg2 = {'source': 'deb $MIRROR $RELEASE main',
+                'filename': self.aptlistfile2}
+        cfg3 = {'source': 'deb $MIRROR $RELEASE universe',
+                'filename': self.aptlistfile3}
+        self.apt_src_replace_tri([cfg1, cfg2, cfg3])
+
+    def test_apt_src_replace_dict_tri(self):
+        """Test triple Autoreplacement in source specs (dict)"""
+        cfg = {self.aptlistfile: {'source': 'deb $MIRROR $RELEASE multiverse'},
+               'notused': {'source': 'deb $MIRROR $RELEASE main',
+                           'filename': self.aptlistfile2},
+               self.aptlistfile3: {'source': 'deb $MIRROR $RELEASE universe'}}
+        self.apt_src_replace_tri(cfg)
+
+    def test_apt_src_replace_nofn(self):
+        """Test Autoreplacement of MIRROR and RELEASE in source specs nofile"""
+        cfg = {'source': 'deb $MIRROR $RELEASE multiverse'}
+        with mock.patch.object(os.path, 'join', side_effect=self.myjoin):
+            self.apt_src_replacement(self.fallbackfn, [cfg])
+
+    def apt_src_keyid(self, filename, cfg, keynum):
+        """apt_src_keyid
+        Test specification of a source + keyid
+        """
+        cfg = self.wrapv1conf(cfg)
+
+        with mock.patch.object(util, 'subp',
+                               return_value=('fakekey 1234', '')) as mockobj:
+            cc_apt_configure.handle("test", cfg, self.fakecloud, None, None)
+
+        # check if it added the right ammount of keys
+        calls = []
+        for _ in range(keynum):
+            calls.append(call(['apt-key', 'add', '-'],
+                              data=b'fakekey 1234',
+                              target=None))
+        mockobj.assert_has_calls(calls, any_order=True)
+
+        self.assertTrue(os.path.isfile(filename))
+
+        contents = load_tfile_or_url(filename)
+        self.assertTrue(re.search(r"%s %s %s %s\n" %
+                                  ("deb",
+                                   ('http://ppa.launchpad.net/smoser/'
+                                    'cloud-init-test/ubuntu'),
+                                   "xenial", "main"),
+                                  contents, flags=re.IGNORECASE))
+
+    def test_apt_src_keyid(self):
+        """Test specification of a source + keyid with filename being set"""
+        cfg = {'source': ('deb '
+                          'http://ppa.launchpad.net/'
+                          'smoser/cloud-init-test/ubuntu'
+                          ' xenial main'),
+               'keyid': "03683F77",
+               'filename': self.aptlistfile}
+        self.apt_src_keyid(self.aptlistfile, [cfg], 1)
+
+    def test_apt_src_keyid_tri(self):
+        """Test 3x specification of a source + keyid with filename being set"""
+        cfg1 = {'source': ('deb '
+                           'http://ppa.launchpad.net/'
+                           'smoser/cloud-init-test/ubuntu'
+                           ' xenial main'),
+                'keyid': "03683F77",
+                'filename': self.aptlistfile}
+        cfg2 = {'source': ('deb '
+                           'http://ppa.launchpad.net/'
+                           'smoser/cloud-init-test/ubuntu'
+                           ' xenial universe'),
+                'keyid': "03683F77",
+                'filename': self.aptlistfile2}
+        cfg3 = {'source': ('deb '
+                           'http://ppa.launchpad.net/'
+                           'smoser/cloud-init-test/ubuntu'
+                           ' xenial multiverse'),
+                'keyid': "03683F77",
+                'filename': self.aptlistfile3}
+
+        self.apt_src_keyid(self.aptlistfile, [cfg1, cfg2, cfg3], 3)
+        contents = load_tfile_or_url(self.aptlistfile2)
+        self.assertTrue(re.search(r"%s %s %s %s\n" %
+                                  ("deb",
+                                   ('http://ppa.launchpad.net/smoser/'
+                                    'cloud-init-test/ubuntu'),
+                                   "xenial", "universe"),
+                                  contents, flags=re.IGNORECASE))
+        contents = load_tfile_or_url(self.aptlistfile3)
+        self.assertTrue(re.search(r"%s %s %s %s\n" %
+                                  ("deb",
+                                   ('http://ppa.launchpad.net/smoser/'
+                                    'cloud-init-test/ubuntu'),
+                                   "xenial", "multiverse"),
+                                  contents, flags=re.IGNORECASE))
+
+    def test_apt_src_keyid_nofn(self):
+        """Test specification of a source + keyid without filename being set"""
+        cfg = {'source': ('deb '
+                          'http://ppa.launchpad.net/'
+                          'smoser/cloud-init-test/ubuntu'
+                          ' xenial main'),
+               'keyid': "03683F77"}
+        with mock.patch.object(os.path, 'join', side_effect=self.myjoin):
+            self.apt_src_keyid(self.fallbackfn, [cfg], 1)
+
+    def apt_src_key(self, filename, cfg):
+        """apt_src_key
+        Test specification of a source + key
+        """
+        cfg = self.wrapv1conf([cfg])
+
+        with mock.patch.object(util, 'subp') as mockobj:
+            cc_apt_configure.handle("test", cfg, self.fakecloud, None, None)
+
+        mockobj.assert_called_with(['apt-key', 'add', '-'],
+                                   data=b'fakekey 4321', target=None)
+
+        self.assertTrue(os.path.isfile(filename))
+
+        contents = load_tfile_or_url(filename)
+        self.assertTrue(re.search(r"%s %s %s %s\n" %
+                                  ("deb",
+                                   ('http://ppa.launchpad.net/smoser/'
+                                    'cloud-init-test/ubuntu'),
+                                   "xenial", "main"),
+                                  contents, flags=re.IGNORECASE))
+
+    def test_apt_src_key(self):
+        """Test specification of a source + key with filename being set"""
+        cfg = {'source': ('deb '
+                          'http://ppa.launchpad.net/'
+                          'smoser/cloud-init-test/ubuntu'
+                          ' xenial main'),
+               'key': "fakekey 4321",
+               'filename': self.aptlistfile}
+        self.apt_src_key(self.aptlistfile, cfg)
+
+    def test_apt_src_key_nofn(self):
+        """Test specification of a source + key without filename being set"""
+        cfg = {'source': ('deb '
+                          'http://ppa.launchpad.net/'
+                          'smoser/cloud-init-test/ubuntu'
+                          ' xenial main'),
+               'key': "fakekey 4321"}
+        with mock.patch.object(os.path, 'join', side_effect=self.myjoin):
+            self.apt_src_key(self.fallbackfn, cfg)
+
+    def test_apt_src_keyonly(self):
+        """Test specifying key without source"""
+        cfg = {'key': "fakekey 4242",
+               'filename': self.aptlistfile}
+        cfg = self.wrapv1conf([cfg])
+
+        with mock.patch.object(util, 'subp') as mockobj:
+            cc_apt_configure.handle("test", cfg, self.fakecloud, None, None)
+
+        mockobj.assert_called_once_with(['apt-key', 'add', '-'],
+                                        data=b'fakekey 4242', target=None)
+
+        # filename should be ignored on key only
+        self.assertFalse(os.path.isfile(self.aptlistfile))
+
+    def test_apt_src_keyidonly(self):
+        """Test specification of a keyid without source"""
+        cfg = {'keyid': "03683F77",
+               'filename': self.aptlistfile}
+        cfg = self.wrapv1conf([cfg])
+
+        with mock.patch.object(util, 'subp',
+                               return_value=('fakekey 1212', '')) as mockobj:
+            cc_apt_configure.handle("test", cfg, self.fakecloud, None, None)
+
+        mockobj.assert_called_with(['apt-key', 'add', '-'],
+                                   data=b'fakekey 1212', target=None)
+
+        # filename should be ignored on key only
+        self.assertFalse(os.path.isfile(self.aptlistfile))
+
+    def apt_src_keyid_real(self, cfg, expectedkey):
+        """apt_src_keyid_real
+        Test specification of a keyid without source including
+        up to addition of the key (add_apt_key_raw mocked to keep the
+        environment as is)
+        """
+        key = cfg['keyid']
+        keyserver = cfg.get('keyserver', 'keyserver.ubuntu.com')
+        cfg = self.wrapv1conf([cfg])
+
+        with mock.patch.object(cc_apt_configure, 'add_apt_key_raw') as mockkey:
+            with mock.patch.object(gpg, 'getkeybyid',
+                                   return_value=expectedkey) as mockgetkey:
+                cc_apt_configure.handle("test", cfg, self.fakecloud,
+                                        None, None)
+
+        mockgetkey.assert_called_with(key, keyserver)
+        mockkey.assert_called_with(expectedkey, None)
+
+        # filename should be ignored on key only
+        self.assertFalse(os.path.isfile(self.aptlistfile))
+
+    def test_apt_src_keyid_real(self):
+        """test_apt_src_keyid_real - Test keyid including key add"""
+        keyid = "03683F77"
+        cfg = {'keyid': keyid,
+               'filename': self.aptlistfile}
+
+        self.apt_src_keyid_real(cfg, EXPECTEDKEY)
+
+    def test_apt_src_longkeyid_real(self):
+        """test_apt_src_longkeyid_real - Test long keyid including key add"""
+        keyid = "B59D 5F15 97A5 04B7 E230  6DCA 0620 BBCF 0368 3F77"
+        cfg = {'keyid': keyid,
+               'filename': self.aptlistfile}
+
+        self.apt_src_keyid_real(cfg, EXPECTEDKEY)
+
+    def test_apt_src_longkeyid_ks_real(self):
+        """test_apt_src_longkeyid_ks_real - Test long keyid from other ks"""
+        keyid = "B59D 5F15 97A5 04B7 E230  6DCA 0620 BBCF 0368 3F77"
+        cfg = {'keyid': keyid,
+               'keyserver': 'keys.gnupg.net',
+               'filename': self.aptlistfile}
+
+        self.apt_src_keyid_real(cfg, EXPECTEDKEY)
+
+    def test_apt_src_ppa(self):
+        """Test adding a ppa"""
+        cfg = {'source': 'ppa:smoser/cloud-init-test',
+               'filename': self.aptlistfile}
+        cfg = self.wrapv1conf([cfg])
+
+        with mock.patch.object(util, 'subp') as mockobj:
+            cc_apt_configure.handle("test", cfg, self.fakecloud, None, None)
+        mockobj.assert_called_once_with(['add-apt-repository',
+                                         'ppa:smoser/cloud-init-test'],
+                                        target=None)
+
+        # adding ppa should ignore filename (uses add-apt-repository)
+        self.assertFalse(os.path.isfile(self.aptlistfile))
+
+    def test_apt_src_ppa_tri(self):
+        """Test adding three ppa's"""
+        cfg1 = {'source': 'ppa:smoser/cloud-init-test',
+                'filename': self.aptlistfile}
+        cfg2 = {'source': 'ppa:smoser/cloud-init-test2',
+                'filename': self.aptlistfile2}
+        cfg3 = {'source': 'ppa:smoser/cloud-init-test3',
+                'filename': self.aptlistfile3}
+        cfg = self.wrapv1conf([cfg1, cfg2, cfg3])
+
+        with mock.patch.object(util, 'subp') as mockobj:
+            cc_apt_configure.handle("test", cfg, self.fakecloud,
+                                    None, None)
+        calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test'],
+                      target=None),
+                 call(['add-apt-repository', 'ppa:smoser/cloud-init-test2'],
+                      target=None),
+                 call(['add-apt-repository', 'ppa:smoser/cloud-init-test3'],
+                      target=None)]
+        mockobj.assert_has_calls(calls, any_order=True)
+
+        # adding ppa should ignore all filenames (uses add-apt-repository)
+        self.assertFalse(os.path.isfile(self.aptlistfile))
+        self.assertFalse(os.path.isfile(self.aptlistfile2))
+        self.assertFalse(os.path.isfile(self.aptlistfile3))
+
+    def test_convert_to_new_format(self):
+        """Test the conversion of old to new format"""
+        cfg1 = {'source': 'deb $MIRROR $RELEASE multiverse',
+                'filename': self.aptlistfile}
+        cfg2 = {'source': 'deb $MIRROR $RELEASE main',
+                'filename': self.aptlistfile2}
+        cfg3 = {'source': 'deb $MIRROR $RELEASE universe',
+                'filename': self.aptlistfile3}
+        checkcfg = {self.aptlistfile: {'filename': self.aptlistfile,
+                                       'source': 'deb $MIRROR $RELEASE '
+                                                 'multiverse'},
+                    self.aptlistfile2: {'filename': self.aptlistfile2,
+                                        'source': 'deb $MIRROR $RELEASE main'},
+                    self.aptlistfile3: {'filename': self.aptlistfile3,
+                                        'source': 'deb $MIRROR $RELEASE '
+                                                  'universe'}}
+
+        newcfg = cc_apt_configure.convert_v1_to_v2_apt_format([cfg1, cfg2,
+                                                               cfg3])
+        self.assertEqual(newcfg, checkcfg)
+
+        newcfg2 = cc_apt_configure.convert_v1_to_v2_apt_format(newcfg)
+        self.assertEqual(newcfg2, checkcfg)
+
+        with self.assertRaises(ValueError):
+            cc_apt_configure.convert_v1_to_v2_apt_format(5)
+
+
+# vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py
new file mode 100644
index 0000000..75556b6
--- /dev/null
+++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py
@@ -0,0 +1,1103 @@
+"""test_handler_apt_source_v3
+Testing various config variations of the apt_source custom config
+This tries to call all in the new v3 format and cares about new features
+"""
+import glob
+import os
+import re
+import shutil
+import socket
+import tempfile
+
+from unittest import TestCase
+
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+from mock import call
+
+from cloudinit import cloud
+from cloudinit import distros
+from cloudinit import gpg
+from cloudinit import helpers
+from cloudinit import util
+
+from cloudinit.config import cc_apt_configure
+from cloudinit.sources import DataSourceNone
+
+from .. import helpers as t_help
+
+EXPECTEDKEY = u"""-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mI0ESuZLUgEEAKkqq3idtFP7g9hzOu1a8+v8ImawQN4TrvlygfScMU1TIS1eC7UQ
+NUA8Qqgr9iUaGnejb0VciqftLrU9D6WYHSKz+EITefgdyJ6SoQxjoJdsCpJ7o9Jy
+8PQnpRttiFm4qHu6BVnKnBNxw/z3ST9YMqW5kbMQpfxbGe+obRox59NpABEBAAG0
+HUxhdW5jaHBhZCBQUEEgZm9yIFNjb3R0IE1vc2VyiLYEEwECACAFAkrmS1ICGwMG
+CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRAGILvPA2g/d3aEA/9tVjc10HOZwV29
+OatVuTeERjjrIbxflO586GLA8cp0C9RQCwgod/R+cKYdQcHjbqVcP0HqxveLg0RZ
+FJpWLmWKamwkABErwQLGlM/Hwhjfade8VvEQutH5/0JgKHmzRsoqfR+LMO6OS+Sm
+S0ORP6HXET3+jC8BMG4tBWCTK/XEZw==
+=ACB2
+-----END PGP PUBLIC KEY BLOCK-----"""
+
+ADD_APT_REPO_MATCH = r"^[\w-]+:\w"
+
+TARGET = None
+
+
+def load_tfile(*args, **kwargs):
+    """load_tfile_or_url
+    load file and return content after decoding
+    """
+    return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents)
+
+
+class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
+    """TestAptSourceConfig
+    Main Class to test apt configs
+    """
+    def setUp(self):
+        super(TestAptSourceConfig, self).setUp()
+        self.tmp = tempfile.mkdtemp()
+        self.new_root = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.tmp)
+        self.aptlistfile = os.path.join(self.tmp, "single-deb.list")
+        self.aptlistfile2 = os.path.join(self.tmp, "single-deb2.list")
+        self.aptlistfile3 = os.path.join(self.tmp, "single-deb3.list")
+        self.join = os.path.join
+        self.matcher = re.compile(ADD_APT_REPO_MATCH).search
+
+    @staticmethod
+    def _add_apt_sources(*args, **kwargs):
+        with mock.patch.object(cc_apt_configure, 'update_packages'):
+            cc_apt_configure.add_apt_sources(*args, **kwargs)
+
+    @staticmethod
+    def _get_default_params():
+        """get_default_params
+        Get the most basic default mrror and release info to be used in tests
+        """
+        params = {}
+        params['RELEASE'] = util.lsb_release()['codename']
+        arch = 'amd64'
+        params['MIRROR'] = cc_apt_configure.\
+            get_default_mirrors(arch)["PRIMARY"]
+        return params
+
+    def _myjoin(self, *args, **kwargs):
+        """_myjoin - redir into writable tmpdir"""
+        if (args[0] == "/etc/apt/sources.list.d/" and
+                args[1] == "cloud_config_sources.list" and
+                len(args) == 2):
+            return self.join(self.tmp, args[0].lstrip("/"), args[1])
+        else:
+            return self.join(*args, **kwargs)
+
+    def _get_cloud(self, distro, metadata=None):
+        self.patchUtils(self.new_root)
+        paths = helpers.Paths({})
+        cls = distros.fetch(distro)
+        mydist = cls(distro, {}, paths)
+        myds = DataSourceNone.DataSourceNone({}, mydist, paths)
+        if metadata:
+            myds.metadata.update(metadata)
+        return cloud.Cloud(myds, paths, {}, mydist, None)
+
+    def _apt_src_basic(self, filename, cfg):
+        """_apt_src_basic
+        Test Fix deb source string, has to overwrite mirror conf in params
+        """
+        params = self._get_default_params()
+
+        self._add_apt_sources(cfg, TARGET, template_params=params,
+                              aa_repo_match=self.matcher)
+
+        self.assertTrue(os.path.isfile(filename))
+
+        contents = load_tfile(filename)
+        self.assertTrue(re.search(r"%s %s %s %s\n" %
+                                  ("deb", "http://test.ubuntu.com/ubuntu",
+                                   "karmic-backports",
+                                   "main universe multiverse restricted"),
+                                  contents, flags=re.IGNORECASE))
+
+    def test_apt_v3_src_basic(self):
+        """test_apt_v3_src_basic - Test fix deb source string"""
+        cfg = {self.aptlistfile: {'source':
+                                  ('deb http://test.ubuntu.com/ubuntu'
+                                   ' karmic-backports'
+                                   ' main universe multiverse restricted')}}
+        self._apt_src_basic(self.aptlistfile, cfg)
+
+    def test_apt_v3_src_basic_tri(self):
+        """test_apt_v3_src_basic_tri - Test multiple fix deb source strings"""
+        cfg = {self.aptlistfile: {'source':
+                                  ('deb http://test.ubuntu.com/ubuntu'
+                                   ' karmic-backports'
+                                   ' main universe multiverse restricted')},
+               self.aptlistfile2: {'source':
+                                   ('deb http://test.ubuntu.com/ubuntu'
+                                    ' precise-backports'
+                                    ' main universe multiverse restricted')},
+               self.aptlistfile3: {'source':
+                                   ('deb http://test.ubuntu.com/ubuntu'
+                                    ' lucid-backports'
+                                    ' main universe multiverse restricted')}}
+        self._apt_src_basic(self.aptlistfile, cfg)
+
+        # extra verify on two extra files of this test
+        contents = load_tfile(self.aptlistfile2)
+        self.assertTrue(re.search(r"%s %s %s %s\n" %
+                                  ("deb", "http://test.ubuntu.com/ubuntu",
+                                   "precise-backports",
+                                   "main universe multiverse restricted"),
+                                  contents, flags=re.IGNORECASE))
+        contents = load_tfile(self.aptlistfile3)
+        self.assertTrue(re.search(r"%s %s %s %s\n" %
+                                  ("deb", "http://test.ubuntu.com/ubuntu",
+                                   "lucid-backports",
+                                   "main universe multiverse restricted"),
+                                  contents, flags=re.IGNORECASE))
+
+    def _apt_src_replacement(self, filename, cfg):
+        """apt_src_replace
+        Test Autoreplacement of MIRROR and RELEASE in source specs
+        """
+        params = self._get_default_params()
+        self._add_apt_sources(cfg, TARGET, template_params=params,
+                              aa_repo_match=self.matcher)
+
+        self.assertTrue(os.path.isfile(filename))
+
+        contents = load_tfile(filename)
+        self.assertTrue(re.search(r"%s %s %s %s\n" %
+                                  ("deb", params['MIRROR'], params['RELEASE'],
+                                   "multiverse"),
+                                  contents, flags=re.IGNORECASE))
+
+    def test_apt_v3_src_replace(self):
+        """test_apt_v3_src_replace - Test replacement of MIRROR & RELEASE"""
+        cfg = {self.aptlistfile: {'source': 'deb $MIRROR $RELEASE multiverse'}}
+        self._apt_src_replacement(self.aptlistfile, cfg)
+
+    def test_apt_v3_src_replace_fn(self):
+        """test_apt_v3_src_replace_fn - Test filename overwritten in dict"""
+        cfg = {'ignored': {'source': 'deb $MIRROR $RELEASE multiverse',
+                           'filename': self.aptlistfile}}
+        # second file should overwrite the dict key
+        self._apt_src_replacement(self.aptlistfile, cfg)
+
+    def _apt_src_replace_tri(self, cfg):
+        """_apt_src_replace_tri
+        Test three autoreplacements of MIRROR and RELEASE in source specs with
+        generic part
+        """
+        self._apt_src_replacement(self.aptlistfile, cfg)
+
+        # extra verify on two extra files of this test
+        params = self._get_default_params()
+        contents = load_tfile(self.aptlistfile2)
+        self.assertTrue(re.search(r"%s %s %s %s\n" %
+                                  ("deb", params['MIRROR'], params['RELEASE'],
+                                   "main"),
+                                  contents, flags=re.IGNORECASE))
+        contents = load_tfile(self.aptlistfile3)
+        self.assertTrue(re.search(r"%s %s %s %s\n" %
+                                  ("deb", params['MIRROR'], params['RELEASE'],
+                                   "universe"),
+                                  contents, flags=re.IGNORECASE))
+
+    def test_apt_v3_src_replace_tri(self):
+        """test_apt_v3_src_replace_tri - Test multiple replace/overwrites"""
+        cfg = {self.aptlistfile: {'source': 'deb $MIRROR $RELEASE multiverse'},
+               'notused': {'source': 'deb $MIRROR $RELEASE main',
+                           'filename': self.aptlistfile2},
+               self.aptlistfile3: {'source': 'deb $MIRROR $RELEASE universe'}}
+        self._apt_src_replace_tri(cfg)
+
+    def _apt_src_keyid(self, filename, cfg, keynum):
+        """_apt_src_keyid
+        Test specification of a source + keyid
+        """
+        params = self._get_default_params()
+
+        with mock.patch("cloudinit.util.subp",
+                        return_value=('fakekey 1234', '')) as mockobj:
+            self._add_apt_sources(cfg, TARGET, template_params=params,
+                                  aa_repo_match=self.matcher)
+
+        # check if it added the right ammount of keys
+        calls = []
+        for _ in range(keynum):
+            calls.append(call(['apt-key', 'add', '-'], data=b'fakekey 1234',
+                              target=TARGET))
+        mockobj.assert_has_calls(calls, any_order=True)
+
+        self.assertTrue(os.path.isfile(filename))
+
+        contents = load_tfile(filename)
+        self.assertTrue(re.search(r"%s %s %s %s\n" %
+                                  ("deb",
+                                   ('http://ppa.launchpad.net/smoser/'
+                                    'cloud-init-test/ubuntu'),
+                                   "xenial", "main"),
+                                  contents, flags=re.IGNORECASE))
+
+    def test_apt_v3_src_keyid(self):
+        """test_apt_v3_src_keyid - Test source + keyid with filename"""
+        cfg = {self.aptlistfile: {'source': ('deb '
+                                             'http://ppa.launchpad.net/'
+                                             'smoser/cloud-init-test/ubuntu'
+                                             ' xenial main'),
+                                  'keyid': "03683F77"}}
+        self._apt_src_keyid(self.aptlistfile, cfg, 1)
+
+    def test_apt_v3_src_keyid_tri(self):
+        """test_apt_v3_src_keyid_tri - Test multiple src+key+filen writes"""
+        cfg = {self.aptlistfile: {'source': ('deb '
+                                             'http://ppa.launchpad.net/'
+                                             'smoser/cloud-init-test/ubuntu'
+                                             ' xenial main'),
+                                  'keyid': "03683F77"},
+               'ignored': {'source': ('deb '
+                                      'http://ppa.launchpad.net/'
+                                      'smoser/cloud-init-test/ubuntu'
+                                      ' xenial universe'),
+                           'keyid': "03683F77",
+                           'filename': self.aptlistfile2},
+               self.aptlistfile3: {'source': ('deb '
+                                              'http://ppa.launchpad.net/'
+                                              'smoser/cloud-init-test/ubuntu'
+                                              ' xenial multiverse'),
+                                   'keyid': "03683F77"}}
+
+        self._apt_src_keyid(self.aptlistfile, cfg, 3)
+        contents = load_tfile(self.aptlistfile2)
+        self.assertTrue(re.search(r"%s %s %s %s\n" %
+                                  ("deb",
+                                   ('http://ppa.launchpad.net/smoser/'
+                                    'cloud-init-test/ubuntu'),
+                                   "xenial", "universe"),
+                                  contents, flags=re.IGNORECASE))
+        contents = load_tfile(self.aptlistfile3)
+        self.assertTrue(re.search(r"%s %s %s %s\n" %
+                                  ("deb",
+                                   ('http://ppa.launchpad.net/smoser/'
+                                    'cloud-init-test/ubuntu'),
+                                   "xenial", "multiverse"),
+                                  contents, flags=re.IGNORECASE))
+
+    def test_apt_v3_src_key(self):
+        """test_apt_v3_src_key - Test source + key"""
+        params = self._get_default_params()
+        cfg = {self.aptlistfile: {'source': ('deb '
+                                             'http://ppa.launchpad.net/'
+                                             'smoser/cloud-init-test/ubuntu'
+                                             ' xenial main'),
+                                  'key': "fakekey 4321"}}
+
+        with mock.patch.object(util, 'subp') as mockobj:
+            self._add_apt_sources(cfg, TARGET, template_params=params,
+                                  aa_repo_match=self.matcher)
+
+        mockobj.assert_any_call(['apt-key', 'add', '-'], data=b'fakekey 4321',
+                                target=TARGET)
+
+        self.assertTrue(os.path.isfile(self.aptlistfile))
+
+        contents = load_tfile(self.aptlistfile)
+        self.assertTrue(re.search(r"%s %s %s %s\n" %
+                                  ("deb",
+                                   ('http://ppa.launchpad.net/smoser/'
+                                    'cloud-init-test/ubuntu'),
+                                   "xenial", "main"),
+                                  contents, flags=re.IGNORECASE))
+
+    def test_apt_v3_src_keyonly(self):
+        """test_apt_v3_src_keyonly - Test key without source"""
+        params = self._get_default_params()
+        cfg = {self.aptlistfile: {'key': "fakekey 4242"}}
+
+        with mock.patch.object(util, 'subp') as mockobj:
+            self._add_apt_sources(cfg, TARGET, template_params=params,
+                                  aa_repo_match=self.matcher)
+
+        mockobj.assert_any_call(['apt-key', 'add', '-'], data=b'fakekey 4242',
+                                target=TARGET)
+
+        # filename should be ignored on key only
+        self.assertFalse(os.path.isfile(self.aptlistfile))
+
+    def test_apt_v3_src_keyidonly(self):
+        """test_apt_v3_src_keyidonly - Test keyid without source"""
+        params = self._get_default_params()
+        cfg = {self.aptlistfile: {'keyid': "03683F77"}}
+
+        with mock.patch.object(util, 'subp',
+                               return_value=('fakekey 1212', '')) as mockobj:
+            self._add_apt_sources(cfg, TARGET, template_params=params,
+                                  aa_repo_match=self.matcher)
+
+        mockobj.assert_any_call(['apt-key', 'add', '-'], data=b'fakekey 1212',
+                                target=TARGET)
+
+        # filename should be ignored on key only
+        self.assertFalse(os.path.isfile(self.aptlistfile))
+
+    def apt_src_keyid_real(self, cfg, expectedkey):
+        """apt_src_keyid_real
+        Test specification of a keyid without source including
+        up to addition of the key (add_apt_key_raw mocked to keep the
+        environment as is)
+        """
+        params = self._get_default_params()
+
+        with mock.patch.object(cc_apt_configure, 'add_apt_key_raw') as mockkey:
+            with mock.patch.object(gpg, 'getkeybyid',
+                                   return_value=expectedkey) as mockgetkey:
+                self._add_apt_sources(cfg, TARGET, template_params=params,
+                                      aa_repo_match=self.matcher)
+
+        keycfg = cfg[self.aptlistfile]
+        mockgetkey.assert_called_with(keycfg['keyid'],
+                                      keycfg.get('keyserver',
+                                                 'keyserver.ubuntu.com'))
+        mockkey.assert_called_with(expectedkey, TARGET)
+
+        # filename should be ignored on key only
+        self.assertFalse(os.path.isfile(self.aptlistfile))
+
+    def test_apt_v3_src_keyid_real(self):
+        """test_apt_v3_src_keyid_real - Test keyid including key add"""
+        keyid = "03683F77"
+        cfg = {self.aptlistfile: {'keyid': keyid}}
+
+        self.apt_src_keyid_real(cfg, EXPECTEDKEY)
+
+    def test_apt_v3_src_longkeyid_real(self):
+        """test_apt_v3_src_longkeyid_real Test long keyid including key add"""
+        keyid = "B59D 5F15 97A5 04B7 E230  6DCA 0620 BBCF 0368 3F77"
+        cfg = {self.aptlistfile: {'keyid': keyid}}
+
+        self.apt_src_keyid_real(cfg, EXPECTEDKEY)
+
+    def test_apt_v3_src_longkeyid_ks_real(self):
+        """test_apt_v3_src_longkeyid_ks_real Test long keyid from other ks"""
+        keyid = "B59D 5F15 97A5 04B7 E230  6DCA 0620 BBCF 0368 3F77"
+        cfg = {self.aptlistfile: {'keyid': keyid,
+                                  'keyserver': 'keys.gnupg.net'}}
+
+        self.apt_src_keyid_real(cfg, EXPECTEDKEY)
+
+    def test_apt_v3_src_keyid_keyserver(self):
+        """test_apt_v3_src_keyid_keyserver - Test custom keyserver"""
+        keyid = "03683F77"
+        params = self._get_default_params()
+        cfg = {self.aptlistfile: {'keyid': keyid,
+                                  'keyserver': 'test.random.com'}}
+
+        # in some test environments only *.ubuntu.com is reachable
+        # so mock the call and check if the config got there
+        with mock.patch.object(gpg, 'getkeybyid',
+                               return_value="fakekey") as mockgetkey:
+            with mock.patch.object(cc_apt_configure,
+                                   'add_apt_key_raw') as mockadd:
+                self._add_apt_sources(cfg, TARGET, template_params=params,
+                                      aa_repo_match=self.matcher)
+
+        mockgetkey.assert_called_with('03683F77', 'test.random.com')
+        mockadd.assert_called_with('fakekey', TARGET)
+
+        # filename should be ignored on key only
+        self.assertFalse(os.path.isfile(self.aptlistfile))
+
+    def test_apt_v3_src_ppa(self):
+        """test_apt_v3_src_ppa - Test specification of a ppa"""
+        params = self._get_default_params()
+        cfg = {self.aptlistfile: {'source': 'ppa:smoser/cloud-init-test'}}
+
+        with mock.patch("cloudinit.util.subp") as mockobj:
+            self._add_apt_sources(cfg, TARGET, template_params=params,
+                                  aa_repo_match=self.matcher)
+        mockobj.assert_any_call(['add-apt-repository',
+                                 'ppa:smoser/cloud-init-test'], target=TARGET)
+
+        # adding ppa should ignore filename (uses add-apt-repository)
+        self.assertFalse(os.path.isfile(self.aptlistfile))
+
+    def test_apt_v3_src_ppa_tri(self):
+        """test_apt_v3_src_ppa_tri - Test specification of multiple ppa's"""
+        params = self._get_default_params()
+        cfg = {self.aptlistfile: {'source': 'ppa:smoser/cloud-init-test'},
+               self.aptlistfile2: {'source': 'ppa:smoser/cloud-init-test2'},
+               self.aptlistfile3: {'source': 'ppa:smoser/cloud-init-test3'}}
+
+        with mock.patch("cloudinit.util.subp") as mockobj:
+            self._add_apt_sources(cfg, TARGET, template_params=params,
+                                  aa_repo_match=self.matcher)
+        calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test'],
+                      target=TARGET),
+                 call(['add-apt-repository', 'ppa:smoser/cloud-init-test2'],
+                      target=TARGET),
+                 call(['add-apt-repository', 'ppa:smoser/cloud-init-test3'],
+                      target=TARGET)]
+        mockobj.assert_has_calls(calls, any_order=True)
+
+        # adding ppa should ignore all filenames (uses add-apt-repository)
+        self.assertFalse(os.path.isfile(self.aptlistfile))
+        self.assertFalse(os.path.isfile(self.aptlistfile2))
+        self.assertFalse(os.path.isfile(self.aptlistfile3))
+
+    @mock.patch("cloudinit.config.cc_apt_configure.util.get_architecture")
+    def test_apt_v3_list_rename(self, m_get_architecture):
+        """test_apt_v3_list_rename - Test find mirror and apt list renaming"""
+        pre = "/var/lib/apt/lists"
+        # filenames are archive dependent
+
+        arch = 's390x'
+        m_get_architecture.return_value = arch
+        component = "ubuntu-ports"
+        archive = "ports.ubuntu.com"
+
+        cfg = {'primary': [{'arches': ["default"],
+                            'uri':
+                            'http://test.ubuntu.com/%s/' % component}],
+               'security': [{'arches': ["default"],
+                             'uri':
+                             'http://testsec.ubuntu.com/%s/' % component}]}
+        post = ("%s_dists_%s-updates_InRelease" %
+                (component, util.lsb_release()['codename']))
+        fromfn = ("%s/%s_%s" % (pre, archive, post))
+        tofn = ("%s/test.ubuntu.com_%s" % (pre, post))
+
+        mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, arch)
+
+        self.assertEqual(mirrors['MIRROR'],
+                         "http://test.ubuntu.com/%s/" % component)
+        self.assertEqual(mirrors['PRIMARY'],
+                         "http://test.ubuntu.com/%s/" % component)
+        self.assertEqual(mirrors['SECURITY'],
+                         "http://testsec.ubuntu.com/%s/" % component)
+
+        with mock.patch.object(os, 'rename') as mockren:
+            with mock.patch.object(glob, 'glob',
+                                   return_value=[fromfn]):
+                cc_apt_configure.rename_apt_lists(mirrors, TARGET)
+
+        mockren.assert_any_call(fromfn, tofn)
+
+    @mock.patch("cloudinit.config.cc_apt_configure.util.get_architecture")
+    def test_apt_v3_list_rename_non_slash(self, m_get_architecture):
+        target = os.path.join(self.tmp, "rename_non_slash")
+        apt_lists_d = os.path.join(target, "./" + cc_apt_configure.APT_LISTS)
+
+        m_get_architecture.return_value = 'amd64'
+
+        mirror_path = "some/random/path/"
+        primary = "http://test.ubuntu.com/" + mirror_path
+        security = "http://test-security.ubuntu.com/" + mirror_path
+        mirrors = {'PRIMARY': primary, 'SECURITY': security}
+
+        # these match default archive prefixes
+        opri_pre = "archive.ubuntu.com_ubuntu_dists_xenial"
+        osec_pre = "security.ubuntu.com_ubuntu_dists_xenial"
+        # this one won't match and should not be renamed defaults.
+        other_pre = "dl.google.com_linux_chrome_deb_dists_stable"
+        # these are our new expected prefixes
+        npri_pre = "test.ubuntu.com_some_random_path_dists_xenial"
+        nsec_pre = "test-security.ubuntu.com_some_random_path_dists_xenial"
+
+        files = [
+            # orig prefix, new prefix, suffix
+            (opri_pre, npri_pre, "_main_binary-amd64_Packages"),
+            (opri_pre, npri_pre, "_main_binary-amd64_InRelease"),
+            (opri_pre, npri_pre, "-updates_main_binary-amd64_Packages"),
+            (opri_pre, npri_pre, "-updates_main_binary-amd64_InRelease"),
+            (other_pre, other_pre, "_main_binary-amd64_Packages"),
+            (other_pre, other_pre, "_Release"),
+            (other_pre, other_pre, "_Release.gpg"),
+            (osec_pre, nsec_pre, "_InRelease"),
+            (osec_pre, nsec_pre, "_main_binary-amd64_Packages"),
+            (osec_pre, nsec_pre, "_universe_binary-amd64_Packages"),
+        ]
+
+        expected = sorted([npre + suff for opre, npre, suff in files])
+        # create files
+        for (opre, npre, suff) in files:
+            fpath = os.path.join(apt_lists_d, opre + suff)
+            util.write_file(fpath, content=fpath)
+
+        cc_apt_configure.rename_apt_lists(mirrors, target)
+        found = sorted(os.listdir(apt_lists_d))
+        self.assertEqual(expected, found)
+
+    @staticmethod
+    def test_apt_v3_proxy():
+        """test_apt_v3_proxy - Test apt_*proxy configuration"""
+        cfg = {"proxy": "foobar1",
+               "http_proxy": "foobar2",
+               "ftp_proxy": "foobar3",
+               "https_proxy": "foobar4"}
+
+        with mock.patch.object(util, 'write_file') as mockobj:
+            cc_apt_configure.apply_apt_config(cfg, "proxyfn", "notused")
+
+        mockobj.assert_called_with('proxyfn',
+                                   ('Acquire::http::Proxy "foobar1";\n'
+                                    'Acquire::http::Proxy "foobar2";\n'
+                                    'Acquire::ftp::Proxy "foobar3";\n'
+                                    'Acquire::https::Proxy "foobar4";\n'))
+
+    def test_apt_v3_mirror(self):
+        """test_apt_v3_mirror - Test defining a mirror"""
+        pmir = "http://us.archive.ubuntu.com/ubuntu/"
+        smir = "http://security.ubuntu.com/ubuntu/"
+        cfg = {"primary": [{'arches': ["default"],
+                            "uri": pmir}],
+               "security": [{'arches': ["default"],
+                             "uri": smir}]}
+
+        mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, 'amd64')
+
+        self.assertEqual(mirrors['MIRROR'],
+                         pmir)
+        self.assertEqual(mirrors['PRIMARY'],
+                         pmir)
+        self.assertEqual(mirrors['SECURITY'],
+                         smir)
+
+    def test_apt_v3_mirror_default(self):
+        """test_apt_v3_mirror_default - Test without defining a mirror"""
+        arch = 'amd64'
+        default_mirrors = cc_apt_configure.get_default_mirrors(arch)
+        pmir = default_mirrors["PRIMARY"]
+        smir = default_mirrors["SECURITY"]
+        mycloud = self._get_cloud('ubuntu')
+        mirrors = cc_apt_configure.find_apt_mirror_info({}, mycloud, arch)
+
+        self.assertEqual(mirrors['MIRROR'],
+                         pmir)
+        self.assertEqual(mirrors['PRIMARY'],
+                         pmir)
+        self.assertEqual(mirrors['SECURITY'],
+                         smir)
+
+    def test_apt_v3_mirror_arches(self):
+        """test_apt_v3_mirror_arches - Test arches selection of mirror"""
+        pmir = "http://my-primary.ubuntu.com/ubuntu/"
+        smir = "http://my-security.ubuntu.com/ubuntu/"
+        arch = 'ppc64el'
+        cfg = {"primary": [{'arches': ["default"], "uri": "notthis-primary"},
+                           {'arches': [arch], "uri": pmir}],
+               "security": [{'arches': ["default"], "uri": "nothis-security"},
+                            {'arches': [arch], "uri": smir}]}
+
+        mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, arch)
+
+        self.assertEqual(mirrors['PRIMARY'], pmir)
+        self.assertEqual(mirrors['MIRROR'], pmir)
+        self.assertEqual(mirrors['SECURITY'], smir)
+
+    def test_apt_v3_mirror_arches_default(self):
+        """test_apt_v3_mirror_arches - Test falling back to default arch"""
+        pmir = "http://us.archive.ubuntu.com/ubuntu/"
+        smir = "http://security.ubuntu.com/ubuntu/"
+        cfg = {"primary": [{'arches': ["default"],
+                            "uri": pmir},
+                           {'arches': ["thisarchdoesntexist"],
+                            "uri": "notthis"}],
+               "security": [{'arches': ["thisarchdoesntexist"],
+                             "uri": "nothat"},
+                            {'arches': ["default"],
+                             "uri": smir}]}
+
+        mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, 'amd64')
+
+        self.assertEqual(mirrors['MIRROR'],
+                         pmir)
+        self.assertEqual(mirrors['PRIMARY'],
+                         pmir)
+        self.assertEqual(mirrors['SECURITY'],
+                         smir)
+
+    @mock.patch("cloudinit.config.cc_apt_configure.util.get_architecture")
+    def test_apt_v3_get_def_mir_non_intel_no_arch(self, m_get_architecture):
+        arch = 'ppc64el'
+        m_get_architecture.return_value = arch
+        expected = {'PRIMARY': 'http://ports.ubuntu.com/ubuntu-ports',
+                    'SECURITY': 'http://ports.ubuntu.com/ubuntu-ports'}
+        self.assertEqual(expected, cc_apt_configure.get_default_mirrors())
+
+    def test_apt_v3_get_default_mirrors_non_intel_with_arch(self):
+        found = cc_apt_configure.get_default_mirrors('ppc64el')
+
+        expected = {'PRIMARY': 'http://ports.ubuntu.com/ubuntu-ports',
+                    'SECURITY': 'http://ports.ubuntu.com/ubuntu-ports'}
+        self.assertEqual(expected, found)
+
+    def test_apt_v3_mirror_arches_sysdefault(self):
+        """test_apt_v3_mirror_arches - Test arches fallback to sys default"""
+        arch = 'amd64'
+        default_mirrors = cc_apt_configure.get_default_mirrors(arch)
+        pmir = default_mirrors["PRIMARY"]
+        smir = default_mirrors["SECURITY"]
+        mycloud = self._get_cloud('ubuntu')
+        cfg = {"primary": [{'arches': ["thisarchdoesntexist_64"],
+                            "uri": "notthis"},
+                           {'arches': ["thisarchdoesntexist"],
+                            "uri": "notthiseither"}],
+               "security": [{'arches': ["thisarchdoesntexist"],
+                             "uri": "nothat"},
+                            {'arches': ["thisarchdoesntexist_64"],
+                             "uri": "nothateither"}]}
+
+        mirrors = cc_apt_configure.find_apt_mirror_info(cfg, mycloud, arch)
+
+        self.assertEqual(mirrors['MIRROR'], pmir)
+        self.assertEqual(mirrors['PRIMARY'], pmir)
+        self.assertEqual(mirrors['SECURITY'], smir)
+
+    def test_apt_v3_mirror_search(self):
+        """test_apt_v3_mirror_search - Test searching mirrors in a list
+            mock checks to avoid relying on network connectivity"""
+        pmir = "http://us.archive.ubuntu.com/ubuntu/"
+        smir = "http://security.ubuntu.com/ubuntu/"
+        cfg = {"primary": [{'arches': ["default"],
+                            "search": ["pfailme", pmir]}],
+               "security": [{'arches': ["default"],
+                             "search": ["sfailme", smir]}]}
+
+        with mock.patch.object(cc_apt_configure, 'search_for_mirror',
+                               side_effect=[pmir, smir]) as mocksearch:
+            mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None,
+                                                            'amd64')
+
+        calls = [call(["pfailme", pmir]),
+                 call(["sfailme", smir])]
+        mocksearch.assert_has_calls(calls)
+
+        self.assertEqual(mirrors['MIRROR'],
+                         pmir)
+        self.assertEqual(mirrors['PRIMARY'],
+                         pmir)
+        self.assertEqual(mirrors['SECURITY'],
+                         smir)
+
+    def test_apt_v3_mirror_search_many2(self):
+        """test_apt_v3_mirror_search_many3 - Test both mirrors specs at once"""
+        pmir = "http://us.archive.ubuntu.com/ubuntu/"
+        smir = "http://security.ubuntu.com/ubuntu/"
+        cfg = {"primary": [{'arches': ["default"],
+                            "uri": pmir,
+                            "search": ["pfailme", "foo"]}],
+               "security": [{'arches': ["default"],
+                             "uri": smir,
+                             "search": ["sfailme", "bar"]}]}
+
+        arch = 'amd64'
+
+        # should be called only once per type, despite two mirror configs
+        mycloud = None
+        with mock.patch.object(cc_apt_configure, 'get_mirror',
+                               return_value="http://mocked/foo") as mockgm:
+            mirrors = cc_apt_configure.find_apt_mirror_info(cfg, mycloud, arch)
+        calls = [call(cfg, 'primary', arch, mycloud),
+                 call(cfg, 'security', arch, mycloud)]
+        mockgm.assert_has_calls(calls)
+
+        # should not be called, since primary is specified
+        with mock.patch.object(cc_apt_configure,
+                               'search_for_mirror') as mockse:
+            mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, arch)
+        mockse.assert_not_called()
+
+        self.assertEqual(mirrors['MIRROR'],
+                         pmir)
+        self.assertEqual(mirrors['PRIMARY'],
+                         pmir)
+        self.assertEqual(mirrors['SECURITY'],
+                         smir)
+
+    def test_apt_v3_url_resolvable(self):
+        """test_apt_v3_url_resolvable - Test resolving urls"""
+
+        with mock.patch.object(util, 'is_resolvable') as mockresolve:
+            util.is_resolvable_url("http://1.2.3.4/ubuntu")
+        mockresolve.assert_called_with("1.2.3.4")
+
+        with mock.patch.object(util, 'is_resolvable') as mockresolve:
+            util.is_resolvable_url("http://us.archive.ubuntu.com/ubuntu")
+        mockresolve.assert_called_with("us.archive.ubuntu.com")
+
+        # former tests can leave this set (or not if the test is ran directly)
+        # do a hard reset to ensure a stable result
+        util._DNS_REDIRECT_IP = None
+        bad = [(None, None, None, "badname", ["10.3.2.1"])]
+        good = [(None, None, None, "goodname", ["10.2.3.4"])]
+        with mock.patch.object(socket, 'getaddrinfo',
+                               side_effect=[bad, bad, bad, good,
+                                            good]) as mocksock:
+            ret = util.is_resolvable_url("http://us.archive.ubuntu.com/ubuntu")
+            ret2 = util.is_resolvable_url("http://1.2.3.4/ubuntu")
+        mocksock.assert_any_call('does-not-exist.example.com.', None,
+                                 0, 0, 1, 2)
+        mocksock.assert_any_call('example.invalid.', None, 0, 0, 1, 2)
+        mocksock.assert_any_call('us.archive.ubuntu.com', None)
+        mocksock.assert_any_call('1.2.3.4', None)
+
+        self.assertTrue(ret)
+        self.assertTrue(ret2)
+
+        # side effect need only bad ret after initial call
+        with mock.patch.object(socket, 'getaddrinfo',
+                               side_effect=[bad]) as mocksock:
+            ret3 = util.is_resolvable_url("http://failme.com/ubuntu")
+        calls = [call('failme.com', None)]
+        mocksock.assert_has_calls(calls)
+        self.assertFalse(ret3)
+
+    def test_apt_v3_disable_suites(self):
+        """test_disable_suites - disable_suites with many configurations"""
+        release = "xenial"
+        orig = """deb http://ubuntu.com//ubuntu xenial main
+deb http://ubuntu.com//ubuntu xenial-updates main
+deb http://ubuntu.com//ubuntu xenial-security main
+deb-src http://ubuntu.com//ubuntu universe multiverse
+deb http://ubuntu.com/ubuntu/ xenial-proposed main"""
+
+        # disable nothing
+        disabled = []
+        expect = """deb http://ubuntu.com//ubuntu xenial main
+deb http://ubuntu.com//ubuntu xenial-updates main
+deb http://ubuntu.com//ubuntu xenial-security main
+deb-src http://ubuntu.com//ubuntu universe multiverse
+deb http://ubuntu.com/ubuntu/ xenial-proposed main"""
+        result = cc_apt_configure.disable_suites(disabled, orig, release)
+        self.assertEqual(expect, result)
+
+        # single disable release suite
+        disabled = ["$RELEASE"]
+        expect = """\
+# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu xenial main
+deb http://ubuntu.com//ubuntu xenial-updates main
+deb http://ubuntu.com//ubuntu xenial-security main
+deb-src http://ubuntu.com//ubuntu universe multiverse
+deb http://ubuntu.com/ubuntu/ xenial-proposed main"""
+        result = cc_apt_configure.disable_suites(disabled, orig, release)
+        self.assertEqual(expect, result)
+
+        # single disable other suite
+        disabled = ["$RELEASE-updates"]
+        expect = ("""deb http://ubuntu.com//ubuntu xenial main
+# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu"""
+                  """ xenial-updates main
+deb http://ubuntu.com//ubuntu xenial-security main
+deb-src http://ubuntu.com//ubuntu universe multiverse
+deb http://ubuntu.com/ubuntu/ xenial-proposed main""")
+        result = cc_apt_configure.disable_suites(disabled, orig, release)
+        self.assertEqual(expect, result)
+
+        # multi disable
+        disabled = ["$RELEASE-updates", "$RELEASE-security"]
+        expect = ("""deb http://ubuntu.com//ubuntu xenial main
+# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu """
+                  """xenial-updates main
+# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu """
+                  """xenial-security main
+deb-src http://ubuntu.com//ubuntu universe multiverse
+deb http://ubuntu.com/ubuntu/ xenial-proposed main""")
+        result = cc_apt_configure.disable_suites(disabled, orig, release)
+        self.assertEqual(expect, result)
+
+        # multi line disable (same suite multiple times in input)
+        disabled = ["$RELEASE-updates", "$RELEASE-security"]
+        orig = """deb http://ubuntu.com//ubuntu xenial main
+deb http://ubuntu.com//ubuntu xenial-updates main
+deb http://ubuntu.com//ubuntu xenial-security main
+deb-src http://ubuntu.com//ubuntu universe multiverse
+deb http://UBUNTU.com//ubuntu xenial-updates main
+deb http://UBUNTU.COM//ubuntu xenial-updates main
+deb http://ubuntu.com/ubuntu/ xenial-proposed main"""
+        expect = ("""deb http://ubuntu.com//ubuntu xenial main
+# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu """
+                  """xenial-updates main
+# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu """
+                  """xenial-security main
+deb-src http://ubuntu.com//ubuntu universe multiverse
+# suite disabled by cloud-init: deb http://UBUNTU.com//ubuntu """
+                  """xenial-updates main
+# suite disabled by cloud-init: deb http://UBUNTU.COM//ubuntu """
+                  """xenial-updates main
+deb http://ubuntu.com/ubuntu/ xenial-proposed main""")
+        result = cc_apt_configure.disable_suites(disabled, orig, release)
+        self.assertEqual(expect, result)
+
+        # comment in input
+        disabled = ["$RELEASE-updates", "$RELEASE-security"]
+        orig = """deb http://ubuntu.com//ubuntu xenial main
+deb http://ubuntu.com//ubuntu xenial-updates main
+deb http://ubuntu.com//ubuntu xenial-security main
+deb-src http://ubuntu.com//ubuntu universe multiverse
+#foo
+#deb http://UBUNTU.com//ubuntu xenial-updates main
+deb http://UBUNTU.COM//ubuntu xenial-updates main
+deb http://ubuntu.com/ubuntu/ xenial-proposed main"""
+        expect = ("""deb http://ubuntu.com//ubuntu xenial main
+# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu """
+                  """xenial-updates main
+# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu """
+                  """xenial-security main
+deb-src http://ubuntu.com//ubuntu universe multiverse
+#foo
+#deb http://UBUNTU.com//ubuntu xenial-updates main
+# suite disabled by cloud-init: deb http://UBUNTU.COM//ubuntu """
+                  """xenial-updates main
+deb http://ubuntu.com/ubuntu/ xenial-proposed main""")
+        result = cc_apt_configure.disable_suites(disabled, orig, release)
+        self.assertEqual(expect, result)
+
+        # single disable custom suite
+        disabled = ["foobar"]
+        orig = """deb http://ubuntu.com//ubuntu xenial main
+deb http://ubuntu.com//ubuntu xenial-updates main
+deb http://ubuntu.com//ubuntu xenial-security main
+deb http://ubuntu.com/ubuntu/ foobar main"""
+        expect = """deb http://ubuntu.com//ubuntu xenial main
+deb http://ubuntu.com//ubuntu xenial-updates main
+deb http://ubuntu.com//ubuntu xenial-security main
+# suite disabled by cloud-init: deb http://ubuntu.com/ubuntu/ foobar main"""
+        result = cc_apt_configure.disable_suites(disabled, orig, release)
+        self.assertEqual(expect, result)
+
+        # single disable non existing suite
+        disabled = ["foobar"]
+        orig = """deb http://ubuntu.com//ubuntu xenial main
+deb http://ubuntu.com//ubuntu xenial-updates main
+deb http://ubuntu.com//ubuntu xenial-security main
+deb http://ubuntu.com/ubuntu/ notfoobar main"""
+        expect = """deb http://ubuntu.com//ubuntu xenial main
+deb http://ubuntu.com//ubuntu xenial-updates main
+deb http://ubuntu.com//ubuntu xenial-security main
+deb http://ubuntu.com/ubuntu/ notfoobar main"""
+        result = cc_apt_configure.disable_suites(disabled, orig, release)
+        self.assertEqual(expect, result)
+
+        # single disable suite with option
+        disabled = ["$RELEASE-updates"]
+        orig = """deb http://ubuntu.com//ubuntu xenial main
+deb [a=b] http://ubu.com//ubu xenial-updates main
+deb http://ubuntu.com//ubuntu xenial-security main
+deb-src http://ubuntu.com//ubuntu universe multiverse
+deb http://ubuntu.com/ubuntu/ xenial-proposed main"""
+        expect = ("""deb http://ubuntu.com//ubuntu xenial main
+# suite disabled by cloud-init: deb [a=b] http://ubu.com//ubu """
+                  """xenial-updates main
+deb http://ubuntu.com//ubuntu xenial-security main
+deb-src http://ubuntu.com//ubuntu universe multiverse
+deb http://ubuntu.com/ubuntu/ xenial-proposed main""")
+        result = cc_apt_configure.disable_suites(disabled, orig, release)
+        self.assertEqual(expect, result)
+
+        # single disable suite with more options and auto $RELEASE expansion
+        disabled = ["updates"]
+        orig = """deb http://ubuntu.com//ubuntu xenial main
+deb [a=b c=d] http://ubu.com//ubu xenial-updates main
+deb http://ubuntu.com//ubuntu xenial-security main
+deb-src http://ubuntu.com//ubuntu universe multiverse
+deb http://ubuntu.com/ubuntu/ xenial-proposed main"""
+        expect = """deb http://ubuntu.com//ubuntu xenial main
+# suite disabled by cloud-init: deb [a=b c=d] \
+http://ubu.com//ubu xenial-updates main
+deb http://ubuntu.com//ubuntu xenial-security main
+deb-src http://ubuntu.com//ubuntu universe multiverse
+deb http://ubuntu.com/ubuntu/ xenial-proposed main"""
+        result = cc_apt_configure.disable_suites(disabled, orig, release)
+        self.assertEqual(expect, result)
+
+        # single disable suite while options at others
+        disabled = ["$RELEASE-security"]
+        orig = """deb http://ubuntu.com//ubuntu xenial main
+deb [arch=foo] http://ubuntu.com//ubuntu xenial-updates main
+deb http://ubuntu.com//ubuntu xenial-security main
+deb-src http://ubuntu.com//ubuntu universe multiverse
+deb http://ubuntu.com/ubuntu/ xenial-proposed main"""
+        expect = ("""deb http://ubuntu.com//ubuntu xenial main
+deb [arch=foo] http://ubuntu.com//ubuntu xenial-updates main
+# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu """
+                  """xenial-security main
+deb-src http://ubuntu.com//ubuntu universe multiverse
+deb http://ubuntu.com/ubuntu/ xenial-proposed main""")
+        result = cc_apt_configure.disable_suites(disabled, orig, release)
+        self.assertEqual(expect, result)
+
+    def test_disable_suites_blank_lines(self):
+        """test_disable_suites_blank_lines - ensure blank lines allowed"""
+        lines = ["deb %(repo)s %(rel)s main universe",
+                 "",
+                 "deb %(repo)s %(rel)s-updates main universe",
+                 "   # random comment",
+                 "#comment here",
+                 ""]
+        rel = "trusty"
+        repo = 'http://example.com/mirrors/ubuntu'
+        orig = "\n".join(lines) % {'repo': repo, 'rel': rel}
+        self.assertEqual(
+            orig, cc_apt_configure.disable_suites(["proposed"], orig, rel))
+
+    def test_apt_v3_mirror_search_dns(self):
+        """test_apt_v3_mirror_search_dns - Test searching dns patterns"""
+        pmir = "phit"
+        smir = "shit"
+        arch = 'amd64'
+        mycloud = self._get_cloud('ubuntu')
+        cfg = {"primary": [{'arches': ["default"],
+                            "search_dns": True}],
+               "security": [{'arches': ["default"],
+                             "search_dns": True}]}
+
+        with mock.patch.object(cc_apt_configure, 'get_mirror',
+                               return_value="http://mocked/foo") as mockgm:
+            mirrors = cc_apt_configure.find_apt_mirror_info(cfg, mycloud, arch)
+        calls = [call(cfg, 'primary', arch, mycloud),
+                 call(cfg, 'security', arch, mycloud)]
+        mockgm.assert_has_calls(calls)
+
+        with mock.patch.object(cc_apt_configure, 'search_for_mirror_dns',
+                               return_value="http://mocked/foo") as mocksdns:
+            mirrors = cc_apt_configure.find_apt_mirror_info(cfg, mycloud, arch)
+        calls = [call(True, 'primary', cfg, mycloud),
+                 call(True, 'security', cfg, mycloud)]
+        mocksdns.assert_has_calls(calls)
+
+        # first return is for the non-dns call before
+        with mock.patch.object(cc_apt_configure, 'search_for_mirror',
+                               side_effect=[None, pmir, None, smir]) as mockse:
+            mirrors = cc_apt_configure.find_apt_mirror_info(cfg, mycloud, arch)
+
+        calls = [call(None),
+                 call(['http://ubuntu-mirror.localdomain/ubuntu',
+                       'http://ubuntu-mirror/ubuntu']),
+                 call(None),
+                 call(['http://ubuntu-security-mirror.localdomain/ubuntu',
+                       'http://ubuntu-security-mirror/ubuntu'])]
+        mockse.assert_has_calls(calls)
+
+        self.assertEqual(mirrors['MIRROR'],
+                         pmir)
+        self.assertEqual(mirrors['PRIMARY'],
+                         pmir)
+        self.assertEqual(mirrors['SECURITY'],
+                         smir)
+
+
+class TestDebconfSelections(TestCase):
+
+    @mock.patch("cloudinit.config.cc_apt_configure.debconf_set_selections")
+    def test_no_set_sel_if_none_to_set(self, m_set_sel):
+        cc_apt_configure.apply_debconf_selections({'foo': 'bar'})
+        m_set_sel.assert_not_called()
+
+    @mock.patch("cloudinit.config.cc_apt_configure."
+                "debconf_set_selections")
+    @mock.patch("cloudinit.config.cc_apt_configure."
+                "util.get_installed_packages")
+    def test_set_sel_call_has_expected_input(self, m_get_inst, m_set_sel):
+        data = {
+            'set1': 'pkga pkga/q1 mybool false',
+            'set2': ('pkgb\tpkgb/b1\tstr\tthis is a string\n'
+                     'pkgc\tpkgc/ip\tstring\t10.0.0.1')}
+        lines = '\n'.join(data.values()).split('\n')
+
+        m_get_inst.return_value = ["adduser", "apparmor"]
+        m_set_sel.return_value = None
+
+        cc_apt_configure.apply_debconf_selections({'debconf_selections': data})
+        self.assertTrue(m_get_inst.called)
+        self.assertEqual(m_set_sel.call_count, 1)
+
+        # assumes called with *args value.
+        selections = m_set_sel.call_args_list[0][0][0].decode()
+
+        missing = [l for l in lines if l not in selections.splitlines()]
+        self.assertEqual([], missing)
+
+    @mock.patch("cloudinit.config.cc_apt_configure.dpkg_reconfigure")
+    @mock.patch("cloudinit.config.cc_apt_configure.debconf_set_selections")
+    @mock.patch("cloudinit.config.cc_apt_configure."
+                "util.get_installed_packages")
+    def test_reconfigure_if_intersection(self, m_get_inst, m_set_sel,
+                                         m_dpkg_r):
+        data = {
+            'set1': 'pkga pkga/q1 mybool false',
+            'set2': ('pkgb\tpkgb/b1\tstr\tthis is a string\n'
+                     'pkgc\tpkgc/ip\tstring\t10.0.0.1'),
+            'cloud-init': ('cloud-init cloud-init/datasources'
+                           'multiselect MAAS')}
+
+        m_set_sel.return_value = None
+        m_get_inst.return_value = ["adduser", "apparmor", "pkgb",
+                                   "cloud-init", 'zdog']
+
+        cc_apt_configure.apply_debconf_selections({'debconf_selections': data})
+
+        # reconfigure should be called with the intersection
+        # of (packages in config, packages installed)
+        self.assertEqual(m_dpkg_r.call_count, 1)
+        # assumes called with *args (dpkg_reconfigure([a,b,c], target=))
+        packages = m_dpkg_r.call_args_list[0][0][0]
+        self.assertEqual(set(['cloud-init', 'pkgb']), set(packages))
+
+    @mock.patch("cloudinit.config.cc_apt_configure.dpkg_reconfigure")
+    @mock.patch("cloudinit.config.cc_apt_configure.debconf_set_selections")
+    @mock.patch("cloudinit.config.cc_apt_configure."
+                "util.get_installed_packages")
+    def test_reconfigure_if_no_intersection(self, m_get_inst, m_set_sel,
+                                            m_dpkg_r):
+        data = {'set1': 'pkga pkga/q1 mybool false'}
+
+        m_get_inst.return_value = ["adduser", "apparmor", "pkgb",
+                                   "cloud-init", 'zdog']
+        m_set_sel.return_value = None
+
+        cc_apt_configure.apply_debconf_selections({'debconf_selections': data})
+
+        self.assertTrue(m_get_inst.called)
+        self.assertEqual(m_dpkg_r.call_count, 0)
+
+    @mock.patch("cloudinit.config.cc_apt_configure.util.subp")
+    def test_dpkg_reconfigure_does_reconfigure(self, m_subp):
+        target = "/foo-target"
+
+        # due to the way the cleaners are called (via dictionary reference)
+        # mocking clean_cloud_init directly does not work.  So we mock
+        # the CONFIG_CLEANERS dictionary and assert our cleaner is called.
+        ci_cleaner = mock.MagicMock()
+        with mock.patch.dict(("cloudinit.config.cc_apt_configure."
+                              "CONFIG_CLEANERS"),
+                             values={'cloud-init': ci_cleaner}, clear=True):
+            cc_apt_configure.dpkg_reconfigure(['pkga', 'cloud-init'],
+                                              target=target)
+        # cloud-init is actually the only package we have a cleaner for
+        # so for now, its the only one that should reconfigured
+        self.assertTrue(m_subp.called)
+        ci_cleaner.assert_called_with(target)
+        self.assertEqual(m_subp.call_count, 1)
+        found = m_subp.call_args_list[0][0][0]
+        expected = ['dpkg-reconfigure', '--frontend=noninteractive',
+                    'cloud-init']
+        self.assertEqual(expected, found)
+
+    @mock.patch("cloudinit.config.cc_apt_configure.util.subp")
+    def test_dpkg_reconfigure_not_done_on_no_data(self, m_subp):
+        cc_apt_configure.dpkg_reconfigure([])
+        m_subp.assert_not_called()
+
+    @mock.patch("cloudinit.config.cc_apt_configure.util.subp")
+    def test_dpkg_reconfigure_not_done_if_no_cleaners(self, m_subp):
+        cc_apt_configure.dpkg_reconfigure(['pkgfoo', 'pkgbar'])
+        m_subp.assert_not_called()
+
+#
+# vi: ts=4 expandtab
diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
index 73369cd..d2031f5 100644
--- a/tests/unittests/test_util.py
+++ b/tests/unittests/test_util.py
@@ -508,4 +508,73 @@ class TestReadSeeded(helpers.TestCase):
         self.assertEqual(found_md, {'key1': 'val1'})
         self.assertEqual(found_ud, ud)
 
+
+class TestSubp(helpers.TestCase):
+
+    stdin2err = ['bash', '-c', 'cat >&2']
+    stdin2out = ['cat']
+    utf8_invalid = b'ab\xaadef'
+    utf8_valid = b'start \xc3\xa9 end'
+    utf8_valid_2 = b'd\xc3\xa9j\xc8\xa7'
+
+    def printf_cmd(self, *args):
+        # bash's printf supports \xaa.  So does /usr/bin/printf
+        # but by using bash, we remove dependency on another program.
+        return(['bash', '-c', 'printf "$@"', 'printf'] + list(args))
+
+    def test_subp_handles_utf8(self):
+        # The given bytes contain utf-8 accented characters as seen in e.g.
+        # the "deja dup" package in Ubuntu.
+        cmd = self.printf_cmd(self.utf8_valid_2)
+        (out, _err) = util.subp(cmd, capture=True)
+        self.assertEqual(out, self.utf8_valid_2.decode('utf-8'))
+
+    def test_subp_respects_decode_false(self):
+        (out, err) = util.subp(self.stdin2out, capture=True, decode=False,
+                               data=self.utf8_valid)
+        self.assertTrue(isinstance(out, bytes))
+        self.assertTrue(isinstance(err, bytes))
+        self.assertEqual(out, self.utf8_valid)
+
+    def test_subp_decode_ignore(self):
+        # this executes a string that writes invalid utf-8 to stdout
+        (out, _err) = util.subp(self.printf_cmd('abc\\xaadef'),
+                                capture=True, decode='ignore')
+        self.assertEqual(out, 'abcdef')
+
+    def test_subp_decode_strict_valid_utf8(self):
+        (out, _err) = util.subp(self.stdin2out, capture=True,
+                                decode='strict', data=self.utf8_valid)
+        self.assertEqual(out, self.utf8_valid.decode('utf-8'))
+
+    def test_subp_decode_invalid_utf8_replaces(self):
+        (out, _err) = util.subp(self.stdin2out, capture=True,
+                                data=self.utf8_invalid)
+        expected = self.utf8_invalid.decode('utf-8', errors='replace')
+        self.assertEqual(out, expected)
+
+    def test_subp_decode_strict_raises(self):
+        args = []
+        kwargs = {'args': self.stdin2out, 'capture': True,
+                  'decode': 'strict', 'data': self.utf8_invalid}
+        self.assertRaises(UnicodeDecodeError, util.subp, *args, **kwargs)
+
+    def test_subp_capture_stderr(self):
+        data = b'hello world'
+        (out, err) = util.subp(self.stdin2err, capture=True,
+                               decode=False, data=data)
+        self.assertEqual(err, data)
+        self.assertEqual(out, b'')
+
+    def test_returns_none_if_no_capture(self):
+        (out, err) = util.subp(self.stdin2out, data=b'', capture=False)
+        self.assertEqual(err, None)
+        self.assertEqual(out, None)
+
+    def test_bunch_of_slashes_in_path(self):
+        self.assertEqual("/target/my/path/",
+                         util.target_path("/target/", "//my/path/"))
+        self.assertEqual("/target/my/path/",
+                         util.target_path("/target/", "///my/path/"))
+
 # vi: ts=4 expandtab