Merge ~raharper/cloud-init:ubuntu-devel-new-artful-release-v2 into cloud-init:ubuntu/devel

[Ryan Harper]

Create a new artful release to ubuntu/devel

Following new upstream release instructions:
  https://gist.github.com/smoser/6391b854e6a80475aac473bba4ef0310#file-ubuntu-release-process-md

[Scott Moser (smoser)]

Uploaded.

Thank you Ryan,
https://lists.ubuntu.com/archives/artful-changes/2017-August/008659.html

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:46afa1266e511cd3e48f4703fed49650e108a33b
https://jenkins.ubuntu.com/server/job/cloud-init-ci/177/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Unit & Style Tests
    SUCCESS: Ubuntu LTS: Build
    SUCCESS: Ubuntu LTS: Integration
    SUCCESS: MAAS Compatability Testing
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/177/rebuild

review: Approve (continuous-integration)

diff --git a/Makefile b/Makefile
index e9f5498..9e7f4ee 100644
--- a/Makefile
+++ b/Makefile
@@ -46,15 +46,15 @@ pyflakes:
 
 pyflakes3:
 	@$(CWD)/tools/run-pyflakes3
-	
+
 unittest: clean_pyc
-	nosetests $(noseopts) tests/unittests
+	nosetests $(noseopts) tests/unittests cloudinit
 
 unittest3: clean_pyc
 	nosetests3 $(noseopts) tests/unittests
 
 ci-deps-ubuntu:
-	@$(PYVER) $(CWD)/tools/read-dependencies --distro-ubuntu --test-distro
+	@$(PYVER) $(CWD)/tools/read-dependencies --distro ubuntu --test-distro
 
 ci-deps-centos:
 	@$(PYVER) $(CWD)/tools/read-dependencies --distro centos --test-distro
diff --git a/cloudinit/analyze/__init__.py b/cloudinit/analyze/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/cloudinit/analyze/__init__.py
diff --git a/cloudinit/analyze/__main__.py b/cloudinit/analyze/__main__.py
new file mode 100644
index 0000000..71cba4f
--- /dev/null
+++ b/cloudinit/analyze/__main__.py
@@ -0,0 +1,155 @@
+# Copyright (C) 2017 Canonical Ltd.
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import argparse
+import re
+import sys
+
+from . import dump
+from . import show
+
+
+def get_parser(parser=None):
+    if not parser:
+        parser = argparse.ArgumentParser(
+            prog='cloudinit-analyze',
+            description='Devel tool: Analyze cloud-init logs and data')
+    subparsers = parser.add_subparsers(title='Subcommands', dest='subcommand')
+    subparsers.required = True
+
+    parser_blame = subparsers.add_parser(
+        'blame', help='Print list of executed stages ordered by time to init')
+    parser_blame.add_argument(
+        '-i', '--infile', action='store', dest='infile',
+        default='/var/log/cloud-init.log',
+        help='specify where to read input.')
+    parser_blame.add_argument(
+        '-o', '--outfile', action='store', dest='outfile', default='-',
+        help='specify where to write output. ')
+    parser_blame.set_defaults(action=('blame', analyze_blame))
+
+    parser_show = subparsers.add_parser(
+        'show', help='Print list of in-order events during execution')
+    parser_show.add_argument('-f', '--format', action='store',
+                             dest='print_format', default='%I%D @%Es +%ds',
+                             help='specify formatting of output.')
+    parser_show.add_argument('-i', '--infile', action='store',
+                             dest='infile', default='/var/log/cloud-init.log',
+                             help='specify where to read input.')
+    parser_show.add_argument('-o', '--outfile', action='store',
+                             dest='outfile', default='-',
+                             help='specify where to write output.')
+    parser_show.set_defaults(action=('show', analyze_show))
+    parser_dump = subparsers.add_parser(
+        'dump', help='Dump cloud-init events in JSON format')
+    parser_dump.add_argument('-i', '--infile', action='store',
+                             dest='infile', default='/var/log/cloud-init.log',
+                             help='specify where to read input. ')
+    parser_dump.add_argument('-o', '--outfile', action='store',
+                             dest='outfile', default='-',
+                             help='specify where to write output. ')
+    parser_dump.set_defaults(action=('dump', analyze_dump))
+    return parser
+
+
+def analyze_blame(name, args):
+    """Report a list of records sorted by largest time delta.
+
+    For example:
+      30.210s (init-local) searching for datasource
+       8.706s (init-network) reading and applying user-data
+        166ms (modules-config) ....
+        807us (modules-final) ...
+
+    We generate event records parsing cloud-init logs, formatting the output
+    and sorting by record data ('delta')
+    """
+    (infh, outfh) = configure_io(args)
+    blame_format = '     %ds (%n)'
+    r = re.compile('(^\s+\d+\.\d+)', re.MULTILINE)
+    for idx, record in enumerate(show.show_events(_get_events(infh),
+                                                  blame_format)):
+        srecs = sorted(filter(r.match, record), reverse=True)
+        outfh.write('-- Boot Record %02d --\n' % (idx + 1))
+        outfh.write('\n'.join(srecs) + '\n')
+        outfh.write('\n')
+    outfh.write('%d boot records analyzed\n' % (idx + 1))
+
+
+def analyze_show(name, args):
+    """Generate output records using the 'standard' format to printing events.
+
+    Example output follows:
+        Starting stage: (init-local)
+          ...
+        Finished stage: (init-local) 0.105195 seconds
+
+        Starting stage: (init-network)
+          ...
+        Finished stage: (init-network) 0.339024 seconds
+
+        Starting stage: (modules-config)
+          ...
+        Finished stage: (modules-config) 0.NNN seconds
+
+        Starting stage: (modules-final)
+          ...
+        Finished stage: (modules-final) 0.NNN seconds
+    """
+    (infh, outfh) = configure_io(args)
+    for idx, record in enumerate(show.show_events(_get_events(infh),
+                                                  args.print_format)):
+        outfh.write('-- Boot Record %02d --\n' % (idx + 1))
+        outfh.write('The total time elapsed since completing an event is'
+                    ' printed after the "@" character.\n')
+        outfh.write('The time the event takes is printed after the "+" '
+                    'character.\n\n')
+        outfh.write('\n'.join(record) + '\n')
+    outfh.write('%d boot records analyzed\n' % (idx + 1))
+
+
+def analyze_dump(name, args):
+    """Dump cloud-init events in json format"""
+    (infh, outfh) = configure_io(args)
+    outfh.write(dump.json_dumps(_get_events(infh)) + '\n')
+
+
+def _get_events(infile):
+    rawdata = None
+    events, rawdata = show.load_events(infile, None)
+    if not events:
+        events, _ = dump.dump_events(rawdata=rawdata)
+    return events
+
+
+def configure_io(args):
+    """Common parsing and setup of input/output files"""
+    if args.infile == '-':
+        infh = sys.stdin
+    else:
+        try:
+            infh = open(args.infile, 'r')
+        except (FileNotFoundError, PermissionError):
+            sys.stderr.write('Cannot open file %s\n' % args.infile)
+            sys.exit(1)
+
+    if args.outfile == '-':
+        outfh = sys.stdout
+    else:
+        try:
+            outfh = open(args.outfile, 'w')
+        except PermissionError:
+            sys.stderr.write('Cannot open file %s\n' % args.outfile)
+            sys.exit(1)
+
+    return (infh, outfh)
+
+
+if __name__ == '__main__':
+    parser = get_parser()
+    args = parser.parse_args()
+    (name, action_functor) = args.action
+    action_functor(name, args)
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/analyze/dump.py b/cloudinit/analyze/dump.py
new file mode 100644
index 0000000..ca4da49
--- /dev/null
+++ b/cloudinit/analyze/dump.py
@@ -0,0 +1,176 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import calendar
+from datetime import datetime
+import json
+import sys
+
+from cloudinit import util
+
+stage_to_description = {
+    'finished': 'finished running cloud-init',
+    'init-local': 'starting search for local datasources',
+    'init-network': 'searching for network datasources',
+    'init': 'searching for network datasources',
+    'modules-config': 'running config modules',
+    'modules-final': 'finalizing modules',
+    'modules': 'running modules for',
+    'single': 'running single module ',
+}
+
+# logger's asctime format
+CLOUD_INIT_ASCTIME_FMT = "%Y-%m-%d %H:%M:%S,%f"
+
+# journctl -o short-precise
+CLOUD_INIT_JOURNALCTL_FMT = "%b %d %H:%M:%S.%f %Y"
+
+# other
+DEFAULT_FMT = "%b %d %H:%M:%S %Y"
+
+
+def parse_timestamp(timestampstr):
+    # default syslog time does not include the current year
+    months = [calendar.month_abbr[m] for m in range(1, 13)]
+    if timestampstr.split()[0] in months:
+        # Aug 29 22:55:26
+        FMT = DEFAULT_FMT
+        if '.' in timestampstr:
+            FMT = CLOUD_INIT_JOURNALCTL_FMT
+        dt = datetime.strptime(timestampstr + " " +
+                               str(datetime.now().year),
+                               FMT)
+        timestamp = dt.strftime("%s.%f")
+    elif "," in timestampstr:
+        # 2016-09-12 14:39:20,839
+        dt = datetime.strptime(timestampstr, CLOUD_INIT_ASCTIME_FMT)
+        timestamp = dt.strftime("%s.%f")
+    else:
+        # allow date(1) to handle other formats we don't expect
+        timestamp = parse_timestamp_from_date(timestampstr)
+
+    return float(timestamp)
+
+
+def parse_timestamp_from_date(timestampstr):
+    out, _ = util.subp(['date', '+%s.%3N', '-d', timestampstr])
+    timestamp = out.strip()
+    return float(timestamp)
+
+
+def parse_ci_logline(line):
+    # Stage Starts:
+    # Cloud-init v. 0.7.7 running 'init-local' at \
+    #               Fri, 02 Sep 2016 19:28:07 +0000. Up 1.0 seconds.
+    # Cloud-init v. 0.7.7 running 'init' at \
+    #               Fri, 02 Sep 2016 19:28:08 +0000. Up 2.0 seconds.
+    # Cloud-init v. 0.7.7 finished at
+    # Aug 29 22:55:26 test1 [CLOUDINIT] handlers.py[DEBUG]: \
+    #               finish: modules-final: SUCCESS: running modules for final
+    # 2016-08-30T21:53:25.972325+00:00 y1 [CLOUDINIT] handlers.py[DEBUG]: \
+    #               finish: modules-final: SUCCESS: running modules for final
+    #
+    # Nov 03 06:51:06.074410 x2 cloud-init[106]: [CLOUDINIT] util.py[DEBUG]: \
+    #               Cloud-init v. 0.7.8 running 'init-local' at \
+    #               Thu, 03 Nov 2016 06:51:06 +0000. Up 1.0 seconds.
+    #
+    # 2017-05-22 18:02:01,088 - util.py[DEBUG]: Cloud-init v. 0.7.9 running \
+    #         'init-local' at Mon, 22 May 2017 18:02:01 +0000. Up 2.0 seconds.
+
+    separators = [' - ', ' [CLOUDINIT] ']
+    found = False
+    for sep in separators:
+        if sep in line:
+            found = True
+            break
+
+    if not found:
+        return None
+
+    (timehost, eventstr) = line.split(sep)
+
+    # journalctl -o short-precise
+    if timehost.endswith(":"):
+        timehost = " ".join(timehost.split()[0:-1])
+
+    if "," in timehost:
+        timestampstr, extra = timehost.split(",")
+        timestampstr += ",%s" % extra.split()[0]
+        if ' ' in extra:
+            hostname = extra.split()[-1]
+    else:
+        hostname = timehost.split()[-1]
+        timestampstr = timehost.split(hostname)[0].strip()
+    if 'Cloud-init v.' in eventstr:
+        event_type = 'start'
+        if 'running' in eventstr:
+            stage_and_timestamp = eventstr.split('running')[1].lstrip()
+            event_name, _ = stage_and_timestamp.split(' at ')
+            event_name = event_name.replace("'", "").replace(":", "-")
+            if event_name == "init":
+                event_name = "init-network"
+        else:
+            # don't generate a start for the 'finished at' banner
+            return None
+        event_description = stage_to_description[event_name]
+    else:
+        (pymodloglvl, event_type, event_name) = eventstr.split()[0:3]
+        event_description = eventstr.split(event_name)[1].strip()
+
+    event = {
+        'name': event_name.rstrip(":"),
+        'description': event_description,
+        'timestamp': parse_timestamp(timestampstr),
+        'origin': 'cloudinit',
+        'event_type': event_type.rstrip(":"),
+    }
+    if event['event_type'] == "finish":
+        result = event_description.split(":")[0]
+        desc = event_description.split(result)[1].lstrip(':').strip()
+        event['result'] = result
+        event['description'] = desc.strip()
+
+    return event
+
+
+def json_dumps(data):
+    return json.dumps(data, indent=1, sort_keys=True,
+                      separators=(',', ': '))
+
+
+def dump_events(cisource=None, rawdata=None):
+    events = []
+    event = None
+    CI_EVENT_MATCHES = ['start:', 'finish:', 'Cloud-init v.']
+
+    if not any([cisource, rawdata]):
+        raise ValueError('Either cisource or rawdata parameters are required')
+
+    if rawdata:
+        data = rawdata.splitlines()
+    else:
+        data = cisource.readlines()
+
+    for line in data:
+        for match in CI_EVENT_MATCHES:
+            if match in line:
+                try:
+                    event = parse_ci_logline(line)
+                except ValueError:
+                    sys.stderr.write('Skipping invalid entry\n')
+                if event:
+                    events.append(event)
+
+    return events, data
+
+
+def main():
+    if len(sys.argv) > 1:
+        cisource = open(sys.argv[1])
+    else:
+        cisource = sys.stdin
+
+    return json_dumps(dump_events(cisource))
+
+
+if __name__ == "__main__":
+    print(main())
diff --git a/cloudinit/analyze/show.py b/cloudinit/analyze/show.py
new file mode 100644
index 0000000..3b356bb
--- /dev/null
+++ b/cloudinit/analyze/show.py
@@ -0,0 +1,207 @@
+#   Copyright (C) 2016 Canonical Ltd.
+#
+#   Author: Ryan Harper <[e-mail]>
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import base64
+import datetime
+import json
+import os
+
+from cloudinit import util
+
+#  An event:
+'''
+{
+        "description": "executing late commands",
+        "event_type": "start",
+        "level": "INFO",
+        "name": "cmd-install/stage-late"
+        "origin": "cloudinit",
+        "timestamp": 1461164249.1590767,
+},
+
+    {
+        "description": "executing late commands",
+        "event_type": "finish",
+        "level": "INFO",
+        "name": "cmd-install/stage-late",
+        "origin": "cloudinit",
+        "result": "SUCCESS",
+        "timestamp": 1461164249.1590767
+    }
+
+'''
+format_key = {
+    '%d': 'delta',
+    '%D': 'description',
+    '%E': 'elapsed',
+    '%e': 'event_type',
+    '%I': 'indent',
+    '%l': 'level',
+    '%n': 'name',
+    '%o': 'origin',
+    '%r': 'result',
+    '%t': 'timestamp',
+    '%T': 'total_time',
+}
+
+formatting_help = " ".join(["{0}: {1}".format(k.replace('%', '%%'), v)
+                           for k, v in format_key.items()])
+
+
+def format_record(msg, event):
+    for i, j in format_key.items():
+        if i in msg:
+            # ensure consistent formatting of time values
+            if j in ['delta', 'elapsed', 'timestamp']:
+                msg = msg.replace(i, "{%s:08.5f}" % j)
+            else:
+                msg = msg.replace(i, "{%s}" % j)
+    return msg.format(**event)
+
+
+def dump_event_files(event):
+    content = dict((k, v) for k, v in event.items() if k not in ['content'])
+    files = content['files']
+    saved = []
+    for f in files:
+        fname = f['path']
+        fn_local = os.path.basename(fname)
+        fcontent = base64.b64decode(f['content']).decode('ascii')
+        util.write_file(fn_local, fcontent)
+        saved.append(fn_local)
+
+    return saved
+
+
+def event_name(event):
+    if event:
+        return event.get('name')
+    return None
+
+
+def event_type(event):
+    if event:
+        return event.get('event_type')
+    return None
+
+
+def event_parent(event):
+    if event:
+        return event_name(event).split("/")[0]
+    return None
+
+
+def event_timestamp(event):
+    return float(event.get('timestamp'))
+
+
+def event_datetime(event):
+    return datetime.datetime.utcfromtimestamp(event_timestamp(event))
+
+
+def delta_seconds(t1, t2):
+    return (t2 - t1).total_seconds()
+
+
+def event_duration(start, finish):
+    return delta_seconds(event_datetime(start), event_datetime(finish))
+
+
+def event_record(start_time, start, finish):
+    record = finish.copy()
+    record.update({
+        'delta': event_duration(start, finish),
+        'elapsed': delta_seconds(start_time, event_datetime(start)),
+        'indent': '|' + ' ' * (event_name(start).count('/') - 1) + '`->',
+    })
+
+    return record
+
+
+def total_time_record(total_time):
+    return 'Total Time: %3.5f seconds\n' % total_time
+
+
+def generate_records(events, blame_sort=False,
+                     print_format="(%n) %d seconds in %I%D",
+                     dump_files=False, log_datafiles=False):
+
+    sorted_events = sorted(events, key=lambda x: x['timestamp'])
+    records = []
+    start_time = None
+    total_time = 0.0
+    stage_start_time = {}
+    stages_seen = []
+    boot_records = []
+
+    unprocessed = []
+    for e in range(0, len(sorted_events)):
+        event = events[e]
+        try:
+            next_evt = events[e + 1]
+        except IndexError:
+            next_evt = None
+
+        if event_type(event) == 'start':
+            if event.get('name') in stages_seen:
+                records.append(total_time_record(total_time))
+                boot_records.append(records)
+                records = []
+                start_time = None
+                total_time = 0.0
+
+            if start_time is None:
+                stages_seen = []
+                start_time = event_datetime(event)
+                stage_start_time[event_parent(event)] = start_time
+
+            # see if we have a pair
+            if event_name(event) == event_name(next_evt):
+                if event_type(next_evt) == 'finish':
+                    records.append(format_record(print_format,
+                                                 event_record(start_time,
+                                                              event,
+                                                              next_evt)))
+            else:
+                # This is a parent event
+                records.append("Starting stage: %s" % event.get('name'))
+                unprocessed.append(event)
+                stages_seen.append(event.get('name'))
+                continue
+        else:
+            prev_evt = unprocessed.pop()
+            if event_name(event) == event_name(prev_evt):
+                record = event_record(start_time, prev_evt, event)
+                records.append(format_record("Finished stage: "
+                                             "(%n) %d seconds ",
+                                             record) + "\n")
+                total_time += record.get('delta')
+            else:
+                # not a match, put it back
+                unprocessed.append(prev_evt)
+
+    records.append(total_time_record(total_time))
+    boot_records.append(records)
+    return boot_records
+
+
+def show_events(events, print_format):
+    return generate_records(events, print_format=print_format)
+
+
+def load_events(infile, rawdata=None):
+    if rawdata:
+        data = rawdata.read()
+    else:
+        data = infile.read()
+
+    j = None
+    try:
+        j = json.loads(data)
+    except json.JSONDecodeError:
+        pass
+
+    return j, data
diff --git a/cloudinit/analyze/tests/test_dump.py b/cloudinit/analyze/tests/test_dump.py
new file mode 100644
index 0000000..2c0885d
--- /dev/null
+++ b/cloudinit/analyze/tests/test_dump.py
@@ -0,0 +1,210 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from datetime import datetime
+from textwrap import dedent
+
+from cloudinit.analyze.dump import (
+    dump_events, parse_ci_logline, parse_timestamp)
+from cloudinit.util import subp, write_file
+from tests.unittests.helpers import CiTestCase
+
+
+class TestParseTimestamp(CiTestCase):
+
+    def test_parse_timestamp_handles_cloud_init_default_format(self):
+        """Logs with cloud-init detailed formats will be properly parsed."""
+        trusty_fmt = '%Y-%m-%d %H:%M:%S,%f'
+        trusty_stamp = '2016-09-12 14:39:20,839'
+
+        parsed = parse_timestamp(trusty_stamp)
+
+        # convert ourselves
+        dt = datetime.strptime(trusty_stamp, trusty_fmt)
+        expected = float(dt.strftime('%s.%f'))
+
+        # use date(1)
+        out, _err = subp(['date', '+%s.%3N', '-d', trusty_stamp])
+        timestamp = out.strip()
+        date_ts = float(timestamp)
+
+        self.assertEqual(expected, parsed)
+        self.assertEqual(expected, date_ts)
+        self.assertEqual(date_ts, parsed)
+
+    def test_parse_timestamp_handles_syslog_adding_year(self):
+        """Syslog timestamps lack a year. Add year and properly parse."""
+        syslog_fmt = '%b %d %H:%M:%S %Y'
+        syslog_stamp = 'Aug 08 15:12:51'
+
+        # convert stamp ourselves by adding the missing year value
+        year = datetime.now().year
+        dt = datetime.strptime(syslog_stamp + " " + str(year), syslog_fmt)
+        expected = float(dt.strftime('%s.%f'))
+        parsed = parse_timestamp(syslog_stamp)
+
+        # use date(1)
+        out, _ = subp(['date', '+%s.%3N', '-d', syslog_stamp])
+        timestamp = out.strip()
+        date_ts = float(timestamp)
+
+        self.assertEqual(expected, parsed)
+        self.assertEqual(expected, date_ts)
+        self.assertEqual(date_ts, parsed)
+
+    def test_parse_timestamp_handles_journalctl_format_adding_year(self):
+        """Journalctl precise timestamps lack a year. Add year and parse."""
+        journal_fmt = '%b %d %H:%M:%S.%f %Y'
+        journal_stamp = 'Aug 08 17:15:50.606811'
+
+        # convert stamp ourselves by adding the missing year value
+        year = datetime.now().year
+        dt = datetime.strptime(journal_stamp + " " + str(year), journal_fmt)
+        expected = float(dt.strftime('%s.%f'))
+        parsed = parse_timestamp(journal_stamp)
+
+        # use date(1)
+        out, _ = subp(['date', '+%s.%6N', '-d', journal_stamp])
+        timestamp = out.strip()
+        date_ts = float(timestamp)
+
+        self.assertEqual(expected, parsed)
+        self.assertEqual(expected, date_ts)
+        self.assertEqual(date_ts, parsed)
+
+    def test_parse_unexpected_timestamp_format_with_date_command(self):
+        """Dump sends unexpected timestamp formats to data for processing."""
+        new_fmt = '%H:%M %m/%d %Y'
+        new_stamp = '17:15 08/08'
+
+        # convert stamp ourselves by adding the missing year value
+        year = datetime.now().year
+        dt = datetime.strptime(new_stamp + " " + str(year), new_fmt)
+        expected = float(dt.strftime('%s.%f'))
+        parsed = parse_timestamp(new_stamp)
+
+        # use date(1)
+        out, _ = subp(['date', '+%s.%6N', '-d', new_stamp])
+        timestamp = out.strip()
+        date_ts = float(timestamp)
+
+        self.assertEqual(expected, parsed)
+        self.assertEqual(expected, date_ts)
+        self.assertEqual(date_ts, parsed)
+
+
+class TestParseCILogLine(CiTestCase):
+
+    def test_parse_logline_returns_none_without_separators(self):
+        """When no separators are found, parse_ci_logline returns None."""
+        expected_parse_ignores = [
+            '', '-', 'adsf-asdf', '2017-05-22 18:02:01,088', 'CLOUDINIT']
+        for parse_ignores in expected_parse_ignores:
+            self.assertIsNone(parse_ci_logline(parse_ignores))
+
+    def test_parse_logline_returns_event_for_cloud_init_logs(self):
+        """parse_ci_logline returns an event parse from cloud-init format."""
+        line = (
+            "2017-08-08 20:05:07,147 - util.py[DEBUG]: Cloud-init v. 0.7.9"
+            " running 'init-local' at Tue, 08 Aug 2017 20:05:07 +0000. Up"
+            " 6.26 seconds.")
+        dt = datetime.strptime(
+            '2017-08-08 20:05:07,147', '%Y-%m-%d %H:%M:%S,%f')
+        timestamp = float(dt.strftime('%s.%f'))
+        expected = {
+            'description': 'starting search for local datasources',
+            'event_type': 'start',
+            'name': 'init-local',
+            'origin': 'cloudinit',
+            'timestamp': timestamp}
+        self.assertEqual(expected, parse_ci_logline(line))
+
+    def test_parse_logline_returns_event_for_journalctl_logs(self):
+        """parse_ci_logline returns an event parse from journalctl format."""
+        line = ("Nov 03 06:51:06.074410 x2 cloud-init[106]: [CLOUDINIT]"
+                " util.py[DEBUG]: Cloud-init v. 0.7.8 running 'init-local' at"
+                "  Thu, 03 Nov 2016 06:51:06 +0000. Up 1.0 seconds.")
+        year = datetime.now().year
+        dt = datetime.strptime(
+            'Nov 03 06:51:06.074410 %d' % year, '%b %d %H:%M:%S.%f %Y')
+        timestamp = float(dt.strftime('%s.%f'))
+        expected = {
+            'description': 'starting search for local datasources',
+            'event_type': 'start',
+            'name': 'init-local',
+            'origin': 'cloudinit',
+            'timestamp': timestamp}
+        self.assertEqual(expected, parse_ci_logline(line))
+
+    def test_parse_logline_returns_event_for_finish_events(self):
+        """parse_ci_logline returns a finish event for a parsed log line."""
+        line = ('2016-08-30 21:53:25.972325+00:00 y1 [CLOUDINIT]'
+                ' handlers.py[DEBUG]: finish: modules-final: SUCCESS: running'
+                ' modules for final')
+        expected = {
+            'description': 'running modules for final',
+            'event_type': 'finish',
+            'name': 'modules-final',
+            'origin': 'cloudinit',
+            'result': 'SUCCESS',
+            'timestamp': 1472594005.972}
+        self.assertEqual(expected, parse_ci_logline(line))
+
+
+SAMPLE_LOGS = dedent("""\
+Nov 03 06:51:06.074410 x2 cloud-init[106]: [CLOUDINIT] util.py[DEBUG]:\
+ Cloud-init v. 0.7.8 running 'init-local' at Thu, 03 Nov 2016\
+ 06:51:06 +0000. Up 1.0 seconds.
[phone]:53:25.972325+00:00 y1 [CLOUDINIT] handlers.py[DEBUG]: finish:\
+ modules-final: SUCCESS: running modules for final
+""")
+
+
+class TestDumpEvents(CiTestCase):
+    maxDiff = None
+
+    def test_dump_events_with_rawdata(self):
+        """Rawdata is split and parsed into a tuple of events and data"""
+        events, data = dump_events(rawdata=SAMPLE_LOGS)
+        expected_data = SAMPLE_LOGS.splitlines()
+        year = datetime.now().year
+        dt1 = datetime.strptime(
+            'Nov 03 06:51:06.074410 %d' % year, '%b %d %H:%M:%S.%f %Y')
+        timestamp1 = float(dt1.strftime('%s.%f'))
+        expected_events = [{
+            'description': 'starting search for local datasources',
+            'event_type': 'start',
+            'name': 'init-local',
+            'origin': 'cloudinit',
+            'timestamp': timestamp1}, {
+            'description': 'running modules for final',
+            'event_type': 'finish',
+            'name': 'modules-final',
+            'origin': 'cloudinit',
+            'result': 'SUCCESS',
+            'timestamp': 1472594005.972}]
+        self.assertEqual(expected_events, events)
+        self.assertEqual(expected_data, data)
+
+    def test_dump_events_with_cisource(self):
+        """Cisource file is read and parsed into a tuple of events and data."""
+        tmpfile = self.tmp_path('logfile')
+        write_file(tmpfile, SAMPLE_LOGS)
+        events, data = dump_events(cisource=open(tmpfile))
+        year = datetime.now().year
+        dt1 = datetime.strptime(
+            'Nov 03 06:51:06.074410 %d' % year, '%b %d %H:%M:%S.%f %Y')
+        timestamp1 = float(dt1.strftime('%s.%f'))
+        expected_events = [{
+            'description': 'starting search for local datasources',
+            'event_type': 'start',
+            'name': 'init-local',
+            'origin': 'cloudinit',
+            'timestamp': timestamp1}, {
+            'description': 'running modules for final',
+            'event_type': 'finish',
+            'name': 'modules-final',
+            'origin': 'cloudinit',
+            'result': 'SUCCESS',
+            'timestamp': 1472594005.972}]
+        self.assertEqual(expected_events, events)
+        self.assertEqual(SAMPLE_LOGS.splitlines(), [d.strip() for d in data])
diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py
index 139e03b..9c0ac86 100644
--- a/cloudinit/cmd/main.py
+++ b/cloudinit/cmd/main.py
@@ -50,13 +50,6 @@ WELCOME_MSG_TPL = ("Cloud-init v. {version} running '{action}' at "
 # Module section template
 MOD_SECTION_TPL = "cloud_%s_modules"
 
-# Things u can query on
-QUERY_DATA_TYPES = [
-    'data',
-    'data_raw',
-    'instance_id',
-]
-
 # Frequency shortname to full name
 # (so users don't have to remember the full name...)
 FREQ_SHORT_NAMES = {
@@ -510,11 +503,6 @@ def main_modules(action_name, args):
     return run_module_section(mods, name, name)
 
 
-def main_query(name, _args):
-    raise NotImplementedError(("Action '%s' is not"
-                               " currently implemented") % (name))
-
-
 def main_single(name, args):
     # Cloud-init single stage is broken up into the following sub-stages
     # 1. Ensure that the init object fetches its config without errors
@@ -713,9 +701,11 @@ def main(sysv_args=None):
                         default=False)
 
     parser.set_defaults(reporter=None)
-    subparsers = parser.add_subparsers()
+    subparsers = parser.add_subparsers(title='Subcommands', dest='subcommand')
+    subparsers.required = True
 
     # Each action and its sub-options (if any)
+
     parser_init = subparsers.add_parser('init',
                                         help=('initializes cloud-init and'
                                               ' performs initial modules'))
@@ -737,17 +727,6 @@ def main(sysv_args=None):
                             choices=('init', 'config', 'final'))
     parser_mod.set_defaults(action=('modules', main_modules))
 
-    # These settings are used when you want to query information
-    # stored in the cloud-init data objects/directories/files
-    parser_query = subparsers.add_parser('query',
-                                         help=('query information stored '
-                                               'in cloud-init'))
-    parser_query.add_argument("--name", '-n', action="store",
-                              help="item name to query on",
-                              required=True,
-                              choices=QUERY_DATA_TYPES)
-    parser_query.set_defaults(action=('query', main_query))
-
     # This subcommand allows you to run a single module
     parser_single = subparsers.add_parser('single',
                                           help=('run a single module '))
@@ -781,15 +760,22 @@ def main(sysv_args=None):
                                             help=('list defined features'))
     parser_features.set_defaults(action=('features', main_features))
 
+    parser_analyze = subparsers.add_parser(
+        'analyze', help='Devel tool: Analyze cloud-init logs and data')
+    if sysv_args and sysv_args[0] == 'analyze':
+        # Only load this parser if analyze is specified to avoid file load cost
+        # FIXME put this under 'devel' subcommand (coming in next branch)
+        from cloudinit.analyze.__main__ import get_parser as analyze_parser
+        # Construct analyze subcommand parser
+        analyze_parser(parser_analyze)
+
     args = parser.parse_args(args=sysv_args)
 
-    try:
-        (name, functor) = args.action
-    except AttributeError:
-        parser.error('too few arguments')
+    # Subparsers.required = True and each subparser sets action=(name, functor)
+    (name, functor) = args.action
 
     # Setup basic logging to start (until reinitialized)
-    # iff in debug mode...
+    # iff in debug mode.
     if args.debug:
         logging.setupBasicLogging()
 
diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py
index 31ed64e..a02b4bf 100644
--- a/cloudinit/config/cc_ntp.py
+++ b/cloudinit/config/cc_ntp.py
@@ -50,6 +50,7 @@ LOG = logging.getLogger(__name__)
 
 frequency = PER_INSTANCE
 NTP_CONF = '/etc/ntp.conf'
+TIMESYNCD_CONF = '/etc/systemd/timesyncd.conf.d/cloud-init.conf'
 NR_POOL_SERVERS = 4
 distros = ['centos', 'debian', 'fedora', 'opensuse', 'ubuntu']
 
@@ -132,20 +133,50 @@ def handle(name, cfg, cloud, log, _args):
                             " is a %s %instead"), type_utils.obj_name(ntp_cfg))
 
     validate_cloudconfig_schema(cfg, schema)
+    if ntp_installable():
+        service_name = 'ntp'
+        confpath = NTP_CONF
+        template_name = None
+        packages = ['ntp']
+        check_exe = 'ntpd'
+    else:
+        service_name = 'systemd-timesyncd'
+        confpath = TIMESYNCD_CONF
+        template_name = 'timesyncd.conf'
+        packages = []
+        check_exe = '/lib/systemd/systemd-timesyncd'
+
     rename_ntp_conf()
     # ensure when ntp is installed it has a configuration file
     # to use instead of starting up with packaged defaults
-    write_ntp_config_template(ntp_cfg, cloud)
-    install_ntp(cloud.distro.install_packages, packages=['ntp'],
-                check_exe="ntpd")
-    # if ntp was already installed, it may not have started
+    write_ntp_config_template(ntp_cfg, cloud, confpath, template=template_name)
+    install_ntp(cloud.distro.install_packages, packages=packages,
+                check_exe=check_exe)
+
     try:
-        reload_ntp(systemd=cloud.distro.uses_systemd())
+        reload_ntp(service_name, systemd=cloud.distro.uses_systemd())
     except util.ProcessExecutionError as e:
         LOG.exception("Failed to reload/start ntp service: %s", e)
         raise
 
 
+def ntp_installable():
+    """Check if we can install ntp package
+
+    Ubuntu-Core systems do not have an ntp package available, so
+    we always return False.  Other systems require package managers to install
+    the ntp package If we fail to find one of the package managers, then we
+    cannot install ntp.
+    """
+    if util.system_is_snappy():
+        return False
+
+    if any(map(util.which, ['apt-get', 'dnf', 'yum', 'zypper'])):
+        return True
+
+    return False
+
+
 def install_ntp(install_func, packages=None, check_exe="ntpd"):
     if util.which(check_exe):
         return
@@ -156,7 +187,7 @@ def install_ntp(install_func, packages=None, check_exe="ntpd"):
 
 
 def rename_ntp_conf(config=None):
-    """Rename any existing ntp.conf file and render from template"""
+    """Rename any existing ntp.conf file"""
     if config is None:  # For testing
         config = NTP_CONF
     if os.path.exists(config):
@@ -171,7 +202,7 @@ def generate_server_names(distro):
     return names
 
 
-def write_ntp_config_template(cfg, cloud):
+def write_ntp_config_template(cfg, cloud, path, template=None):
     servers = cfg.get('servers', [])
     pools = cfg.get('pools', [])
 
@@ -185,19 +216,20 @@ def write_ntp_config_template(cfg, cloud):
         'pools': pools,
     }
 
-    template_fn = cloud.get_template_filename('ntp.conf.%s' %
-                                              (cloud.distro.name))
+    if template is None:
+        template = 'ntp.conf.%s' % cloud.distro.name
+
+    template_fn = cloud.get_template_filename(template)
     if not template_fn:
         template_fn = cloud.get_template_filename('ntp.conf')
         if not template_fn:
             raise RuntimeError(("No template found, "
-                                "not rendering %s"), NTP_CONF)
+                                "not rendering %s"), path)
 
-    templater.render_to_file(template_fn, NTP_CONF, params)
+    templater.render_to_file(template_fn, path, params)
 
 
-def reload_ntp(systemd=False):
-    service = 'ntp'
+def reload_ntp(service, systemd=False):
     if systemd:
         cmd = ['systemctl', 'reload-or-restart', service]
     else:
diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py
index b4c0ba7..f87a343 100644
--- a/cloudinit/distros/arch.py
+++ b/cloudinit/distros/arch.py
@@ -14,6 +14,8 @@ from cloudinit.distros.parsers.hostname import HostnameConf
 
 from cloudinit.settings import PER_INSTANCE
 
+import os
+
 LOG = logging.getLogger(__name__)
 
 
@@ -52,31 +54,10 @@ class Distro(distros.Distro):
         entries = net_util.translate_network(settings)
         LOG.debug("Translated ubuntu style network settings %s into %s",
                   settings, entries)
-        dev_names = entries.keys()
-        # Format for netctl
-        for (dev, info) in entries.items():
-            nameservers = []
-            net_fn = self.network_conf_dir + dev
-            net_cfg = {
-                'Connection': 'ethernet',
-                'Interface': dev,
-                'IP': info.get('bootproto'),
-                'Address': "('%s/%s')" % (info.get('address'),
-                                          info.get('netmask')),
-                'Gateway': info.get('gateway'),
-                'DNS': str(tuple(info.get('dns-nameservers'))).replace(',', '')
-            }
-            util.write_file(net_fn, convert_netctl(net_cfg))
-            if info.get('auto'):
-                self._enable_interface(dev)
-            if 'dns-nameservers' in info:
-                nameservers.extend(info['dns-nameservers'])
-
-        if nameservers:
-            util.write_file(self.resolve_conf_fn,
-                            convert_resolv_conf(nameservers))
-
-        return dev_names
+        return _render_network(
+            entries, resolv_conf=self.resolve_conf_fn,
+            conf_dir=self.network_conf_dir,
+            enable_func=self._enable_interface)
 
     def _enable_interface(self, device_name):
         cmd = ['netctl', 'reenable', device_name]
@@ -173,13 +154,60 @@ class Distro(distros.Distro):
                          ["-y"], freq=PER_INSTANCE)
 
 
+def _render_network(entries, target="/", conf_dir="etc/netctl",
+                    resolv_conf="etc/resolv.conf", enable_func=None):
+    """Render the translate_network format into netctl files in target.
+    Paths will be rendered under target.
+    """
+
+    devs = []
+    nameservers = []
+    resolv_conf = util.target_path(target, resolv_conf)
+    conf_dir = util.target_path(target, conf_dir)
+
+    for (dev, info) in entries.items():
+        if dev == 'lo':
+            # no configuration should be rendered for 'lo'
+            continue
+        devs.append(dev)
+        net_fn = os.path.join(conf_dir, dev)
+        net_cfg = {
+            'Connection': 'ethernet',
+            'Interface': dev,
+            'IP': info.get('bootproto'),
+            'Address': "%s/%s" % (info.get('address'),
+                                  info.get('netmask')),
+            'Gateway': info.get('gateway'),
+            'DNS': info.get('dns-nameservers', []),
+        }
+        util.write_file(net_fn, convert_netctl(net_cfg))
+        if enable_func and info.get('auto'):
+            enable_func(dev)
+        if 'dns-nameservers' in info:
+            nameservers.extend(info['dns-nameservers'])
+
+    if nameservers:
+        util.write_file(resolv_conf,
+                        convert_resolv_conf(nameservers))
+    return devs
+
+
 def convert_netctl(settings):
-    """Returns a settings string formatted for netctl."""
-    result = ''
-    if isinstance(settings, dict):
-        for k, v in settings.items():
-            result = result + '%s=%s\n' % (k, v)
-        return result
+    """Given a dictionary, returns a string in netctl profile format.
+
+    netctl profile is described at:
+    https://git.archlinux.org/netctl.git/tree/docs/netctl.profile.5.txt
+
+    Note that the 'Special Quoting Rules' are not handled here."""
+    result = []
+    for key in sorted(settings):
+        val = settings[key]
+        if val is None:
+            val = ""
+        elif isinstance(val, (tuple, list)):
+            val = "(" + ' '.join("'%s'" % v for v in val) + ")"
+        result.append("%s=%s\n" % (key, val))
+    return ''.join(result)
 
 
 def convert_resolv_conf(settings):
diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index d1740e5..a1b0db1 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -10,6 +10,7 @@ import logging
 import os
 import re
 
+from cloudinit.net.network_state import mask_to_net_prefix
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -28,8 +29,13 @@ def _natural_sort_key(s, _nsre=re.compile('([0-9]+)')):
             for text in re.split(_nsre, s)]
 
 
+def get_sys_class_path():
+    """Simple function to return the global SYS_CLASS_NET."""
+    return SYS_CLASS_NET
+
+
 def sys_dev_path(devname, path=""):
-    return SYS_CLASS_NET + devname + "/" + path
+    return get_sys_class_path() + devname + "/" + path
 
 
 def read_sys_net(devname, path, translate=None,
@@ -77,7 +83,7 @@ def read_sys_net_int(iface, field):
         return None
     try:
         return int(val)
-    except TypeError:
+    except ValueError:
         return None
 
 
@@ -149,7 +155,14 @@ def device_devid(devname):
 
 
 def get_devicelist():
-    return os.listdir(SYS_CLASS_NET)
+    try:
+        devs = os.listdir(get_sys_class_path())
+    except OSError as e:
+        if e.errno == errno.ENOENT:
+            devs = []
+        else:
+            raise
+    return devs
 
 
 class ParserError(Exception):
@@ -162,13 +175,8 @@ def is_disabled_cfg(cfg):
     return cfg.get('config') == "disabled"
 
 
-def generate_fallback_config(blacklist_drivers=None, config_driver=None):
-    """Determine which attached net dev is most likely to have a connection and
-       generate network state to run dhcp on that interface"""
-
-    if not config_driver:
-        config_driver = False
-
+def find_fallback_nic(blacklist_drivers=None):
+    """Return the name of the 'fallback' network device."""
     if not blacklist_drivers:
         blacklist_drivers = []
 
@@ -220,15 +228,24 @@ def generate_fallback_config(blacklist_drivers=None, config_driver=None):
     if DEFAULT_PRIMARY_INTERFACE in names:
         names.remove(DEFAULT_PRIMARY_INTERFACE)
         names.insert(0, DEFAULT_PRIMARY_INTERFACE)
-    target_name = None
-    target_mac = None
+
+    # pick the first that has a mac-address
     for name in names:
-        mac = read_sys_net_safe(name, 'address')
-        if mac:
-            target_name = name
-            target_mac = mac
-            break
-    if target_mac and target_name:
+        if read_sys_net_safe(name, 'address'):
+            return name
+    return None
+
+
+def generate_fallback_config(blacklist_drivers=None, config_driver=None):
+    """Determine which attached net dev is most likely to have a connection and
+       generate network state to run dhcp on that interface"""
+
+    if not config_driver:
+        config_driver = False
+
+    target_name = find_fallback_nic(blacklist_drivers=blacklist_drivers)
+    if target_name:
+        target_mac = read_sys_net_safe(target_name, 'address')
         nconf = {'config': [], 'version': 1}
         cfg = {'type': 'physical', 'name': target_name,
                'mac_address': target_mac, 'subnets': [{'type': 'dhcp'}]}
@@ -497,28 +514,8 @@ def get_interfaces_by_mac():
     """Build a dictionary of tuples {mac: name}.
 
     Bridges and any devices that have a 'stolen' mac are excluded."""
-    try:
-        devs = get_devicelist()
-    except OSError as e:
-        if e.errno == errno.ENOENT:
-            devs = []
-        else:
-            raise
     ret = {}
-    empty_mac = '00:00:00:00:00:00'
-    for name in devs:
-        if not interface_has_own_mac(name):
-            continue
-        if is_bridge(name):
-            continue
-        if is_vlan(name):
-            continue
-        mac = get_interface_mac(name)
-        # some devices may not have a mac (tun0)
-        if not mac:
-            continue
-        if mac == empty_mac and name != 'lo':
-            continue
+    for name, mac, _driver, _devid in get_interfaces():
         if mac in ret:
             raise RuntimeError(
                 "duplicate mac found! both '%s' and '%s' have mac '%s'" %
@@ -531,14 +528,8 @@ def get_interfaces():
     """Return list of interface tuples (name, mac, driver, device_id)
 
     Bridges and any devices that have a 'stolen' mac are excluded."""
-    try:
-        devs = get_devicelist()
-    except OSError as e:
-        if e.errno == errno.ENOENT:
-            devs = []
-        else:
-            raise
     ret = []
+    devs = get_devicelist()
     empty_mac = '00:00:00:00:00:00'
     for name in devs:
         if not interface_has_own_mac(name):
@@ -557,6 +548,103 @@ def get_interfaces():
     return ret
 
 
+class EphemeralIPv4Network(object):
+    """Context manager which sets up temporary static network configuration.
+
+    No operations are performed if the provided interface is already connected.
+    If unconnected, bring up the interface with valid ip, prefix and broadcast.
+    If router is provided setup a default route for that interface. Upon
+    context exit, clean up the interface leaving no configuration behind.
+    """
+
+    def __init__(self, interface, ip, prefix_or_mask, broadcast, router=None):
+        """Setup context manager and validate call signature.
+
+        @param interface: Name of the network interface to bring up.
+        @param ip: IP address to assign to the interface.
+        @param prefix_or_mask: Either netmask of the format X.X.X.X or an int
+            prefix.
+        @param broadcast: Broadcast address for the IPv4 network.
+        @param router: Optionally the default gateway IP.
+        """
+        if not all([interface, ip, prefix_or_mask, broadcast]):
+            raise ValueError(
+                'Cannot init network on {0} with {1}/{2} and bcast {3}'.format(
+                    interface, ip, prefix_or_mask, broadcast))
+        try:
+            self.prefix = mask_to_net_prefix(prefix_or_mask)
+        except ValueError as e:
+            raise ValueError(
+                'Cannot setup network: {0}'.format(e))
+        self.interface = interface
+        self.ip = ip
+        self.broadcast = broadcast
+        self.router = router
+        self.cleanup_cmds = []  # List of commands to run to cleanup state.
+
+    def __enter__(self):
+        """Perform ephemeral network setup if interface is not connected."""
+        self._bringup_device()
+        if self.router:
+            self._bringup_router()
+
+    def __exit__(self, excp_type, excp_value, excp_traceback):
+        """Teardown anything we set up."""
+        for cmd in self.cleanup_cmds:
+            util.subp(cmd, capture=True)
+
+    def _delete_address(self, address, prefix):
+        """Perform the ip command to remove the specified address."""
+        util.subp(
+            ['ip', '-family', 'inet', 'addr', 'del',
+             '%s/%s' % (address, prefix), 'dev', self.interface],
+            capture=True)
+
+    def _bringup_device(self):
+        """Perform the ip comands to fully setup the device."""
+        cidr = '{0}/{1}'.format(self.ip, self.prefix)
+        LOG.debug(
+            'Attempting setup of ephemeral network on %s with %s brd %s',
+            self.interface, cidr, self.broadcast)
+        try:
+            util.subp(
+                ['ip', '-family', 'inet', 'addr', 'add', cidr, 'broadcast',
+                 self.broadcast, 'dev', self.interface],
+                capture=True, update_env={'LANG': 'C'})
+        except util.ProcessExecutionError as e:
+            if "File exists" not in e.stderr:
+                raise
+            LOG.debug(
+                'Skip ephemeral network setup, %s already has address %s',
+                self.interface, self.ip)
+        else:
+            # Address creation success, bring up device and queue cleanup
+            util.subp(
+                ['ip', '-family', 'inet', 'link', 'set', 'dev', self.interface,
+                 'up'], capture=True)
+            self.cleanup_cmds.append(
+                ['ip', '-family', 'inet', 'link', 'set', 'dev', self.interface,
+                 'down'])
+            self.cleanup_cmds.append(
+                ['ip', '-family', 'inet', 'addr', 'del', cidr, 'dev',
+                 self.interface])
+
+    def _bringup_router(self):
+        """Perform the ip commands to fully setup the router if needed."""
+        # Check if a default route exists and exit if it does
+        out, _ = util.subp(['ip', 'route', 'show', '0.0.0.0/0'], capture=True)
+        if 'default' in out:
+            LOG.debug(
+                'Skip ephemeral route setup. %s already has default route: %s',
+                self.interface, out.strip())
+            return
+        util.subp(
+            ['ip', '-4', 'route', 'add', 'default', 'via', self.router,
+             'dev', self.interface], capture=True)
+        self.cleanup_cmds.insert(
+            0, ['ip', '-4', 'route', 'del', 'default', 'dev', self.interface])
+
+
 class RendererNotFoundError(RuntimeError):
     pass
 
diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py
new file mode 100644
index 0000000..c7febc5
--- /dev/null
+++ b/cloudinit/net/dhcp.py
@@ -0,0 +1,119 @@
+# Copyright (C) 2017 Canonical Ltd.
+#
+# Author: Chad Smith <[e-mail]>
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import logging
+import os
+import re
+
+from cloudinit.net import find_fallback_nic, get_devicelist
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+
+class InvalidDHCPLeaseFileError(Exception):
+    """Raised when parsing an empty or invalid dhcp.leases file.
+
+    Current uses are DataSourceAzure and DataSourceEc2 during ephemeral
+    boot to scrape metadata.
+    """
+    pass
+
+
+def maybe_perform_dhcp_discovery(nic=None):
+    """Perform dhcp discovery if nic valid and dhclient command exists.
+
+    If the nic is invalid or undiscoverable or dhclient command is not found,
+    skip dhcp_discovery and return an empty dict.
+
+    @param nic: Name of the network interface we want to run dhclient on.
+    @return: A dict of dhcp options from the dhclient discovery if run,
+        otherwise an empty dict is returned.
+    """
+    if nic is None:
+        nic = find_fallback_nic()
+        if nic is None:
+            LOG.debug(
+                'Skip dhcp_discovery: Unable to find fallback nic.')
+            return {}
+    elif nic not in get_devicelist():
+        LOG.debug(
+            'Skip dhcp_discovery: nic %s not found in get_devicelist.', nic)
+        return {}
+    dhclient_path = util.which('dhclient')
+    if not dhclient_path:
+        LOG.debug('Skip dhclient configuration: No dhclient command found.')
+        return {}
+    with util.tempdir(prefix='cloud-init-dhcp-') as tmpdir:
+        return dhcp_discovery(dhclient_path, nic, tmpdir)
+
+
+def parse_dhcp_lease_file(lease_file):
+    """Parse the given dhcp lease file for the most recent lease.
+
+    Return a dict of dhcp options as key value pairs for the most recent lease
+    block.
+
+    @raises: InvalidDHCPLeaseFileError on empty of unparseable leasefile
+        content.
+    """
+    lease_regex = re.compile(r"lease {(?P<lease>[^}]*)}\n")
+    dhcp_leases = []
+    lease_content = util.load_file(lease_file)
+    if len(lease_content) == 0:
+        raise InvalidDHCPLeaseFileError(
+            'Cannot parse empty dhcp lease file {0}'.format(lease_file))
+    for lease in lease_regex.findall(lease_content):
+        lease_options = []
+        for line in lease.split(';'):
+            # Strip newlines, double-quotes and option prefix
+            line = line.strip().replace('"', '').replace('option ', '')
+            if not line:
+                continue
+            lease_options.append(line.split(' ', 1))
+        dhcp_leases.append(dict(lease_options))
+    if not dhcp_leases:
+        raise InvalidDHCPLeaseFileError(
+            'Cannot parse dhcp lease file {0}. No leases found'.format(
+                lease_file))
+    return dhcp_leases
+
+
+def dhcp_discovery(dhclient_cmd_path, interface, cleandir):
+    """Run dhclient on the interface without scripts or filesystem artifacts.
+
+    @param dhclient_cmd_path: Full path to the dhclient used.
+    @param interface: Name of the network inteface on which to dhclient.
+    @param cleandir: The directory from which to run dhclient as well as store
+        dhcp leases.
+
+    @return: A dict of dhcp options parsed from the dhcp.leases file or empty
+        dict.
+    """
+    LOG.debug('Performing a dhcp discovery on %s', interface)
+
+    # XXX We copy dhclient out of /sbin/dhclient to avoid dealing with strict
+    # app armor profiles which disallow running dhclient -sf <our-script-file>.
+    # We want to avoid running /sbin/dhclient-script because of side-effects in
+    # /etc/resolv.conf any any other vendor specific scripts in
+    # /etc/dhcp/dhclient*hooks.d.
+    sandbox_dhclient_cmd = os.path.join(cleandir, 'dhclient')
+    util.copy(dhclient_cmd_path, sandbox_dhclient_cmd)
+    pid_file = os.path.join(cleandir, 'dhclient.pid')
+    lease_file = os.path.join(cleandir, 'dhcp.leases')
+
+    # ISC dhclient needs the interface up to send initial discovery packets.
+    # Generally dhclient relies on dhclient-script PREINIT action to bring the
+    # link up before attempting discovery. Since we are using -sf /bin/true,
+    # we need to do that "link up" ourselves first.
+    util.subp(['ip', 'link', 'set', 'dev', interface, 'up'], capture=True)
+    cmd = [sandbox_dhclient_cmd, '-1', '-v', '-lf', lease_file,
+           '-pf', pid_file, interface, '-sf', '/bin/true']
+    util.subp(cmd, capture=True)
+    return parse_dhcp_lease_file(lease_file)
+
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py
index 9f35b72..3b06fbf 100644
--- a/cloudinit/net/netplan.py
+++ b/cloudinit/net/netplan.py
@@ -4,7 +4,7 @@ import copy
 import os
 
 from . import renderer
-from .network_state import subnet_is_ipv6
+from .network_state import subnet_is_ipv6, NET_CONFIG_TO_V2
 
 from cloudinit import log as logging
 from cloudinit import util
@@ -27,31 +27,6 @@ network:
 """
 
 LOG = logging.getLogger(__name__)
-NET_CONFIG_TO_V2 = {
-    'bond': {'bond-ad-select': 'ad-select',
-             'bond-arp-interval': 'arp-interval',
-             'bond-arp-ip-target': 'arp-ip-target',
-             'bond-arp-validate': 'arp-validate',
-             'bond-downdelay': 'down-delay',
-             'bond-fail-over-mac': 'fail-over-mac-policy',
-             'bond-lacp-rate': 'lacp-rate',
-             'bond-miimon': 'mii-monitor-interval',
-             'bond-min-links': 'min-links',
-             'bond-mode': 'mode',
-             'bond-num-grat-arp': 'gratuitious-arp',
-             'bond-primary-reselect': 'primary-reselect-policy',
-             'bond-updelay': 'up-delay',
-             'bond-xmit-hash-policy': 'transmit-hash-policy'},
-    'bridge': {'bridge_ageing': 'ageing-time',
-               'bridge_bridgeprio': 'priority',
-               'bridge_fd': 'forward-delay',
-               'bridge_gcint': None,
-               'bridge_hello': 'hello-time',
-               'bridge_maxage': 'max-age',
-               'bridge_maxwait': None,
-               'bridge_pathcost': 'path-cost',
-               'bridge_portprio': None,
-               'bridge_waitport': None}}
 
 
 def _get_params_dict_by_match(config, match):
@@ -247,6 +222,14 @@ class Renderer(renderer.Renderer):
             util.subp(cmd, capture=True)
 
     def _render_content(self, network_state):
+
+        # if content already in netplan format, pass it back
+        if network_state.version == 2:
+            LOG.debug('V2 to V2 passthrough')
+            return util.yaml_dumps({'network': network_state.config},
+                                   explicit_start=False,
+                                   explicit_end=False)
+
         ethernets = {}
         wifis = {}
         bridges = {}
diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py
index 87a7222..6faf01b 100644
--- a/cloudinit/net/network_state.py
+++ b/cloudinit/net/network_state.py
@@ -23,6 +23,33 @@ NETWORK_V2_KEY_FILTER = [
     'match', 'mtu', 'nameservers', 'renderer', 'set-name', 'wakeonlan'
 ]
 
+NET_CONFIG_TO_V2 = {
+    'bond': {'bond-ad-select': 'ad-select',
+             'bond-arp-interval': 'arp-interval',
+             'bond-arp-ip-target': 'arp-ip-target',
+             'bond-arp-validate': 'arp-validate',
+             'bond-downdelay': 'down-delay',
+             'bond-fail-over-mac': 'fail-over-mac-policy',
+             'bond-lacp-rate': 'lacp-rate',
+             'bond-miimon': 'mii-monitor-interval',
+             'bond-min-links': 'min-links',
+             'bond-mode': 'mode',
+             'bond-num-grat-arp': 'gratuitious-arp',
+             'bond-primary': 'primary',
+             'bond-primary-reselect': 'primary-reselect-policy',
+             'bond-updelay': 'up-delay',
+             'bond-xmit-hash-policy': 'transmit-hash-policy'},
+    'bridge': {'bridge_ageing': 'ageing-time',
+               'bridge_bridgeprio': 'priority',
+               'bridge_fd': 'forward-delay',
+               'bridge_gcint': None,
+               'bridge_hello': 'hello-time',
+               'bridge_maxage': 'max-age',
+               'bridge_maxwait': None,
+               'bridge_pathcost': 'path-cost',
+               'bridge_portprio': None,
+               'bridge_waitport': None}}
+
 
 def parse_net_config_data(net_config, skip_broken=True):
     """Parses the config, returns NetworkState object
@@ -120,6 +147,10 @@ class NetworkState(object):
         self.use_ipv6 = network_state.get('use_ipv6', False)
 
     @property
+    def config(self):
+        return self._network_state['config']
+
+    @property
     def version(self):
         return self._version
 
@@ -166,12 +197,14 @@ class NetworkStateInterpreter(object):
             'search': [],
         },
         'use_ipv6': False,
+        'config': None,
     }
 
     def __init__(self, version=NETWORK_STATE_VERSION, config=None):
         self._version = version
         self._config = config
         self._network_state = copy.deepcopy(self.initial_network_state)
+        self._network_state['config'] = config
         self._parsed = False
 
     @property
@@ -460,12 +493,15 @@ class NetworkStateInterpreter(object):
         v2_command = {
           bond0: {
             'interfaces': ['interface0', 'interface1'],
-            'miimon': 100,
-            'mode': '802.3ad',
-            'xmit_hash_policy': 'layer3+4'},
+            'parameters': {
+               'mii-monitor-interval': 100,
+               'mode': '802.3ad',
+               'xmit_hash_policy': 'layer3+4'}},
           bond1: {
             'bond-slaves': ['interface2', 'interface7'],
-            'mode': 1
+            'parameters': {
+                'mode': 1,
+            }
           }
         }
 
@@ -554,6 +590,7 @@ class NetworkStateInterpreter(object):
             if not mac_address:
                 LOG.debug('NetworkState Version2: missing "macaddress" info '
                           'in config entry: %s: %s', eth, str(cfg))
+            phy_cmd.update({'mac_address': mac_address})
 
             for key in ['mtu', 'match', 'wakeonlan']:
                 if key in cfg:
@@ -598,8 +635,8 @@ class NetworkStateInterpreter(object):
             self.handle_vlan(vlan_cmd)
 
     def handle_wifis(self, command):
-        raise NotImplementedError("NetworkState V2: "
-                                  "Skipping wifi configuration")
+        LOG.warning('Wifi configuration is only available to distros with'
+                    'netplan rendering support.')
 
     def _v2_common(self, cfg):
         LOG.debug('v2_common: handling config:\n%s', cfg)
@@ -616,6 +653,11 @@ class NetworkStateInterpreter(object):
 
     def _handle_bond_bridge(self, command, cmd_type=None):
         """Common handler for bond and bridge types"""
+
+        # inverse mapping for v2 keynames to v1 keynames
+        v2key_to_v1 = dict((v, k) for k, v in
+                           NET_CONFIG_TO_V2.get(cmd_type).items())
+
         for item_name, item_cfg in command.items():
             item_params = dict((key, value) for (key, value) in
                                item_cfg.items() if key not in
@@ -624,14 +666,20 @@ class NetworkStateInterpreter(object):
                 'type': cmd_type,
                 'name': item_name,
                 cmd_type + '_interfaces': item_cfg.get('interfaces'),
-                'params': item_params,
+                'params': dict((v2key_to_v1[k], v) for k, v in
+                               item_params.get('parameters', {}).items())
             }
             subnets = self._v2_to_v1_ipcfg(item_cfg)
             if len(subnets) > 0:
                 v1_cmd.update({'subnets': subnets})
 
-            LOG.debug('v2(%ss) -> v1(%s):\n%s', cmd_type, cmd_type, v1_cmd)
-            self.handle_bridge(v1_cmd)
+            LOG.debug('v2(%s) -> v1(%s):\n%s', cmd_type, cmd_type, v1_cmd)
+            if cmd_type == "bridge":
+                self.handle_bridge(v1_cmd)
+            elif cmd_type == "bond":
+                self.handle_bond(v1_cmd)
+            else:
+                raise ValueError('Unknown command type: %s', cmd_type)
 
     def _v2_to_v1_ipcfg(self, cfg):
         """Common ipconfig extraction from v2 to v1 subnets array."""
@@ -651,12 +699,6 @@ class NetworkStateInterpreter(object):
                 'address': address,
             }
 
-            routes = []
-            for route in cfg.get('routes', []):
-                routes.append(_normalize_route(
-                    {'address': route.get('to'), 'gateway': route.get('via')}))
-            subnet['routes'] = routes
-
             if ":" in address:
                 if 'gateway6' in cfg and gateway6 is None:
                     gateway6 = cfg.get('gateway6')
@@ -667,6 +709,17 @@ class NetworkStateInterpreter(object):
                     subnet.update({'gateway': gateway4})
 
             subnets.append(subnet)
+
+        routes = []
+        for route in cfg.get('routes', []):
+            routes.append(_normalize_route(
+                {'destination': route.get('to'), 'gateway': route.get('via')}))
+
+        # v2 routes are bound to the interface, in v1 we add them under
+        # the first subnet since there isn't an equivalent interface level.
+        if len(subnets) and len(routes):
+            subnets[0]['routes'] = routes
+
         return subnets
 
 
@@ -721,7 +774,7 @@ def _normalize_net_keys(network, address_keys=()):
     elif netmask:
         prefix = mask_to_net_prefix(netmask)
     elif 'prefix' in net:
-        prefix = int(prefix)
+        prefix = int(net['prefix'])
     else:
         prefix = 64 if ipv6 else 24
 
diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py
index a550f97..f572796 100644
--- a/cloudinit/net/sysconfig.py
+++ b/cloudinit/net/sysconfig.py
@@ -484,7 +484,11 @@ class Renderer(renderer.Renderer):
             content.add_nameserver(nameserver)
         for searchdomain in network_state.dns_searchdomains:
             content.add_search_domain(searchdomain)
-        return "\n".join([_make_header(';'), str(content)])
+        header = _make_header(';')
+        content_str = str(content)
+        if not content_str.startswith(header):
+            content_str = header + '\n' + content_str
+        return content_str
 
     @staticmethod
     def _render_networkmanager_conf(network_state):
diff --git a/cloudinit/net/tests/__init__.py b/cloudinit/net/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/cloudinit/net/tests/__init__.py
diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py
new file mode 100644
index 0000000..47d8d46
--- /dev/null
+++ b/cloudinit/net/tests/test_dhcp.py
@@ -0,0 +1,144 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import mock
+import os
+from textwrap import dedent
+
+from cloudinit.net.dhcp import (
+    InvalidDHCPLeaseFileError, maybe_perform_dhcp_discovery,
+    parse_dhcp_lease_file, dhcp_discovery)
+from cloudinit.util import ensure_file, write_file
+from tests.unittests.helpers import CiTestCase
+
+
+class TestParseDHCPLeasesFile(CiTestCase):
+
+    def test_parse_empty_lease_file_errors(self):
+        """parse_dhcp_lease_file errors when file content is empty."""
+        empty_file = self.tmp_path('leases')
+        ensure_file(empty_file)
+        with self.assertRaises(InvalidDHCPLeaseFileError) as context_manager:
+            parse_dhcp_lease_file(empty_file)
+        error = context_manager.exception
+        self.assertIn('Cannot parse empty dhcp lease file', str(error))
+
+    def test_parse_malformed_lease_file_content_errors(self):
+        """parse_dhcp_lease_file errors when file content isn't dhcp leases."""
+        non_lease_file = self.tmp_path('leases')
+        write_file(non_lease_file, 'hi mom.')
+        with self.assertRaises(InvalidDHCPLeaseFileError) as context_manager:
+            parse_dhcp_lease_file(non_lease_file)
+        error = context_manager.exception
+        self.assertIn('Cannot parse dhcp lease file', str(error))
+
+    def test_parse_multiple_leases(self):
+        """parse_dhcp_lease_file returns a list of all leases within."""
+        lease_file = self.tmp_path('leases')
+        content = dedent("""
+            lease {
+              interface "wlp3s0";
+              fixed-address 192.168.2.74;
+              option subnet-mask 255.255.255.0;
+              option routers 192.168.2.1;
+              renew 4 2017/07/27 18:02:30;
+              expire 5 2017/07/28 07:08:15;
+            }
+            lease {
+              interface "wlp3s0";
+              fixed-address 192.168.2.74;
+              option subnet-mask 255.255.255.0;
+              option routers 192.168.2.1;
+            }
+        """)
+        expected = [
+            {'interface': 'wlp3s0', 'fixed-address': '192.168.2.74',
+             'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1',
+             'renew': '4 2017/07/27 18:02:30',
+             'expire': '5 2017/07/28 07:08:15'},
+            {'interface': 'wlp3s0', 'fixed-address': '192.168.2.74',
+             'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}]
+        write_file(lease_file, content)
+        self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file))
+
+
+class TestDHCPDiscoveryClean(CiTestCase):
+    with_logs = True
+
+    @mock.patch('cloudinit.net.dhcp.find_fallback_nic')
+    def test_no_fallback_nic_found(self, m_fallback_nic):
+        """Log and do nothing when nic is absent and no fallback is found."""
+        m_fallback_nic.return_value = None  # No fallback nic found
+        self.assertEqual({}, maybe_perform_dhcp_discovery())
+        self.assertIn(
+            'Skip dhcp_discovery: Unable to find fallback nic.',
+            self.logs.getvalue())
+
+    def test_provided_nic_does_not_exist(self):
+        """When the provided nic doesn't exist, log a message and no-op."""
+        self.assertEqual({}, maybe_perform_dhcp_discovery('idontexist'))
+        self.assertIn(
+            'Skip dhcp_discovery: nic idontexist not found in get_devicelist.',
+            self.logs.getvalue())
+
+    @mock.patch('cloudinit.net.dhcp.util.which')
+    @mock.patch('cloudinit.net.dhcp.find_fallback_nic')
+    def test_absent_dhclient_command(self, m_fallback, m_which):
+        """When dhclient doesn't exist in the OS, log the issue and no-op."""
+        m_fallback.return_value = 'eth9'
+        m_which.return_value = None  # dhclient isn't found
+        self.assertEqual({}, maybe_perform_dhcp_discovery())
+        self.assertIn(
+            'Skip dhclient configuration: No dhclient command found.',
+            self.logs.getvalue())
+
+    @mock.patch('cloudinit.net.dhcp.dhcp_discovery')
+    @mock.patch('cloudinit.net.dhcp.util.which')
+    @mock.patch('cloudinit.net.dhcp.find_fallback_nic')
+    def test_dhclient_run_with_tmpdir(self, m_fallback, m_which, m_dhcp):
+        """maybe_perform_dhcp_discovery passes tmpdir to dhcp_discovery."""
+        m_fallback.return_value = 'eth9'
+        m_which.return_value = '/sbin/dhclient'
+        m_dhcp.return_value = {'address': '192.168.2.2'}
+        self.assertEqual(
+            {'address': '192.168.2.2'}, maybe_perform_dhcp_discovery())
+        m_dhcp.assert_called_once()
+        call = m_dhcp.call_args_list[0]
+        self.assertEqual('/sbin/dhclient', call[0][0])
+        self.assertEqual('eth9', call[0][1])
+        self.assertIn('/tmp/cloud-init-dhcp-', call[0][2])
+
+    @mock.patch('cloudinit.net.dhcp.util.subp')
+    def test_dhcp_discovery_run_in_sandbox(self, m_subp):
+        """dhcp_discovery brings up the interface and runs dhclient.
+
+        It also returns the parsed dhcp.leases file generated in the sandbox.
+        """
+        tmpdir = self.tmp_dir()
+        dhclient_script = os.path.join(tmpdir, 'dhclient.orig')
+        script_content = '#!/bin/bash\necho fake-dhclient'
+        write_file(dhclient_script, script_content, mode=0o755)
+        lease_content = dedent("""
+            lease {
+              interface "eth9";
+              fixed-address 192.168.2.74;
+              option subnet-mask 255.255.255.0;
+              option routers 192.168.2.1;
+            }
+        """)
+        lease_file = os.path.join(tmpdir, 'dhcp.leases')
+        write_file(lease_file, lease_content)
+        self.assertItemsEqual(
+            [{'interface': 'eth9', 'fixed-address': '192.168.2.74',
+              'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}],
+            dhcp_discovery(dhclient_script, 'eth9', tmpdir))
+        # dhclient script got copied
+        with open(os.path.join(tmpdir, 'dhclient')) as stream:
+            self.assertEqual(script_content, stream.read())
+        # Interface was brought up before dhclient called from sandbox
+        m_subp.assert_has_calls([
+            mock.call(
+                ['ip', 'link', 'set', 'dev', 'eth9', 'up'], capture=True),
+            mock.call(
+                [os.path.join(tmpdir, 'dhclient'), '-1', '-v', '-lf',
+                 lease_file, '-pf', os.path.join(tmpdir, 'dhclient.pid'),
+                 'eth9', '-sf', '/bin/true'], capture=True)])
diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py
new file mode 100644
index 0000000..cc052a7
--- /dev/null
+++ b/cloudinit/net/tests/test_init.py
@@ -0,0 +1,522 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import copy
+import errno
+import mock
+import os
+
+import cloudinit.net as net
+from cloudinit.util import ensure_file, write_file, ProcessExecutionError
+from tests.unittests.helpers import CiTestCase
+
+
+class TestSysDevPath(CiTestCase):
+
+    def test_sys_dev_path(self):
+        """sys_dev_path returns a path under SYS_CLASS_NET for a device."""
+        dev = 'something'
+        path = 'attribute'
+        expected = net.SYS_CLASS_NET + dev + '/' + path
+        self.assertEqual(expected, net.sys_dev_path(dev, path))
+
+    def test_sys_dev_path_without_path(self):
+        """When path param isn't provided it defaults to empty string."""
+        dev = 'something'
+        expected = net.SYS_CLASS_NET + dev + '/'
+        self.assertEqual(expected, net.sys_dev_path(dev))
+
+
+class TestReadSysNet(CiTestCase):
+    with_logs = True
+
+    def setUp(self):
+        super(TestReadSysNet, self).setUp()
+        sys_mock = mock.patch('cloudinit.net.get_sys_class_path')
+        self.m_sys_path = sys_mock.start()
+        self.sysdir = self.tmp_dir() + '/'
+        self.m_sys_path.return_value = self.sysdir
+        self.addCleanup(sys_mock.stop)
+
+    def test_read_sys_net_strips_contents_of_sys_path(self):
+        """read_sys_net strips whitespace from the contents of a sys file."""
+        content = 'some stuff with trailing whitespace\t\r\n'
+        write_file(os.path.join(self.sysdir, 'dev', 'attr'), content)
+        self.assertEqual(content.strip(), net.read_sys_net('dev', 'attr'))
+
+    def test_read_sys_net_reraises_oserror(self):
+        """read_sys_net raises OSError/IOError when file doesn't exist."""
+        # Non-specific Exception because versions of python OSError vs IOError.
+        with self.assertRaises(Exception) as context_manager:  # noqa: H202
+            net.read_sys_net('dev', 'attr')
+        error = context_manager.exception
+        self.assertIn('No such file or directory', str(error))
+
+    def test_read_sys_net_handles_error_with_on_enoent(self):
+        """read_sys_net handles OSError/IOError with on_enoent if provided."""
+        handled_errors = []
+
+        def on_enoent(e):
+            handled_errors.append(e)
+
+        net.read_sys_net('dev', 'attr', on_enoent=on_enoent)
+        error = handled_errors[0]
+        self.assertIsInstance(error, Exception)
+        self.assertIn('No such file or directory', str(error))
+
+    def test_read_sys_net_translates_content(self):
+        """read_sys_net translates content when translate dict is provided."""
+        content = "you're welcome\n"
+        write_file(os.path.join(self.sysdir, 'dev', 'attr'), content)
+        translate = {"you're welcome": 'de nada'}
+        self.assertEqual(
+            'de nada',
+            net.read_sys_net('dev', 'attr', translate=translate))
+
+    def test_read_sys_net_errors_on_translation_failures(self):
+        """read_sys_net raises a KeyError and logs details on failure."""
+        content = "you're welcome\n"
+        write_file(os.path.join(self.sysdir, 'dev', 'attr'), content)
+        with self.assertRaises(KeyError) as context_manager:
+            net.read_sys_net('dev', 'attr', translate={})
+        error = context_manager.exception
+        self.assertEqual('"you\'re welcome"', str(error))
+        self.assertIn(
+            "Found unexpected (not translatable) value 'you're welcome' in "
+            "'{0}dev/attr".format(self.sysdir),
+            self.logs.getvalue())
+
+    def test_read_sys_net_handles_handles_with_onkeyerror(self):
+        """read_sys_net handles translation errors calling on_keyerror."""
+        content = "you're welcome\n"
+        write_file(os.path.join(self.sysdir, 'dev', 'attr'), content)
+        handled_errors = []
+
+        def on_keyerror(e):
+            handled_errors.append(e)
+
+        net.read_sys_net('dev', 'attr', translate={}, on_keyerror=on_keyerror)
+        error = handled_errors[0]
+        self.assertIsInstance(error, KeyError)
+        self.assertEqual('"you\'re welcome"', str(error))
+
+    def test_read_sys_net_safe_false_on_translate_failure(self):
+        """read_sys_net_safe returns False on translation failures."""
+        content = "you're welcome\n"
+        write_file(os.path.join(self.sysdir, 'dev', 'attr'), content)
+        self.assertFalse(net.read_sys_net_safe('dev', 'attr', translate={}))
+
+    def test_read_sys_net_safe_returns_false_on_noent_failure(self):
+        """read_sys_net_safe returns False on file not found failures."""
+        self.assertFalse(net.read_sys_net_safe('dev', 'attr'))
+
+    def test_read_sys_net_int_returns_none_on_error(self):
+        """read_sys_net_safe returns None on failures."""
+        self.assertFalse(net.read_sys_net_int('dev', 'attr'))
+
+    def test_read_sys_net_int_returns_none_on_valueerror(self):
+        """read_sys_net_safe returns None when content is not an int."""
+        write_file(os.path.join(self.sysdir, 'dev', 'attr'), 'NOTINT\n')
+        self.assertFalse(net.read_sys_net_int('dev', 'attr'))
+
+    def test_read_sys_net_int_returns_integer_from_content(self):
+        """read_sys_net_safe returns None on failures."""
+        write_file(os.path.join(self.sysdir, 'dev', 'attr'), '1\n')
+        self.assertEqual(1, net.read_sys_net_int('dev', 'attr'))
+
+    def test_is_up_true(self):
+        """is_up is True if sys/net/devname/operstate is 'up' or 'unknown'."""
+        for state in ['up', 'unknown']:
+            write_file(os.path.join(self.sysdir, 'eth0', 'operstate'), state)
+            self.assertTrue(net.is_up('eth0'))
+
+    def test_is_up_false(self):
+        """is_up is False if sys/net/devname/operstate is 'down' or invalid."""
+        for state in ['down', 'incomprehensible']:
+            write_file(os.path.join(self.sysdir, 'eth0', 'operstate'), state)
+            self.assertFalse(net.is_up('eth0'))
+
+    def test_is_wireless(self):
+        """is_wireless is True when /sys/net/devname/wireless exists."""
+        self.assertFalse(net.is_wireless('eth0'))
+        ensure_file(os.path.join(self.sysdir, 'eth0', 'wireless'))
+        self.assertTrue(net.is_wireless('eth0'))
+
+    def test_is_bridge(self):
+        """is_bridge is True when /sys/net/devname/bridge exists."""
+        self.assertFalse(net.is_bridge('eth0'))
+        ensure_file(os.path.join(self.sysdir, 'eth0', 'bridge'))
+        self.assertTrue(net.is_bridge('eth0'))
+
+    def test_is_bond(self):
+        """is_bond is True when /sys/net/devname/bonding exists."""
+        self.assertFalse(net.is_bond('eth0'))
+        ensure_file(os.path.join(self.sysdir, 'eth0', 'bonding'))
+        self.assertTrue(net.is_bond('eth0'))
+
+    def test_is_vlan(self):
+        """is_vlan is True when /sys/net/devname/uevent has DEVTYPE=vlan."""
+        ensure_file(os.path.join(self.sysdir, 'eth0', 'uevent'))
+        self.assertFalse(net.is_vlan('eth0'))
+        content = 'junk\nDEVTYPE=vlan\njunk\n'
+        write_file(os.path.join(self.sysdir, 'eth0', 'uevent'), content)
+        self.assertTrue(net.is_vlan('eth0'))
+
+    def test_is_connected_when_physically_connected(self):
+        """is_connected is True when /sys/net/devname/iflink reports 2."""
+        self.assertFalse(net.is_connected('eth0'))
+        write_file(os.path.join(self.sysdir, 'eth0', 'iflink'), "2")
+        self.assertTrue(net.is_connected('eth0'))
+
+    def test_is_connected_when_wireless_and_carrier_active(self):
+        """is_connected is True if wireless /sys/net/devname/carrier is 1."""
+        self.assertFalse(net.is_connected('eth0'))
+        ensure_file(os.path.join(self.sysdir, 'eth0', 'wireless'))
+        self.assertFalse(net.is_connected('eth0'))
+        write_file(os.path.join(self.sysdir, 'eth0', 'carrier'), "1")
+        self.assertTrue(net.is_connected('eth0'))
+
+    def test_is_physical(self):
+        """is_physical is True when /sys/net/devname/device exists."""
+        self.assertFalse(net.is_physical('eth0'))
+        ensure_file(os.path.join(self.sysdir, 'eth0', 'device'))
+        self.assertTrue(net.is_physical('eth0'))
+
+    def test_is_present(self):
+        """is_present is True when /sys/net/devname exists."""
+        self.assertFalse(net.is_present('eth0'))
+        ensure_file(os.path.join(self.sysdir, 'eth0', 'device'))
+        self.assertTrue(net.is_present('eth0'))
+
+
+class TestGenerateFallbackConfig(CiTestCase):
+
+    def setUp(self):
+        super(TestGenerateFallbackConfig, self).setUp()
+        sys_mock = mock.patch('cloudinit.net.get_sys_class_path')
+        self.m_sys_path = sys_mock.start()
+        self.sysdir = self.tmp_dir() + '/'
+        self.m_sys_path.return_value = self.sysdir
+        self.addCleanup(sys_mock.stop)
+
+    def test_generate_fallback_finds_connected_eth_with_mac(self):
+        """generate_fallback_config finds any connected device with a mac."""
+        write_file(os.path.join(self.sysdir, 'eth0', 'carrier'), '1')
+        write_file(os.path.join(self.sysdir, 'eth1', 'carrier'), '1')
+        mac = 'aa:bb:cc:aa:bb:cc'
+        write_file(os.path.join(self.sysdir, 'eth1', 'address'), mac)
+        expected = {
+            'config': [{'type': 'physical', 'mac_address': mac,
+                        'name': 'eth1', 'subnets': [{'type': 'dhcp'}]}],
+            'version': 1}
+        self.assertEqual(expected, net.generate_fallback_config())
+
+    def test_generate_fallback_finds_dormant_eth_with_mac(self):
+        """generate_fallback_config finds any dormant device with a mac."""
+        write_file(os.path.join(self.sysdir, 'eth0', 'dormant'), '1')
+        mac = 'aa:bb:cc:aa:bb:cc'
+        write_file(os.path.join(self.sysdir, 'eth0', 'address'), mac)
+        expected = {
+            'config': [{'type': 'physical', 'mac_address': mac,
+                        'name': 'eth0', 'subnets': [{'type': 'dhcp'}]}],
+            'version': 1}
+        self.assertEqual(expected, net.generate_fallback_config())
+
+    def test_generate_fallback_finds_eth_by_operstate(self):
+        """generate_fallback_config finds any dormant device with a mac."""
+        mac = 'aa:bb:cc:aa:bb:cc'
+        write_file(os.path.join(self.sysdir, 'eth0', 'address'), mac)
+        expected = {
+            'config': [{'type': 'physical', 'mac_address': mac,
+                        'name': 'eth0', 'subnets': [{'type': 'dhcp'}]}],
+            'version': 1}
+        valid_operstates = ['dormant', 'down', 'lowerlayerdown', 'unknown']
+        for state in valid_operstates:
+            write_file(os.path.join(self.sysdir, 'eth0', 'operstate'), state)
+            self.assertEqual(expected, net.generate_fallback_config())
+        write_file(os.path.join(self.sysdir, 'eth0', 'operstate'), 'noworky')
+        self.assertIsNone(net.generate_fallback_config())
+
+    def test_generate_fallback_config_skips_veth(self):
+        """generate_fallback_config will skip any veth interfaces."""
+        # A connected veth which gets ignored
+        write_file(os.path.join(self.sysdir, 'veth0', 'carrier'), '1')
+        self.assertIsNone(net.generate_fallback_config())
+
+    def test_generate_fallback_config_skips_bridges(self):
+        """generate_fallback_config will skip any bridges interfaces."""
+        # A connected veth which gets ignored
+        write_file(os.path.join(self.sysdir, 'eth0', 'carrier'), '1')
+        mac = 'aa:bb:cc:aa:bb:cc'
+        write_file(os.path.join(self.sysdir, 'eth0', 'address'), mac)
+        ensure_file(os.path.join(self.sysdir, 'eth0', 'bridge'))
+        self.assertIsNone(net.generate_fallback_config())
+
+    def test_generate_fallback_config_skips_bonds(self):
+        """generate_fallback_config will skip any bonded interfaces."""
+        # A connected veth which gets ignored
+        write_file(os.path.join(self.sysdir, 'eth0', 'carrier'), '1')
+        mac = 'aa:bb:cc:aa:bb:cc'
+        write_file(os.path.join(self.sysdir, 'eth0', 'address'), mac)
+        ensure_file(os.path.join(self.sysdir, 'eth0', 'bonding'))
+        self.assertIsNone(net.generate_fallback_config())
+
+
+class TestGetDeviceList(CiTestCase):
+
+    def setUp(self):
+        super(TestGetDeviceList, self).setUp()
+        sys_mock = mock.patch('cloudinit.net.get_sys_class_path')
+        self.m_sys_path = sys_mock.start()
+        self.sysdir = self.tmp_dir() + '/'
+        self.m_sys_path.return_value = self.sysdir
+        self.addCleanup(sys_mock.stop)
+
+    def test_get_devicelist_raise_oserror(self):
+        """get_devicelist raise any non-ENOENT OSerror."""
+        error = OSError('Can not do it')
+        error.errno = errno.EPERM  # Set non-ENOENT
+        self.m_sys_path.side_effect = error
+        with self.assertRaises(OSError) as context_manager:
+            net.get_devicelist()
+        exception = context_manager.exception
+        self.assertEqual('Can not do it', str(exception))
+
+    def test_get_devicelist_empty_without_sys_net(self):
+        """get_devicelist returns empty list when missing SYS_CLASS_NET."""
+        self.m_sys_path.return_value = 'idontexist'
+        self.assertEqual([], net.get_devicelist())
+
+    def test_get_devicelist_empty_with_no_devices_in_sys_net(self):
+        """get_devicelist returns empty directoty listing for SYS_CLASS_NET."""
+        self.assertEqual([], net.get_devicelist())
+
+    def test_get_devicelist_lists_any_subdirectories_in_sys_net(self):
+        """get_devicelist returns a directory listing for SYS_CLASS_NET."""
+        write_file(os.path.join(self.sysdir, 'eth0', 'operstate'), 'up')
+        write_file(os.path.join(self.sysdir, 'eth1', 'operstate'), 'up')
+        self.assertItemsEqual(['eth0', 'eth1'], net.get_devicelist())
+
+
+class TestGetInterfaceMAC(CiTestCase):
+
+    def setUp(self):
+        super(TestGetInterfaceMAC, self).setUp()
+        sys_mock = mock.patch('cloudinit.net.get_sys_class_path')
+        self.m_sys_path = sys_mock.start()
+        self.sysdir = self.tmp_dir() + '/'
+        self.m_sys_path.return_value = self.sysdir
+        self.addCleanup(sys_mock.stop)
+
+    def test_get_interface_mac_false_with_no_mac(self):
+        """get_device_list returns False when no mac is reported."""
+        ensure_file(os.path.join(self.sysdir, 'eth0', 'bonding'))
+        mac_path = os.path.join(self.sysdir, 'eth0', 'address')
+        self.assertFalse(os.path.exists(mac_path))
+        self.assertFalse(net.get_interface_mac('eth0'))
+
+    def test_get_interface_mac(self):
+        """get_interfaces returns the mac from SYS_CLASS_NET/dev/address."""
+        mac = 'aa:bb:cc:aa:bb:cc'
+        write_file(os.path.join(self.sysdir, 'eth1', 'address'), mac)
+        self.assertEqual(mac, net.get_interface_mac('eth1'))
+
+    def test_get_interface_mac_grabs_bonding_address(self):
+        """get_interfaces returns the source device mac for bonded devices."""
+        source_dev_mac = 'aa:bb:cc:aa:bb:cc'
+        bonded_mac = 'dd:ee:ff:dd:ee:ff'
+        write_file(os.path.join(self.sysdir, 'eth1', 'address'), bonded_mac)
+        write_file(
+            os.path.join(self.sysdir, 'eth1', 'bonding_slave', 'perm_hwaddr'),
+            source_dev_mac)
+        self.assertEqual(source_dev_mac, net.get_interface_mac('eth1'))
+
+    def test_get_interfaces_empty_list_without_sys_net(self):
+        """get_interfaces returns an empty list when missing SYS_CLASS_NET."""
+        self.m_sys_path.return_value = 'idontexist'
+        self.assertEqual([], net.get_interfaces())
+
+    def test_get_interfaces_by_mac_skips_empty_mac(self):
+        """Ignore 00:00:00:00:00:00 addresses from get_interfaces_by_mac."""
+        empty_mac = '00:00:00:00:00:00'
+        mac = 'aa:bb:cc:aa:bb:cc'
+        write_file(os.path.join(self.sysdir, 'eth1', 'address'), empty_mac)
+        write_file(os.path.join(self.sysdir, 'eth1', 'addr_assign_type'), '0')
+        write_file(os.path.join(self.sysdir, 'eth2', 'addr_assign_type'), '0')
+        write_file(os.path.join(self.sysdir, 'eth2', 'address'), mac)
+        expected = [('eth2', 'aa:bb:cc:aa:bb:cc', None, None)]
+        self.assertEqual(expected, net.get_interfaces())
+
+    def test_get_interfaces_by_mac_skips_missing_mac(self):
+        """Ignore interfaces without an address from get_interfaces_by_mac."""
+        write_file(os.path.join(self.sysdir, 'eth1', 'addr_assign_type'), '0')
+        address_path = os.path.join(self.sysdir, 'eth1', 'address')
+        self.assertFalse(os.path.exists(address_path))
+        mac = 'aa:bb:cc:aa:bb:cc'
+        write_file(os.path.join(self.sysdir, 'eth2', 'addr_assign_type'), '0')
+        write_file(os.path.join(self.sysdir, 'eth2', 'address'), mac)
+        expected = [('eth2', 'aa:bb:cc:aa:bb:cc', None, None)]
+        self.assertEqual(expected, net.get_interfaces())
+
+
+class TestInterfaceHasOwnMAC(CiTestCase):
+
+    def setUp(self):
+        super(TestInterfaceHasOwnMAC, self).setUp()
+        sys_mock = mock.patch('cloudinit.net.get_sys_class_path')
+        self.m_sys_path = sys_mock.start()
+        self.sysdir = self.tmp_dir() + '/'
+        self.m_sys_path.return_value = self.sysdir
+        self.addCleanup(sys_mock.stop)
+
+    def test_interface_has_own_mac_false_when_stolen(self):
+        """Return False from interface_has_own_mac when address is stolen."""
+        write_file(os.path.join(self.sysdir, 'eth1', 'addr_assign_type'), '2')
+        self.assertFalse(net.interface_has_own_mac('eth1'))
+
+    def test_interface_has_own_mac_true_when_not_stolen(self):
+        """Return False from interface_has_own_mac when mac isn't stolen."""
+        valid_assign_types = ['0', '1', '3']
+        assign_path = os.path.join(self.sysdir, 'eth1', 'addr_assign_type')
+        for _type in valid_assign_types:
+            write_file(assign_path, _type)
+            self.assertTrue(net.interface_has_own_mac('eth1'))
+
+    def test_interface_has_own_mac_strict_errors_on_absent_assign_type(self):
+        """When addr_assign_type is absent, interface_has_own_mac errors."""
+        with self.assertRaises(ValueError):
+            net.interface_has_own_mac('eth1', strict=True)
+
+
[e-mail]('cloudinit.net.util.subp')
+class TestEphemeralIPV4Network(CiTestCase):
+
+    with_logs = True
+
+    def setUp(self):
+        super(TestEphemeralIPV4Network, self).setUp()
+        sys_mock = mock.patch('cloudinit.net.get_sys_class_path')
+        self.m_sys_path = sys_mock.start()
+        self.sysdir = self.tmp_dir() + '/'
+        self.m_sys_path.return_value = self.sysdir
+        self.addCleanup(sys_mock.stop)
+
+    def test_ephemeral_ipv4_network_errors_on_missing_params(self, m_subp):
+        """No required params for EphemeralIPv4Network can be None."""
+        required_params = {
+            'interface': 'eth0', 'ip': '192.168.2.2',
+            'prefix_or_mask': '255.255.255.0', 'broadcast': '192.168.2.255'}
+        for key in required_params.keys():
+            params = copy.deepcopy(required_params)
+            params[key] = None
+            with self.assertRaises(ValueError) as context_manager:
+                net.EphemeralIPv4Network(**params)
+            error = context_manager.exception
+            self.assertIn('Cannot init network on', str(error))
+            self.assertEqual(0, m_subp.call_count)
+
+    def test_ephemeral_ipv4_network_errors_invalid_mask_prefix(self, m_subp):
+        """Raise an error when prefix_or_mask is not a netmask or prefix."""
+        params = {
+            'interface': 'eth0', 'ip': '192.168.2.2',
+            'broadcast': '192.168.2.255'}
+        invalid_masks = ('invalid', 'invalid.', '123.123.123')
+        for error_val in invalid_masks:
+            params['prefix_or_mask'] = error_val
+            with self.assertRaises(ValueError) as context_manager:
+                with net.EphemeralIPv4Network(**params):
+                    pass
+            error = context_manager.exception
+            self.assertIn('Cannot setup network: netmask', str(error))
+            self.assertEqual(0, m_subp.call_count)
+
+    def test_ephemeral_ipv4_network_performs_teardown(self, m_subp):
+        """EphemeralIPv4Network performs teardown on the device if setup."""
+        expected_setup_calls = [
+            mock.call(
+                ['ip', '-family', 'inet', 'addr', 'add', '192.168.2.2/24',
+                 'broadcast', '192.168.2.255', 'dev', 'eth0'],
+                capture=True, update_env={'LANG': 'C'}),
+            mock.call(
+                ['ip', '-family', 'inet', 'link', 'set', 'dev', 'eth0', 'up'],
+                capture=True)]
+        expected_teardown_calls = [
+            mock.call(
+                ['ip', '-family', 'inet', 'link', 'set', 'dev', 'eth0',
+                 'down'], capture=True),
+            mock.call(
+                ['ip', '-family', 'inet', 'addr', 'del', '192.168.2.2/24',
+                 'dev', 'eth0'], capture=True)]
+        params = {
+            'interface': 'eth0', 'ip': '192.168.2.2',
+            'prefix_or_mask': '255.255.255.0', 'broadcast': '192.168.2.255'}
+        with net.EphemeralIPv4Network(**params):
+            self.assertEqual(expected_setup_calls, m_subp.call_args_list)
+        m_subp.assert_has_calls(expected_teardown_calls)
+
+    def test_ephemeral_ipv4_network_noop_when_configured(self, m_subp):
+        """EphemeralIPv4Network handles exception when address is setup.
+
+        It performs no cleanup as the interface was already setup.
+        """
+        params = {
+            'interface': 'eth0', 'ip': '192.168.2.2',
+            'prefix_or_mask': '255.255.255.0', 'broadcast': '192.168.2.255'}
+        m_subp.side_effect = ProcessExecutionError(
+            '', 'RTNETLINK answers: File exists', 2)
+        expected_calls = [
+            mock.call(
+                ['ip', '-family', 'inet', 'addr', 'add', '192.168.2.2/24',
+                 'broadcast', '192.168.2.255', 'dev', 'eth0'],
+                capture=True, update_env={'LANG': 'C'})]
+        with net.EphemeralIPv4Network(**params):
+            pass
+        self.assertEqual(expected_calls, m_subp.call_args_list)
+        self.assertIn(
+            'Skip ephemeral network setup, eth0 already has address',
+            self.logs.getvalue())
+
+    def test_ephemeral_ipv4_network_with_prefix(self, m_subp):
+        """EphemeralIPv4Network takes a valid prefix to setup the network."""
+        params = {
+            'interface': 'eth0', 'ip': '192.168.2.2',
+            'prefix_or_mask': '24', 'broadcast': '192.168.2.255'}
+        for prefix_val in ['24', 16]:  # prefix can be int or string
+            params['prefix_or_mask'] = prefix_val
+            with net.EphemeralIPv4Network(**params):
+                pass
+        m_subp.assert_has_calls([mock.call(
+            ['ip', '-family', 'inet', 'addr', 'add', '192.168.2.2/24',
+             'broadcast', '192.168.2.255', 'dev', 'eth0'],
+            capture=True, update_env={'LANG': 'C'})])
+        m_subp.assert_has_calls([mock.call(
+            ['ip', '-family', 'inet', 'addr', 'add', '192.168.2.2/16',
+             'broadcast', '192.168.2.255', 'dev', 'eth0'],
+            capture=True, update_env={'LANG': 'C'})])
+
+    def test_ephemeral_ipv4_network_with_new_default_route(self, m_subp):
+        """Add the route when router is set and no default route exists."""
+        params = {
+            'interface': 'eth0', 'ip': '192.168.2.2',
+            'prefix_or_mask': '255.255.255.0', 'broadcast': '192.168.2.255',
+            'router': '192.168.2.1'}
+        m_subp.return_value = '', ''  # Empty response from ip route gw check
+        expected_setup_calls = [
+            mock.call(
+                ['ip', '-family', 'inet', 'addr', 'add', '192.168.2.2/24',
+                 'broadcast', '192.168.2.255', 'dev', 'eth0'],
+                capture=True, update_env={'LANG': 'C'}),
+            mock.call(
+                ['ip', '-family', 'inet', 'link', 'set', 'dev', 'eth0', 'up'],
+                capture=True),
+            mock.call(
+                ['ip', 'route', 'show', '0.0.0.0/0'], capture=True),
+            mock.call(
+                ['ip', '-4', 'route', 'add', 'default', 'via',
+                 '192.168.2.1', 'dev', 'eth0'], capture=True)]
+        expected_teardown_calls = [mock.call(
+            ['ip', '-4', 'route', 'del', 'default', 'dev', 'eth0'],
+            capture=True)]
+
+        with net.EphemeralIPv4Network(**params):
+            self.assertEqual(expected_setup_calls, m_subp.call_args_list)
+        m_subp.assert_has_calls(expected_teardown_calls)
diff --git a/cloudinit/sources/DataSourceAliYun.py b/cloudinit/sources/DataSourceAliYun.py
index 380e27c..43a7e42 100644
--- a/cloudinit/sources/DataSourceAliYun.py
+++ b/cloudinit/sources/DataSourceAliYun.py
@@ -6,17 +6,20 @@ from cloudinit import sources
 from cloudinit.sources import DataSourceEc2 as EC2
 from cloudinit import util
 
-DEF_MD_VERSION = "2016-01-01"
 ALIYUN_PRODUCT = "Alibaba Cloud ECS"
 
 
 class DataSourceAliYun(EC2.DataSourceEc2):
-    metadata_urls = ["http://100.100.100.200"]
+
+    metadata_urls = ['http://100.100.100.200']
+
+    # The minimum supported metadata_version from the ec2 metadata apis
+    min_metadata_version = '2016-01-01'
+    extended_metadata_versions = []
 
     def __init__(self, sys_cfg, distro, paths):
         super(DataSourceAliYun, self).__init__(sys_cfg, distro, paths)
         self.seed_dir = os.path.join(paths.seed_dir, "AliYun")
-        self.api_ver = DEF_MD_VERSION
 
     def get_hostname(self, fqdn=False, _resolve_ip=False):
         return self.metadata.get('hostname', 'localhost.localdomain')
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 4ec9592..8e5f8ee 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -13,6 +13,8 @@ import time
 
 from cloudinit import ec2_utils as ec2
 from cloudinit import log as logging
+from cloudinit import net
+from cloudinit.net import dhcp
 from cloudinit import sources
 from cloudinit import url_helper as uhelp
 from cloudinit import util
@@ -20,8 +22,7 @@ from cloudinit import warnings
 
 LOG = logging.getLogger(__name__)
 
-# Which version we are requesting of the ec2 metadata apis
-DEF_MD_VERSION = '2009-04-04'
+SKIP_METADATA_URL_CODES = frozenset([uhelp.NOT_FOUND])
 
 STRICT_ID_PATH = ("datasource", "Ec2", "strict_id")
 STRICT_ID_DEFAULT = "warn"
@@ -41,17 +42,28 @@ class Platforms(object):
 
 
 class DataSourceEc2(sources.DataSource):
+
     # Default metadata urls that will be used if none are provided
     # They will be checked for 'resolveability' and some of the
     # following may be discarded if they do not resolve
     metadata_urls = ["http://169.254.169.254", "http://instance-data.:8773"]
+
+    # The minimum supported metadata_version from the ec2 metadata apis
+    min_metadata_version = '2009-04-04'
+
+    # Priority ordered list of additional metadata versions which will be tried
+    # for extended metadata content. IPv6 support comes in 2016-09-02
+    extended_metadata_versions = ['2016-09-02']
+
     _cloud_platform = None
 
+    # Whether we want to get network configuration from the metadata service.
+    get_network_metadata = False
+
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
         self.metadata_address = None
         self.seed_dir = os.path.join(paths.seed_dir, "ec2")
-        self.api_ver = DEF_MD_VERSION
 
     def get_data(self):
         seed_ret = {}
@@ -73,21 +85,27 @@ class DataSourceEc2(sources.DataSource):
         elif self.cloud_platform == Platforms.NO_EC2_METADATA:
             return False
 
-        try:
-            if not self.wait_for_metadata_service():
+        if self.get_network_metadata:  # Setup networking in init-local stage.
+            if util.is_FreeBSD():
+                LOG.debug("FreeBSD doesn't support running dhclient with -sf")
                 return False
-            start_time = time.time()
-            self.userdata_raw = \
-                ec2.get_instance_userdata(self.api_ver, self.metadata_address)
-            self.metadata = ec2.get_instance_metadata(self.api_ver,
-                                                      self.metadata_address)
-            LOG.debug("Crawl of metadata service took %.3f seconds",
-                      time.time() - start_time)
-            return True
-        except Exception:
-            util.logexc(LOG, "Failed reading from metadata address %s",
-                        self.metadata_address)
-            return False
+            dhcp_leases = dhcp.maybe_perform_dhcp_discovery()
+            if not dhcp_leases:
+                # DataSourceEc2Local failed in init-local stage. DataSourceEc2
+                # will still run in init-network stage.
+                return False
+            dhcp_opts = dhcp_leases[-1]
+            net_params = {'interface': dhcp_opts.get('interface'),
+                          'ip': dhcp_opts.get('fixed-address'),
+                          'prefix_or_mask': dhcp_opts.get('subnet-mask'),
+                          'broadcast': dhcp_opts.get('broadcast-address'),
+                          'router': dhcp_opts.get('routers')}
+            with net.EphemeralIPv4Network(**net_params):
+                return util.log_time(
+                    logfunc=LOG.debug, msg='Crawl of metadata service',
+                    func=self._crawl_metadata)
+        else:
+            return self._crawl_metadata()
 
     @property
     def launch_index(self):
@@ -95,6 +113,32 @@ class DataSourceEc2(sources.DataSource):
             return None
         return self.metadata.get('ami-launch-index')
 
+    def get_metadata_api_version(self):
+        """Get the best supported api version from the metadata service.
+
+        Loop through all extended support metadata versions in order and
+        return the most-fully featured metadata api version discovered.
+
+        If extended_metadata_versions aren't present, return the datasource's
+        min_metadata_version.
+        """
+        # Assumes metadata service is already up
+        for api_ver in self.extended_metadata_versions:
+            url = '{0}/{1}/meta-data/instance-id'.format(
+                self.metadata_address, api_ver)
+            try:
+                resp = uhelp.readurl(url=url)
+            except uhelp.UrlError as e:
+                LOG.debug('url %s raised exception %s', url, e)
+            else:
+                if resp.code == 200:
+                    LOG.debug('Found preferred metadata version %s', api_ver)
+                    return api_ver
+                elif resp.code == 404:
+                    msg = 'Metadata api version %s not present. Headers: %s'
+                    LOG.debug(msg, api_ver, resp.headers)
+        return self.min_metadata_version
+
     def get_instance_id(self):
         return self.metadata['instance-id']
 
@@ -138,21 +182,22 @@ class DataSourceEc2(sources.DataSource):
         urls = []
         url2base = {}
         for url in mdurls:
-            cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver)
+            cur = '{0}/{1}/meta-data/instance-id'.format(
+                url, self.min_metadata_version)
             urls.append(cur)
             url2base[cur] = url
 
         start_time = time.time()
-        url = uhelp.wait_for_url(urls=urls, max_wait=max_wait,
-                                 timeout=timeout, status_cb=LOG.warn)
+        url = uhelp.wait_for_url(
+            urls=urls, max_wait=max_wait, timeout=timeout, status_cb=LOG.warn)
 
         if url:
-            LOG.debug("Using metadata source: '%s'", url2base[url])
+            self.metadata_address = url2base[url]
+            LOG.debug("Using metadata source: '%s'", self.metadata_address)
         else:
             LOG.critical("Giving up on md from %s after %s seconds",
                          urls, int(time.time() - start_time))
 
-        self.metadata_address = url2base.get(url)
         return bool(url)
 
     def device_name_to_device(self, name):
@@ -234,6 +279,37 @@ class DataSourceEc2(sources.DataSource):
                 util.get_cfg_by_path(cfg, STRICT_ID_PATH, STRICT_ID_DEFAULT),
                 cfg)
 
+    def _crawl_metadata(self):
+        """Crawl metadata service when available.
+
+        @returns: True on success, False otherwise.
+        """
+        if not self.wait_for_metadata_service():
+            return False
+        api_version = self.get_metadata_api_version()
+        try:
+            self.userdata_raw = ec2.get_instance_userdata(
+                api_version, self.metadata_address)
+            self.metadata = ec2.get_instance_metadata(
+                api_version, self.metadata_address)
+        except Exception:
+            util.logexc(
+                LOG, "Failed reading from metadata address %s",
+                self.metadata_address)
+            return False
+        return True
+
+
+class DataSourceEc2Local(DataSourceEc2):
+    """Datasource run at init-local which sets up network to query metadata.
+
+    In init-local, no network is available. This subclass sets up minimal
+    networking with dhclient on a viable nic so that it can talk to the
+    metadata service. If the metadata service provides network configuration
+    then render the network configuration for that instance based on metadata.
+    """
+    get_network_metadata = True  # Get metadata network config if present
+
 
 def read_strict_mode(cfgval, default):
     try:
@@ -349,6 +425,7 @@ def _collect_platform_data():
 
 # Used to match classes to dependencies
 datasources = [
+    (DataSourceEc2Local, (sources.DEP_FILESYSTEM,)),  # Run at init-local
     (DataSourceEc2, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
 ]
 
diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py
index f20c9a6..73d3877 100644
--- a/cloudinit/sources/DataSourceOVF.py
+++ b/cloudinit/sources/DataSourceOVF.py
@@ -25,6 +25,8 @@ from cloudinit.sources.helpers.vmware.imc.config_file \
     import ConfigFile
 from cloudinit.sources.helpers.vmware.imc.config_nic \
     import NicConfigurator
+from cloudinit.sources.helpers.vmware.imc.config_passwd \
+    import PasswordConfigurator
 from cloudinit.sources.helpers.vmware.imc.guestcust_error \
     import GuestCustErrorEnum
 from cloudinit.sources.helpers.vmware.imc.guestcust_event \
@@ -117,6 +119,8 @@ class DataSourceOVF(sources.DataSource):
                 (md, ud, cfg) = read_vmware_imc(conf)
                 dirpath = os.path.dirname(vmwareImcConfigFilePath)
                 nics = get_nics_to_enable(dirpath)
+                markerid = conf.marker_id
+                markerexists = check_marker_exists(markerid)
             except Exception as e:
                 LOG.debug("Error parsing the customization Config File")
                 LOG.exception(e)
@@ -127,7 +131,6 @@ class DataSourceOVF(sources.DataSource):
                 return False
             finally:
                 util.del_dir(os.path.dirname(vmwareImcConfigFilePath))
-
             try:
                 LOG.debug("Applying the Network customization")
                 nicConfigurator = NicConfigurator(conf.nics)
@@ -140,6 +143,35 @@ class DataSourceOVF(sources.DataSource):
                     GuestCustEventEnum.GUESTCUST_EVENT_NETWORK_SETUP_FAILED)
                 enable_nics(nics)
                 return False
+            if markerid and not markerexists:
+                LOG.debug("Applying password customization")
+                pwdConfigurator = PasswordConfigurator()
+                adminpwd = conf.admin_password
+                try:
+                    resetpwd = conf.reset_password
+                    if adminpwd or resetpwd:
+                        pwdConfigurator.configure(adminpwd, resetpwd,
+                                                  self.distro)
+                    else:
+                        LOG.debug("Changing password is not needed")
+                except Exception as e:
+                    LOG.debug("Error applying Password Configuration: %s", e)
+                    set_customization_status(
+                        GuestCustStateEnum.GUESTCUST_STATE_RUNNING,
+                        GuestCustEventEnum.GUESTCUST_EVENT_CUSTOMIZE_FAILED)
+                    enable_nics(nics)
+                    return False
+            if markerid:
+                LOG.debug("Handle marker creation")
+                try:
+                    setup_marker_files(markerid)
+                except Exception as e:
+                    LOG.debug("Error creating marker files: %s", e)
+                    set_customization_status(
+                        GuestCustStateEnum.GUESTCUST_STATE_RUNNING,
+                        GuestCustEventEnum.GUESTCUST_EVENT_CUSTOMIZE_FAILED)
+                    enable_nics(nics)
+                    return False
 
             vmwarePlatformFound = True
             set_customization_status(
@@ -445,4 +477,33 @@ datasources = (
 def get_datasource_list(depends):
     return sources.list_from_depends(depends, datasources)
 
+
+# To check if marker file exists
+def check_marker_exists(markerid):
+    """
+    Check the existence of a marker file.
+    Presence of marker file determines whether a certain code path is to be
+    executed. It is needed for partial guest customization in VMware.
+    """
+    if not markerid:
+        return False
+    markerfile = "/.markerfile-" + markerid
+    if os.path.exists(markerfile):
+        return True
+    return False
+
+
+# Create a marker file
+def setup_marker_files(markerid):
+    """
+    Create a new marker file.
+    Marker files are unique to a full customization workflow in VMware
+    environment.
+    """
+    if not markerid:
+        return
+    markerfile = "/.markerfile-" + markerid
+    util.del_file("/.markerfile-*.txt")
+    open(markerfile, 'w').close()
+
 # vi: ts=4 expandtab
diff --git a/cloudinit/sources/helpers/vmware/imc/config.py b/cloudinit/sources/helpers/vmware/imc/config.py
index 9a5e3a8..49d441d 100644
--- a/cloudinit/sources/helpers/vmware/imc/config.py
+++ b/cloudinit/sources/helpers/vmware/imc/config.py
@@ -5,6 +5,7 @@
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
+
 from .nic import Nic
 
 
@@ -14,13 +15,16 @@ class Config(object):
     Specification file.
     """
 
+    CUSTOM_SCRIPT = 'CUSTOM-SCRIPT|SCRIPT-NAME'
     DNS = 'DNS|NAMESERVER|'
-    SUFFIX = 'DNS|SUFFIX|'
+    DOMAINNAME = 'NETWORK|DOMAINNAME'
+    HOSTNAME = 'NETWORK|HOSTNAME'
+    MARKERID = 'MISC|MARKER-ID'
     PASS = 'PASSWORD|-PASS'
+    RESETPASS = 'PASSWORD|RESET'
+    SUFFIX = 'DNS|SUFFIX|'
     TIMEZONE = 'DATETIME|TIMEZONE'
     UTC = 'DATETIME|UTC'
-    HOSTNAME = 'NETWORK|HOSTNAME'
-    DOMAINNAME = 'NETWORK|DOMAINNAME'
 
     def __init__(self, configFile):
         self._configFile = configFile
@@ -82,4 +86,18 @@ class Config(object):
 
         return res
 
+    @property
+    def reset_password(self):
+        """Retreives if the root password needs to be reset."""
+        resetPass = self._configFile.get(Config.RESETPASS, 'no')
+        resetPass = resetPass.lower()
+        if resetPass not in ('yes', 'no'):
+            raise ValueError('ResetPassword value should be yes/no')
+        return resetPass == 'yes'
+
+    @property
+    def marker_id(self):
+        """Returns marker id."""
+        return self._configFile.get(Config.MARKERID, None)
+
 # vi: ts=4 expandtab
diff --git a/cloudinit/sources/helpers/vmware/imc/config_passwd.py b/cloudinit/sources/helpers/vmware/imc/config_passwd.py
new file mode 100644
index 0000000..75cfbaa
--- /dev/null
+++ b/cloudinit/sources/helpers/vmware/imc/config_passwd.py
@@ -0,0 +1,67 @@
+#    Copyright (C) 2016 Canonical Ltd.
+#    Copyright (C) 2016 VMware INC.
+#
+#    Author: Maitreyee Saikia <[e-mail]>
+#
+#    This file is part of cloud-init. See LICENSE file for license information.
+
+
+import logging
+import os
+
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+
+class PasswordConfigurator(object):
+    """
+    Class for changing configurations related to passwords in a VM. Includes
+    setting and expiring passwords.
+    """
+    def configure(self, passwd, resetPasswd, distro):
+        """
+        Main method to perform all functionalities based on configuration file
+        inputs.
+        @param passwd: encoded admin password.
+        @param resetPasswd: boolean to determine if password needs to be reset.
+        @return cfg: dict to be used by cloud-init set_passwd code.
+        """
+        LOG.info('Starting password configuration')
+        if passwd:
+            passwd = util.b64d(passwd)
+        allRootUsers = []
+        for line in open('/etc/passwd', 'r'):
+            if line.split(':')[2] == '0':
+                allRootUsers.append(line.split(':')[0])
+        # read shadow file and check for each user, if its uid0 or root.
+        uidUsersList = []
+        for line in open('/etc/shadow', 'r'):
+            user = line.split(':')[0]
+            if user in allRootUsers:
+                uidUsersList.append(user)
+        if passwd:
+            LOG.info('Setting admin password')
+            distro.set_passwd('root', passwd)
+        if resetPasswd:
+            self.reset_password(uidUsersList)
+        LOG.info('Configure Password completed!')
+
+    def reset_password(self, uidUserList):
+        """
+        Method to reset password. Use passwd --expire command. Use chage if
+        not succeeded using passwd command. Log failure message otherwise.
+        @param: list of users for which to expire password.
+        """
+        LOG.info('Expiring password.')
+        for user in uidUserList:
+            try:
+                out, err = util.subp(['passwd', '--expire', user])
+            except util.ProcessExecutionError as e:
+                if os.path.exists('/usr/bin/chage'):
+                    out, e = util.subp(['chage', '-d', '0', user])
+                else:
+                    LOG.warning('Failed to expire password for %s with error: '
+                                '%s', user, e)
+
+# vi: ts=4 expandtab
diff --git a/debian/changelog b/debian/changelog
index c5f5136..ea6872e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,31 @@
+cloud-init (0.7.9-243-ge74d775-0ubuntu1) artful; urgency=medium
+
+  * New upstream snapshot.
+    - tools: Add tooling for basic cloud-init performance analysis.
+      [Chad Smith] (LP: #1709761)
+    - network: add v2 passthrough and fix parsing v2 config with bonds/bridge
+      params [Ryan Harper] (LP: #1709180)
+    - doc: update capabilities with features available, link doc reference,
+      cli example [Ryan Harper]
+    - vcloud directory: Guest Customization support for passwords
+      [Maitreyee Saikia]
+    - ec2: Allow Ec2 to run in init-local using dhclient in a sandbox.
+      [Chad Smith] (LP: #1709772)
+    - cc_ntp: fallback on timesyncd configuration if ntp is not installable
+      [Ryan Harper] (LP: #1686485)
+    - net: Reduce duplicate code. Have get_interfaces_by_mac use
+      get_interfaces.
+    - tests: Fix build tree integration tests [Joshua Powers]
+    - sysconfig: Dont repeat header when rendering resolv.conf
+      [Ryan Harper] (LP: #1701420)
+    - archlinux: Fix bug with empty dns, do not render 'lo' devices.
+      (LP: #1663045, #1706593)
+    - cloudinit.net: add initialize_network_device function and tests
+      [Chad Smith]
+    - makefile: fix ci-deps-ubuntu target [Chad Smith]
+
+ -- Ryan Harper <[e-mail]>  Mon, 21 Aug 2017 15:09:36 -0500
+
 cloud-init (0.7.9-231-g80bf98b9-0ubuntu1) artful; urgency=medium
 
   * New upstream snapshot.
diff --git a/doc/rtd/index.rst b/doc/rtd/index.rst
index a691103..de67f36 100644
--- a/doc/rtd/index.rst
+++ b/doc/rtd/index.rst
@@ -40,6 +40,7 @@ initialization of a cloud instance.
    topics/merging.rst
    topics/network-config.rst
    topics/vendordata.rst
+   topics/debugging.rst
    topics/moreinfo.rst
    topics/hacking.rst
    topics/tests.rst
diff --git a/doc/rtd/topics/capabilities.rst b/doc/rtd/topics/capabilities.rst
index 2c8770b..b8034b0 100644
--- a/doc/rtd/topics/capabilities.rst
+++ b/doc/rtd/topics/capabilities.rst
@@ -31,19 +31,49 @@ support. This allows other applications to detect what features the installed
 cloud-init supports without having to parse its version number. If present,
 this list of features will be located at ``cloudinit.version.FEATURES``.
 
-When checking if cloud-init supports a feature, in order to not break the
-detection script on older versions of cloud-init without the features list, a
-script similar to the following should be used. Note that this will exit 0 if
-the feature is supported and 1 otherwise::
+Currently defined feature names include:
 
-    import sys
-    from cloudinit import version
-    sys.exit('<FEATURE_NAME>' not in getattr(version, 'FEATURES', []))
+ - ``NETWORK_CONFIG_V1`` support for v1 networking configuration,
+   see :ref:`network_config_v1` documentation for examples.
+ - ``NETWORK_CONFIG_V2`` support for v2 networking configuration,
+   see :ref:`network_config_v2` documentation for examples.
 
-Currently defined feature names include:
 
- - ``NETWORK_CONFIG_V1`` support for v1 networking configuration, see curtin
-   documentation for examples.
+CLI Interface :
+
+``cloud-init features`` will print out each feature supported.  If cloud-init
+does not have the features subcommand, it also does not support any features
+described in this document.
+
+.. code-block:: bash
+
+  % cloud-init --help
+  usage: cloud-init [-h] [--version] [--file FILES] [--debug] [--force]
+                    {init,modules,query,single,dhclient-hook,features} ...
+
+  positional arguments:
+    {init,modules,query,single,dhclient-hook,features}
+      init                initializes cloud-init and performs initial modules
+      modules             activates modules using a given configuration key
+      query               query information stored in cloud-init
+      single              run a single module
+      dhclient-hook       run the dhclient hookto record network info
+      features            list defined features
+
+  optional arguments:
+    -h, --help            show this help message and exit
+    --version, -v         show program's version number and exit
+    --file FILES, -f FILES
+                          additional yaml configuration files to use
+    --debug, -d           show additional pre-action logging (default: False)
+    --force               force running even if no datasource is found (use at
+                          your own risk)
+
+
+  % cloud-init features
+  NETWORK_CONFIG_V1
+  NETWORK_CONFIG_V2
+
 
 .. _Cloud-init: https://launchpad.net/cloud-init
 .. vi: textwidth=78
diff --git a/doc/rtd/topics/debugging.rst b/doc/rtd/topics/debugging.rst
new file mode 100644
index 0000000..4e43dd5
--- /dev/null
+++ b/doc/rtd/topics/debugging.rst
@@ -0,0 +1,146 @@
+**********************
+Testing and debugging cloud-init
+**********************
+
+Overview
+========
+This topic will discuss general approaches for test and debug of cloud-init on
+deployed instances.
+
+
+Boot Time Analysis - cloud-init analyze
+======================================
+Occasionally instances don't appear as performant as we would like and
+cloud-init packages a simple facility to inspect what operations took
+cloud-init the longest during boot and setup.
+
+The script **/usr/bin/cloud-init** has an analyze sub-command **analyze**
+which parses any cloud-init.log file into formatted and sorted events. It
+allows for detailed analysis of the most costly cloud-init operations are to
+determine the long-pole in cloud-init configuration and setup. These
+subcommands default to reading /var/log/cloud-init.log.
+
+* ``analyze show`` Parse and organize cloud-init.log events by stage and
+include each sub-stage granularity with time delta reports.
+
+.. code-block:: bash
+
+    $ cloud-init analyze show -i my-cloud-init.log
+    -- Boot Record 01 --
+    The total time elapsed since completing an event is printed after the "@"
+    character.
+    The time the event takes is printed after the "+" character.
+
+    Starting stage: modules-config
+    |`->config-emit_upstart ran successfully @05.47600s +00.00100s
+    |`->config-snap_config ran successfully @05.47700s +00.00100s
+    |`->config-ssh-import-id ran successfully @05.47800s +00.00200s
+    |`->config-locale ran successfully @05.48000s +00.00100s
+    ...
+
+
+* ``analyze dump`` Parse cloud-init.log into event records and return a list of
+dictionaries that can be consumed for other reporting needs.
+
+.. code-block:: bash
+
+    $ cloud-init analyze blame -i my-cloud-init.log
+    [
+     {
+      "description": "running config modules",
+      "event_type": "start",
+      "name": "modules-config",
+      "origin": "cloudinit",
+      "timestamp": 1510807493.0
+     },...
+
+* ``analyze blame`` Parse cloud-init.log into event records and sort them based
+on highest time cost for quick assessment of areas of cloud-init that may need
+improvement.
+
+.. code-block:: bash
+
+    $ cloud-init analyze blame -i my-cloud-init.log
+    -- Boot Record 11 --
+         00.01300s (modules-final/config-scripts-per-boot)
+         00.00400s (modules-final/config-final-message)
+         00.00100s (modules-final/config-rightscale_userdata)
+         ...
+
+
+Analyze quickstart - LXC
+---------------------------
+To quickly obtain a cloud-init log try using lxc on any ubuntu system:
+
+.. code-block:: bash
+
+  $ lxc init ubuntu-daily:xenial x1
+  $ lxc start x1
+  # Take lxc's cloud-init.log and pipe it to the analyzer
+  $ lxc file pull x1/var/log/cloud-init.log - | cloud-init analyze dump -i -
+  $ lxc file pull x1/var/log/cloud-init.log - | \
+  python3 -m cloudinit.analyze dump -i -
+
+Analyze quickstart - KVM
+---------------------------
+To quickly analyze a KVM a cloud-init log:
+
+1. Download the current cloud image
+  wget https://cloud-images.ubuntu.com/daily/server/xenial/current/xenial-server-cloudimg-amd64.img
+2. Create a snapshot image to preserve the original cloud-image
+
+.. code-block:: bash
+
+    $ qemu-img create -b xenial-server-cloudimg-amd64.img -f qcow2 \
+    test-cloudinit.qcow2
+
+3. Create a seed image with metadata using `cloud-localds`
+
+.. code-block:: bash
+
+    $ cat > user-data <<EOF
+      #cloud-config
+      password: passw0rd
+      chpasswd: { expire: False }
+      EOF
+    $  cloud-localds my-seed.img user-data
+
+4. Launch your modified VM
+
+.. code-block:: bash
+
+    $  kvm -m 512 -net nic -net user -redir tcp:2222::22 \
+   -drive file=test-cloudinit.qcow2,if=virtio,format=qcow2 \
+   -drive file=my-seed.img,if=virtio,format=raw
+
+5. Analyze the boot (blame, dump, show)
+
+.. code-block:: bash
+
+    $ ssh -p 2222 ubuntu@localhost 'cat /var/log/cloud-init.log' | \
+   cloud-init analyze blame -i -
+
+
+Running single cloud config modules
+===================================
+This subcommand is not called by the init system. It can be called manually to
+load the configured datasource and run a single cloud-config module once using
+the cached userdata and metadata after the instance has booted. Each
+cloud-config module has a module FREQUENCY configured: PER_INSTANCE, PER_BOOT,
+PER_ONCE or PER_ALWAYS. When a module is run by cloud-init, it stores a
+semaphore file in
+``/var/lib/cloud/instance/sem/config_<module_name>.<frequency>`` which marks
+when the module last successfully ran. Presence of this semaphore file
+prevents a module from running again if it has already been run. To ensure that
+a module is run again, the desired frequency can be overridden on the
+commandline:
+
+.. code-block:: bash
+
+  $ sudo cloud-init single --name cc_ssh --frequency always
+  ...
+  Generating public/private ed25519 key pair
+  ...
+
+Inspect cloud-init.log for output of what operations were performed as a
+result.
diff --git a/setup.py b/setup.py
index b1bde43..5c65c7f 100755
--- a/setup.py
+++ b/setup.py
@@ -240,7 +240,7 @@ setuptools.setup(
     author='Scott Moser',
     author_email='[e-mail]',
     url='http://launchpad.net/cloud-init/',
-    packages=setuptools.find_packages(exclude=['tests']),
+    packages=setuptools.find_packages(exclude=['tests.*', '*.tests', 'tests']),
     scripts=['tools/cloud-init-per'],
     license='Dual-licensed under GPLv3 or Apache 2.0',
     data_files=data_files,
diff --git a/templates/timesyncd.conf.tmpl b/templates/timesyncd.conf.tmpl
new file mode 100644
index 0000000..6b98301
--- /dev/null
+++ b/templates/timesyncd.conf.tmpl
@@ -0,0 +1,8 @@
+## template:jinja
+# cloud-init generated file
+# See timesyncd.conf(5) for details.
+
+[Time]
+{% if servers or pools -%}
+NTP={% for host in servers|list + pools|list %}{{ host }} {% endfor -%}
+{% endif -%}
diff --git a/tests/cloud_tests/bddeb.py b/tests/cloud_tests/bddeb.py
index 53dbf74..fe80535 100644
--- a/tests/cloud_tests/bddeb.py
+++ b/tests/cloud_tests/bddeb.py
@@ -11,7 +11,7 @@ from tests.cloud_tests import (config, LOG)
 from tests.cloud_tests import (platforms, images, snapshots, instances)
 from tests.cloud_tests.stage import (PlatformComponent, run_stage, run_single)
 
-build_deps = ['devscripts', 'equivs', 'git', 'tar']
+pre_reqs = ['devscripts', 'equivs', 'git', 'tar']
 
 
 def _out(cmd_res):
@@ -26,13 +26,10 @@ def build_deb(args, instance):
     @return_value: tuple of results and fail count
     """
     # update remote system package list and install build deps
-    LOG.debug('installing build deps')
-    pkgs = ' '.join(build_deps)
+    LOG.debug('installing pre-reqs')
+    pkgs = ' '.join(pre_reqs)
     cmd = 'apt-get update && apt-get install --yes {}'.format(pkgs)
     instance.execute(['/bin/sh', '-c', cmd])
-    # TODO Remove this call once we have a ci-deps Makefile target
-    instance.execute(['mk-build-deps', '--install', '-t',
-                      'apt-get --no-install-recommends --yes', 'cloud-init'])
 
     # local tmpfile that must be deleted
     local_tarball = tempfile.NamedTemporaryFile().name
@@ -40,7 +37,7 @@ def build_deb(args, instance):
     # paths to use in remote system
     output_link = '/root/cloud-init_all.deb'
     remote_tarball = _out(instance.execute(['mktemp']))
-    extract_dir = _out(instance.execute(['mktemp', '--directory']))
+    extract_dir = '/root'
     bddeb_path = os.path.join(extract_dir, 'packages', 'bddeb')
     git_env = {'GIT_DIR': os.path.join(extract_dir, '.git'),
                'GIT_WORK_TREE': extract_dir}
@@ -56,6 +53,11 @@ def build_deb(args, instance):
     instance.execute(['git', 'commit', '-a', '-m', 'tmp', '--allow-empty'],
                      env=git_env)
 
+    LOG.debug('installing deps')
+    deps_path = os.path.join(extract_dir, 'tools', 'read-dependencies')
+    instance.execute([deps_path, '--install', '--test-distro',
+                      '--distro', 'ubuntu', '--python-version', '3'])
+
     LOG.debug('building deb in remote system at: %s', output_link)
     bddeb_args = args.bddeb_args.split() if args.bddeb_args else []
     instance.execute([bddeb_path, '-d'] + bddeb_args, env=git_env)
diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py
index 08c5c46..bf1dc5d 100644
--- a/tests/unittests/helpers.py
+++ b/tests/unittests/helpers.py
@@ -278,7 +278,7 @@ class FilesystemMockingTestCase(ResourceUsingTestCase):
         return root
 
 
-class HttprettyTestCase(TestCase):
+class HttprettyTestCase(CiTestCase):
     # necessary as http_proxy gets in the way of httpretty
     # https://github.com/gabrielfalcao/HTTPretty/issues/122
     def setUp(self):
diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py
index 06f366b..7780f16 100644
--- a/tests/unittests/test_cli.py
+++ b/tests/unittests/test_cli.py
@@ -31,9 +31,90 @@ class TestCLI(test_helpers.FilesystemMockingTestCase):
 
     def test_no_arguments_shows_error_message(self):
         exit_code = self._call_main()
-        self.assertIn('cloud-init: error: too few arguments',
-                      self.stderr.getvalue())
+        missing_subcommand_message = [
+            'too few arguments',  # python2.7 msg
+            'the following arguments are required: subcommand'  # python3 msg
+        ]
+        error = self.stderr.getvalue()
+        matches = ([msg in error for msg in missing_subcommand_message])
+        self.assertTrue(
+            any(matches), 'Did not find error message for missing subcommand')
         self.assertEqual(2, exit_code)
 
+    def test_all_subcommands_represented_in_help(self):
+        """All known subparsers are represented in the cloud-int help doc."""
+        self._call_main()
+        error = self.stderr.getvalue()
+        expected_subcommands = ['analyze', 'init', 'modules', 'single',
+                                'dhclient-hook', 'features']
+        for subcommand in expected_subcommands:
+            self.assertIn(subcommand, error)
 
-# vi: ts=4 expandtab
+    @mock.patch('cloudinit.cmd.main.status_wrapper')
+    def test_init_subcommand_parser(self, m_status_wrapper):
+        """The subcommand 'init' calls status_wrapper passing init."""
+        self._call_main(['cloud-init', 'init'])
+        (name, parseargs) = m_status_wrapper.call_args_list[0][0]
+        self.assertEqual('init', name)
+        self.assertEqual('init', parseargs.subcommand)
+        self.assertEqual('init', parseargs.action[0])
+        self.assertEqual('main_init', parseargs.action[1].__name__)
+
+    @mock.patch('cloudinit.cmd.main.status_wrapper')
+    def test_modules_subcommand_parser(self, m_status_wrapper):
+        """The subcommand 'modules' calls status_wrapper passing modules."""
+        self._call_main(['cloud-init', 'modules'])
+        (name, parseargs) = m_status_wrapper.call_args_list[0][0]
+        self.assertEqual('modules', name)
+        self.assertEqual('modules', parseargs.subcommand)
+        self.assertEqual('modules', parseargs.action[0])
+        self.assertEqual('main_modules', parseargs.action[1].__name__)
+
+    def test_analyze_subcommand_parser(self):
+        """The subcommand cloud-init analyze calls the correct subparser."""
+        self._call_main(['cloud-init', 'analyze'])
+        # These subcommands only valid for cloud-init analyze script
+        expected_subcommands = ['blame', 'show', 'dump']
+        error = self.stderr.getvalue()
+        for subcommand in expected_subcommands:
+            self.assertIn(subcommand, error)
+
+    @mock.patch('cloudinit.cmd.main.main_single')
+    def test_single_subcommand(self, m_main_single):
+        """The subcommand 'single' calls main_single with valid args."""
+        self._call_main(['cloud-init', 'single', '--name', 'cc_ntp'])
+        (name, parseargs) = m_main_single.call_args_list[0][0]
+        self.assertEqual('single', name)
+        self.assertEqual('single', parseargs.subcommand)
+        self.assertEqual('single', parseargs.action[0])
+        self.assertFalse(parseargs.debug)
+        self.assertFalse(parseargs.force)
+        self.assertIsNone(parseargs.frequency)
+        self.assertEqual('cc_ntp', parseargs.name)
+        self.assertFalse(parseargs.report)
+
+    @mock.patch('cloudinit.cmd.main.dhclient_hook')
+    def test_dhclient_hook_subcommand(self, m_dhclient_hook):
+        """The subcommand 'dhclient-hook' calls dhclient_hook with args."""
+        self._call_main(['cloud-init', 'dhclient-hook', 'net_action', 'eth0'])
+        (name, parseargs) = m_dhclient_hook.call_args_list[0][0]
+        self.assertEqual('dhclient_hook', name)
+        self.assertEqual('dhclient-hook', parseargs.subcommand)
+        self.assertEqual('dhclient_hook', parseargs.action[0])
+        self.assertFalse(parseargs.debug)
+        self.assertFalse(parseargs.force)
+        self.assertEqual('net_action', parseargs.net_action)
+        self.assertEqual('eth0', parseargs.net_interface)
+
+    @mock.patch('cloudinit.cmd.main.main_features')
+    def test_features_hook_subcommand(self, m_features):
+        """The subcommand 'features' calls main_features with args."""
+        self._call_main(['cloud-init', 'features'])
+        (name, parseargs) = m_features.call_args_list[0][0]
+        self.assertEqual('features', name)
+        self.assertEqual('features', parseargs.subcommand)
+        self.assertEqual('features', parseargs.action[0])
+        self.assertFalse(parseargs.debug)
+        self.assertFalse(parseargs.force)
+
+# : ts=4 expandtab
diff --git a/tests/unittests/test_datasource/test_aliyun.py b/tests/unittests/test_datasource/test_aliyun.py
index 990bff2..996560e 100644
--- a/tests/unittests/test_datasource/test_aliyun.py
+++ b/tests/unittests/test_datasource/test_aliyun.py
@@ -70,7 +70,6 @@ class TestAliYunDatasource(test_helpers.HttprettyTestCase):
         paths = helpers.Paths({})
         self.ds = ay.DataSourceAliYun(cfg, distro, paths)
         self.metadata_address = self.ds.metadata_urls[0]
-        self.api_ver = self.ds.api_ver
 
     @property
     def default_metadata(self):
@@ -82,13 +81,15 @@ class TestAliYunDatasource(test_helpers.HttprettyTestCase):
 
     @property
     def metadata_url(self):
-        return os.path.join(self.metadata_address,
-                            self.api_ver, 'meta-data') + '/'
+        return os.path.join(
+            self.metadata_address,
+            self.ds.min_metadata_version, 'meta-data') + '/'
 
     @property
     def userdata_url(self):
-        return os.path.join(self.metadata_address,
-                            self.api_ver, 'user-data')
+        return os.path.join(
+            self.metadata_address,
+            self.ds.min_metadata_version, 'user-data')
 
     def regist_default_server(self):
         register_mock_metaserver(self.metadata_url, self.default_metadata)
diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py
index 413e87a..4802f10 100644
--- a/tests/unittests/test_datasource/test_common.py
+++ b/tests/unittests/test_datasource/test_common.py
@@ -35,6 +35,7 @@ DEFAULT_LOCAL = [
     OpenNebula.DataSourceOpenNebula,
     OVF.DataSourceOVF,
     SmartOS.DataSourceSmartOS,
+    Ec2.DataSourceEc2Local,
 ]
 
 DEFAULT_NETWORK = [
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index 12230ae..33d0261 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -8,35 +8,67 @@ from cloudinit import helpers
 from cloudinit.sources import DataSourceEc2 as ec2
 
 
-# collected from api version 2009-04-04/ with
+# collected from api version 2016-09-02/ with
 # python3 -c 'import json
 # from cloudinit.ec2_utils import get_instance_metadata as gm
-# print(json.dumps(gm("2009-04-04"), indent=1, sort_keys=True))'
+# print(json.dumps(gm("2016-09-02"), indent=1, sort_keys=True))'
 DEFAULT_METADATA = {
-    "ami-id": "ami-80861296",
+    "ami-id": "ami-8b92b4ee",
     "ami-launch-index": "0",
     "ami-manifest-path": "(unknown)",
     "block-device-mapping": {"ami": "/dev/sda1", "root": "/dev/sda1"},
-    "hostname": "ip-10-0-0-149",
+    "hostname": "ip-172-31-31-158.us-east-2.compute.internal",
     "instance-action": "none",
-    "instance-id": "i-0052913950685138c",
-    "instance-type": "t2.micro",
-    "local-hostname": "ip-10-0-0-149",
-    "local-ipv4": "10.0.0.149",
-    "placement": {"availability-zone": "us-east-1b"},
+    "instance-id": "i-0a33f80f09c96477f",
+    "instance-type": "t2.small",
+    "local-hostname": "ip-172-3-3-15.us-east-2.compute.internal",
+    "local-ipv4": "172.3.3.15",
+    "mac": "06:17:04:d7:26:09",
+    "metrics": {"vhostmd": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"},
+    "network": {
+        "interfaces": {
+            "macs": {
+                "06:17:04:d7:26:09": {
+                    "device-number": "0",
+                    "interface-id": "eni-e44ef49e",
+                    "ipv4-associations": {"13.59.77.202": "172.3.3.15"},
+                    "ipv6s": "2600:1f16:aeb:b20b:9d87:a4af:5cc9:73dc",
+                    "local-hostname": ("ip-172-3-3-15.us-east-2."
+                                       "compute.internal"),
+                    "local-ipv4s": "172.3.3.15",
+                    "mac": "06:17:04:d7:26:09",
+                    "owner-id": "950047163771",
+                    "public-hostname": ("ec2-13-59-77-202.us-east-2."
+                                        "compute.amazonaws.com"),
+                    "public-ipv4s": "13.59.77.202",
+                    "security-group-ids": "sg-5a61d333",
+                    "security-groups": "wide-open",
+                    "subnet-id": "subnet-20b8565b",
+                    "subnet-ipv4-cidr-block": "172.31.16.0/20",
+                    "subnet-ipv6-cidr-blocks": "2600:1f16:aeb:b20b::/64",
+                    "vpc-id": "vpc-87e72bee",
+                    "vpc-ipv4-cidr-block": "172.31.0.0/16",
+                    "vpc-ipv4-cidr-blocks": "172.31.0.0/16",
+                    "vpc-ipv6-cidr-blocks": "2600:1f16:aeb:b200::/56"
+                }
+            }
+        }
+    },
+    "placement": {"availability-zone": "us-east-2b"},
     "profile": "default-hvm",
-    "public-hostname": "",
-    "public-ipv4": "107.23.188.247",
+    "public-hostname": "ec2-13-59-77-202.us-east-2.compute.amazonaws.com",
+    "public-ipv4": "13.59.77.202",
     "public-keys": {"brickies": ["ssh-rsa AAAAB3Nz....w== brickies"]},
-    "reservation-id": "r-00a2c173fb5782a08",
-    "security-groups": "wide-open"
+    "reservation-id": "r-01efbc9996bac1bd6",
+    "security-groups": "my-wide-open",
+    "services": {"domain": "amazonaws.com", "partition": "aws"}
 }
 
 
 def _register_ssh_keys(rfunc, base_url, keys_data):
     """handle ssh key inconsistencies.
 
-    public-keys in the ec2 metadata is inconsistently formatted compared
+    public-keys in the ec2 metadata is inconsistently formated compared
     to other entries.
     Given keys_data of {name1: pubkey1, name2: pubkey2}
 
@@ -115,6 +147,8 @@ def register_mock_metaserver(base_url, data):
 
 
 class TestEc2(test_helpers.HttprettyTestCase):
+    with_logs = True
+
     valid_platform_data = {
         'uuid': 'ec212f79-87d1-2f1d-588f-d86dc0fd5412',
         'uuid_source': 'dmi',
@@ -123,16 +157,20 @@ class TestEc2(test_helpers.HttprettyTestCase):
 
     def setUp(self):
         super(TestEc2, self).setUp()
-        self.metadata_addr = ec2.DataSourceEc2.metadata_urls[0]
-        self.api_ver = '2009-04-04'
+        self.datasource = ec2.DataSourceEc2
+        self.metadata_addr = self.datasource.metadata_urls[0]
 
     @property
     def metadata_url(self):
-        return '/'.join([self.metadata_addr, self.api_ver, 'meta-data', ''])
+        return '/'.join([
+            self.metadata_addr,
+            self.datasource.min_metadata_version, 'meta-data', ''])
 
     @property
     def userdata_url(self):
-        return '/'.join([self.metadata_addr, self.api_ver, 'user-data'])
+        return '/'.join([
+            self.metadata_addr,
+            self.datasource.min_metadata_version, 'user-data'])
 
     def _patch_add_cleanup(self, mpath, *args, **kwargs):
         p = mock.patch(mpath, *args, **kwargs)
@@ -144,7 +182,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
         paths = helpers.Paths({})
         if sys_cfg is None:
             sys_cfg = {}
-        ds = ec2.DataSourceEc2(sys_cfg=sys_cfg, distro=distro, paths=paths)
+        ds = self.datasource(sys_cfg=sys_cfg, distro=distro, paths=paths)
         if platform_data is not None:
             self._patch_add_cleanup(
                 "cloudinit.sources.DataSourceEc2._collect_platform_data",
@@ -157,14 +195,16 @@ class TestEc2(test_helpers.HttprettyTestCase):
         return ds
 
     @httpretty.activate
-    def test_valid_platform_with_strict_true(self):
+    @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
+    def test_valid_platform_with_strict_true(self, m_dhcp):
         """Valid platform data should return true with strict_id true."""
         ds = self._setup_ds(
             platform_data=self.valid_platform_data,
             sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
             md=DEFAULT_METADATA)
         ret = ds.get_data()
-        self.assertEqual(True, ret)
+        self.assertTrue(ret)
+        self.assertEqual(0, m_dhcp.call_count)
 
     @httpretty.activate
     def test_valid_platform_with_strict_false(self):
@@ -174,7 +214,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
             sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
             md=DEFAULT_METADATA)
         ret = ds.get_data()
-        self.assertEqual(True, ret)
+        self.assertTrue(ret)
 
     @httpretty.activate
     def test_unknown_platform_with_strict_true(self):
@@ -185,7 +225,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
             sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
             md=DEFAULT_METADATA)
         ret = ds.get_data()
-        self.assertEqual(False, ret)
+        self.assertFalse(ret)
 
     @httpretty.activate
     def test_unknown_platform_with_strict_false(self):
@@ -196,7 +236,55 @@ class TestEc2(test_helpers.HttprettyTestCase):
             sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
             md=DEFAULT_METADATA)
         ret = ds.get_data()
-        self.assertEqual(True, ret)
+        self.assertTrue(ret)
+
+    @httpretty.activate
+    @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD')
+    def test_ec2_local_returns_false_on_bsd(self, m_is_freebsd):
+        """DataSourceEc2Local returns False on BSD.
+
+        FreeBSD dhclient doesn't support dhclient -sf to run in a sandbox.
+        """
+        m_is_freebsd.return_value = True
+        self.datasource = ec2.DataSourceEc2Local
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+            md=DEFAULT_METADATA)
+        ret = ds.get_data()
+        self.assertFalse(ret)
+        self.assertIn(
+            "FreeBSD doesn't support running dhclient with -sf",
+            self.logs.getvalue())
+
+    @httpretty.activate
+    @mock.patch('cloudinit.net.EphemeralIPv4Network')
+    @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
+    @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD')
+    def test_ec2_local_performs_dhcp_on_non_bsd(self, m_is_bsd, m_dhcp, m_net):
+        """Ec2Local returns True for valid platform data on non-BSD with dhcp.
+
+        DataSourceEc2Local will setup initial IPv4 network via dhcp discovery.
+        Then the metadata services is crawled for more network config info.
+        When the platform data is valid, return True.
+        """
+        m_is_bsd.return_value = False
+        m_dhcp.return_value = [{
+            'interface': 'eth9', 'fixed-address': '192.168.2.9',
+            'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0',
+            'broadcast-address': '192.168.2.255'}]
+        self.datasource = ec2.DataSourceEc2Local
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+            md=DEFAULT_METADATA)
+        ret = ds.get_data()
+        self.assertTrue(ret)
+        m_dhcp.assert_called_once_with()
+        m_net.assert_called_once_with(
+            broadcast='192.168.2.255', interface='eth9', ip='192.168.2.9',
+            prefix_or_mask='255.255.255.0', router='192.168.2.1')
+        self.assertIn('Crawl of metadata service took', self.logs.getvalue())
 
 
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_distros/__init__.py b/tests/unittests/test_distros/__init__.py
index e69de29..5394aa5 100644
--- a/tests/unittests/test_distros/__init__.py
+++ b/tests/unittests/test_distros/__init__.py
@@ -0,0 +1,21 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+import copy
+
+from cloudinit import distros
+from cloudinit import helpers
+from cloudinit import settings
+
+
+def _get_distro(dtype, system_info=None):
+    """Return a Distro class of distro 'dtype'.
+
+    cfg is format of CFG_BUILTIN['system_info'].
+
+    example: _get_distro("debian")
+    """
+    if system_info is None:
+        system_info = copy.deepcopy(settings.CFG_BUILTIN['system_info'])
+    system_info['distro'] = dtype
+    paths = helpers.Paths(system_info['paths'])
+    distro_cls = distros.fetch(dtype)
+    return distro_cls(dtype, system_info, paths)
diff --git a/tests/unittests/test_distros/test_arch.py b/tests/unittests/test_distros/test_arch.py
new file mode 100644
index 0000000..3d4c9a7
--- /dev/null
+++ b/tests/unittests/test_distros/test_arch.py
@@ -0,0 +1,45 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from cloudinit.distros.arch import _render_network
+from cloudinit import util
+
+from ..helpers import (CiTestCase, dir2dict)
+
+from . import _get_distro
+
+
+class TestArch(CiTestCase):
+
+    def test_get_distro(self):
+        distro = _get_distro("arch")
+        hostname = "myhostname"
+        hostfile = self.tmp_path("hostfile")
+        distro._write_hostname(hostname, hostfile)
+        self.assertEqual(hostname + "\n", util.load_file(hostfile))
+
+
+class TestRenderNetwork(CiTestCase):
+    def test_basic_static(self):
+        """Just the most basic static config.
+
+        note 'lo' should not be rendered as an interface."""
+        entries = {'eth0': {'auto': True,
+                            'dns-nameservers': ['8.8.8.8'],
+                            'bootproto': 'static',
+                            'address': '10.0.0.2',
+                            'gateway': '10.0.0.1',
+                            'netmask': '255.255.255.0'},
+                   'lo': {'auto': True}}
+        target = self.tmp_dir()
+        devs = _render_network(entries, target=target)
+        files = dir2dict(target, prefix=target)
+        self.assertEqual(['eth0'], devs)
+        self.assertEqual(
+            {'/etc/netctl/eth0': '\n'.join([
+                "Address=10.0.0.2/255.255.255.0",
+                "Connection=ethernet",
+                "DNS=('8.8.8.8')",
+                "Gateway=10.0.0.1",
+                "IP=static",
+                "Interface=eth0", ""]),
+             '/etc/resolv.conf': 'nameserver 8.8.8.8\n'}, files)
diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py
index 2f505d9..6d89dba 100644
--- a/tests/unittests/test_distros/test_netconfig.py
+++ b/tests/unittests/test_distros/test_netconfig.py
@@ -135,7 +135,7 @@ network:
 V2_NET_CFG = {
     'ethernets': {
         'eth7': {
-            'addresses': ['192.168.1.5/255.255.255.0'],
+            'addresses': ['192.168.1.5/24'],
             'gateway4': '192.168.1.254'},
         'eth9': {
             'dhcp4': True}
@@ -151,7 +151,6 @@ V2_TO_V2_NET_CFG_OUTPUT = """
 # /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following:
 # network: {config: disabled}
 network:
-    version: 2
     ethernets:
         eth7:
             addresses:
@@ -159,6 +158,7 @@ network:
             gateway4: 192.168.1.254
         eth9:
             dhcp4: true
+    version: 2
 """
 
 
diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py
index 7f27864..83d5faa 100644
--- a/tests/unittests/test_handler/test_handler_ntp.py
+++ b/tests/unittests/test_handler/test_handler_ntp.py
@@ -16,6 +16,14 @@ servers {{servers}}
 pools {{pools}}
 """
 
+TIMESYNCD_TEMPLATE = b"""\
+## template:jinja
+[Time]
+{% if servers or pools -%}
+NTP={% for host in servers|list + pools|list %}{{ host }} {% endfor -%}
+{% endif -%}
+"""
+
 try:
     import jsonschema
     assert jsonschema  # avoid pyflakes error F401: import unused
@@ -59,6 +67,14 @@ class TestNtp(FilesystemMockingTestCase):
         cc_ntp.install_ntp(install_func, packages=['ntp'], check_exe='ntpd')
         install_func.assert_not_called()
 
+    @mock.patch("cloudinit.config.cc_ntp.util")
+    def test_ntp_install_no_op_with_empty_pkg_list(self, mock_util):
+        """ntp_install calls install_func with empty list"""
+        mock_util.which.return_value = None  # check_exe not found
+        install_func = mock.MagicMock()
+        cc_ntp.install_ntp(install_func, packages=[], check_exe='timesyncd')
+        install_func.assert_called_once_with([])
+
     def test_ntp_rename_ntp_conf(self):
         """When NTP_CONF exists, rename_ntp moves it."""
         ntpconf = self.tmp_path("ntp.conf", self.new_root)
@@ -68,6 +84,30 @@ class TestNtp(FilesystemMockingTestCase):
         self.assertFalse(os.path.exists(ntpconf))
         self.assertTrue(os.path.exists("{0}.dist".format(ntpconf)))
 
+    @mock.patch("cloudinit.config.cc_ntp.util")
+    def test_reload_ntp_defaults(self, mock_util):
+        """Test service is restarted/reloaded (defaults)"""
+        service = 'ntp'
+        cmd = ['service', service, 'restart']
+        cc_ntp.reload_ntp(service)
+        mock_util.subp.assert_called_with(cmd, capture=True)
+
+    @mock.patch("cloudinit.config.cc_ntp.util")
+    def test_reload_ntp_systemd(self, mock_util):
+        """Test service is restarted/reloaded (systemd)"""
+        service = 'ntp'
+        cmd = ['systemctl', 'reload-or-restart', service]
+        cc_ntp.reload_ntp(service, systemd=True)
+        mock_util.subp.assert_called_with(cmd, capture=True)
+
+    @mock.patch("cloudinit.config.cc_ntp.util")
+    def test_reload_ntp_systemd_timesycnd(self, mock_util):
+        """Test service is restarted/reloaded (systemd/timesyncd)"""
+        service = 'systemd-timesycnd'
+        cmd = ['systemctl', 'reload-or-restart', service]
+        cc_ntp.reload_ntp(service, systemd=True)
+        mock_util.subp.assert_called_with(cmd, capture=True)
+
     def test_ntp_rename_ntp_conf_skip_missing(self):
         """When NTP_CONF doesn't exist rename_ntp doesn't create a file."""
         ntpconf = self.tmp_path("ntp.conf", self.new_root)
@@ -94,7 +134,7 @@ class TestNtp(FilesystemMockingTestCase):
         with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream:
             stream.write(NTP_TEMPLATE)
         with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf):
-            cc_ntp.write_ntp_config_template(cfg, mycloud)
+            cc_ntp.write_ntp_config_template(cfg, mycloud, ntp_conf)
         content = util.read_file_or_url('file://' + ntp_conf).contents
         self.assertEqual(
             "servers ['192.168.2.1', '192.168.2.2']\npools []\n",
@@ -120,7 +160,7 @@ class TestNtp(FilesystemMockingTestCase):
         with open('{0}.{1}.tmpl'.format(ntp_conf, distro), 'wb') as stream:
             stream.write(NTP_TEMPLATE)
         with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf):
-            cc_ntp.write_ntp_config_template(cfg, mycloud)
+            cc_ntp.write_ntp_config_template(cfg, mycloud, ntp_conf)
         content = util.read_file_or_url('file://' + ntp_conf).contents
         self.assertEqual(
             "servers []\npools ['10.0.0.1', '10.0.0.2']\n",
@@ -139,7 +179,7 @@ class TestNtp(FilesystemMockingTestCase):
         with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream:
             stream.write(NTP_TEMPLATE)
         with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf):
-            cc_ntp.write_ntp_config_template({}, mycloud)
+            cc_ntp.write_ntp_config_template({}, mycloud, ntp_conf)
         content = util.read_file_or_url('file://' + ntp_conf).contents
         default_pools = [
             "{0}.{1}.pool.ntp.org".format(x, distro)
@@ -152,7 +192,8 @@ class TestNtp(FilesystemMockingTestCase):
                 ",".join(default_pools)),
             self.logs.getvalue())
 
-    def test_ntp_handler_mocked_template(self):
+    @mock.patch("cloudinit.config.cc_ntp.ntp_installable")
+    def test_ntp_handler_mocked_template(self, m_ntp_install):
         """Test ntp handler renders ubuntu ntp.conf template."""
         pools = ['0.mycompany.pool.ntp.org', '3.mycompany.pool.ntp.org']
         servers = ['192.168.23.3', '192.168.23.4']
@@ -164,6 +205,8 @@ class TestNtp(FilesystemMockingTestCase):
         }
         mycloud = self._get_cloud('ubuntu')
         ntp_conf = self.tmp_path('ntp.conf', self.new_root)  # Doesn't exist
+        m_ntp_install.return_value = True
+
         # Create ntp.conf.tmpl
         with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream:
             stream.write(NTP_TEMPLATE)
@@ -176,6 +219,34 @@ class TestNtp(FilesystemMockingTestCase):
             'servers {0}\npools {1}\n'.format(servers, pools),
             content.decode())
 
+    @mock.patch("cloudinit.config.cc_ntp.util")
+    def test_ntp_handler_mocked_template_snappy(self, m_util):
+        """Test ntp handler renders timesycnd.conf template on snappy."""
+        pools = ['0.mycompany.pool.ntp.org', '3.mycompany.pool.ntp.org']
+        servers = ['192.168.23.3', '192.168.23.4']
+        cfg = {
+            'ntp': {
+                'pools': pools,
+                'servers': servers
+            }
+        }
+        mycloud = self._get_cloud('ubuntu')
+        m_util.system_is_snappy.return_value = True
+
+        # Create timesyncd.conf.tmpl
+        tsyncd_conf = self.tmp_path("timesyncd.conf", self.new_root)
+        template = '{0}.tmpl'.format(tsyncd_conf)
+        with open(template, 'wb') as stream:
+            stream.write(TIMESYNCD_TEMPLATE)
+
+        with mock.patch('cloudinit.config.cc_ntp.TIMESYNCD_CONF', tsyncd_conf):
+            cc_ntp.handle('notimportant', cfg, mycloud, None, None)
+
+        content = util.read_file_or_url('file://' + tsyncd_conf).contents
+        self.assertEqual(
+            "[Time]\nNTP=%s %s \n" % (" ".join(servers), " ".join(pools)),
+            content.decode())
+
     def test_ntp_handler_real_distro_templates(self):
         """Test ntp handler renders the shipped distro ntp.conf templates."""
         pools = ['0.mycompany.pool.ntp.org', '3.mycompany.pool.ntp.org']
@@ -333,4 +404,30 @@ class TestNtp(FilesystemMockingTestCase):
             "pools ['0.mypool.org', '0.mypool.org']\n",
             content)
 
+    @mock.patch("cloudinit.config.cc_ntp.ntp_installable")
+    def test_ntp_handler_timesyncd(self, m_ntp_install):
+        """Test ntp handler configures timesyncd"""
+        m_ntp_install.return_value = False
+        distro = 'ubuntu'
+        cfg = {
+            'servers': ['192.168.2.1', '192.168.2.2'],
+            'pools': ['0.mypool.org'],
+        }
+        mycloud = self._get_cloud(distro)
+        tsyncd_conf = self.tmp_path("timesyncd.conf", self.new_root)
+        # Create timesyncd.conf.tmpl
+        template = '{0}.tmpl'.format(tsyncd_conf)
+        print(template)
+        with open(template, 'wb') as stream:
+            stream.write(TIMESYNCD_TEMPLATE)
+        with mock.patch('cloudinit.config.cc_ntp.TIMESYNCD_CONF', tsyncd_conf):
+            cc_ntp.write_ntp_config_template(cfg, mycloud, tsyncd_conf,
+                                             template='timesyncd.conf')
+
+        content = util.read_file_or_url('file://' + tsyncd_conf).contents
+        self.assertEqual(
+            "[Time]\nNTP=192.168.2.1 192.168.2.2 0.mypool.org \n",
+            content.decode())
+
+
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
index e49abcc..f251024 100644
--- a/tests/unittests/test_net.py
+++ b/tests/unittests/test_net.py
@@ -1059,6 +1059,100 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true
                   - type: static
                     address: 2001:1::1/92
             """),
+        'expected_netplan': textwrap.dedent("""
+         network:
+             version: 2
+             ethernets:
+                 bond0s0:
+                     match:
+                         macaddress: aa:bb:cc:dd:e8:00
+                     set-name: bond0s0
+                 bond0s1:
+                     match:
+                         macaddress: aa:bb:cc:dd:e8:01
+                     set-name: bond0s1
+             bonds:
+                 bond0:
+                     addresses:
+                     - 192.168.0.2/24
+                     - 192.168.1.2/24
+                     - 2001:1::1/92
+                     gateway4: 192.168.0.1
+                     interfaces:
+                     - bond0s0
+                     - bond0s1
+                     parameters:
+                         mii-monitor-interval: 100
+                         mode: active-backup
+                         transmit-hash-policy: layer3+4
+                     routes:
+                     -   to: 10.1.3.0/24
+                         via: 192.168.0.3
+        """),
+        'yaml-v2': textwrap.dedent("""
+            version: 2
+            ethernets:
+              eth0:
+                match:
+                    driver: "virtio_net"
+                    macaddress: "aa:bb:cc:dd:e8:00"
+              vf0:
+                set-name: vf0
+                match:
+                    driver: "e1000"
+                    macaddress: "aa:bb:cc:dd:e8:01"
+            bonds:
+              bond0:
+                addresses:
+                - 192.168.0.2/24
+                - 192.168.1.2/24
+                - 2001:1::1/92
+                gateway4: 192.168.0.1
+                interfaces:
+                - eth0
+                - vf0
+                parameters:
+                    mii-monitor-interval: 100
+                    mode: active-backup
+                    primary: vf0
+                    transmit-hash-policy: "layer3+4"
+                routes:
+                -   to: 10.1.3.0/24
+                    via: 192.168.0.3
+            """),
+        'expected_netplan-v2': textwrap.dedent("""
+         network:
+             bonds:
+                 bond0:
+                     addresses:
+                     - 192.168.0.2/24
+                     - 192.168.1.2/24
+                     - 2001:1::1/92
+                     gateway4: 192.168.0.1
+                     interfaces:
+                     - eth0
+                     - vf0
+                     parameters:
+                         mii-monitor-interval: 100
+                         mode: active-backup
+                         primary: vf0
+                         transmit-hash-policy: layer3+4
+                     routes:
+                     -   to: 10.1.3.0/24
+                         via: 192.168.0.3
+             ethernets:
+                 eth0:
+                     match:
+                         driver: virtio_net
+                         macaddress: aa:bb:cc:dd:e8:00
+                 vf0:
+                     match:
+                         driver: e1000
+                         macaddress: aa:bb:cc:dd:e8:01
+                     set-name: vf0
+             version: 2
+        """),
+
         'expected_sysconfig': {
             'ifcfg-bond0': textwrap.dedent("""\
         BONDING_MASTER=yes
@@ -1683,6 +1777,9 @@ USERCTL=no
             ns = network_state.parse_net_config_data(network_cfg,
                                                      skip_broken=False)
             renderer = sysconfig.Renderer()
+            # render a multiple times to simulate reboots
+            renderer.render_network_state(ns, render_dir)
+            renderer.render_network_state(ns, render_dir)
             renderer.render_network_state(ns, render_dir)
             for fn, expected_content in os_sample.get('out_sysconfig', []):
                 with open(os.path.join(render_dir, fn)) as fh:
@@ -2156,6 +2253,27 @@ class TestNetplanRoundTrip(CiTestCase):
         renderer.render_network_state(ns, target)
         return dir2dict(target)
 
+    def testsimple_render_bond_netplan(self):
+        entry = NETWORK_CONFIGS['bond']
+        files = self._render_and_read(network_config=yaml.load(entry['yaml']))
+        print(entry['expected_netplan'])
+        print('-- expected ^ | v rendered --')
+        print(files['/etc/netplan/50-cloud-init.yaml'])
+        self.assertEqual(
+            entry['expected_netplan'].splitlines(),
+            files['/etc/netplan/50-cloud-init.yaml'].splitlines())
+
+    def testsimple_render_bond_v2_input_netplan(self):
+        entry = NETWORK_CONFIGS['bond']
+        files = self._render_and_read(
+            network_config=yaml.load(entry['yaml-v2']))
+        print(entry['expected_netplan-v2'])
+        print('-- expected ^ | v rendered --')
+        print(files['/etc/netplan/50-cloud-init.yaml'])
+        self.assertEqual(
+            entry['expected_netplan-v2'].splitlines(),
+            files['/etc/netplan/50-cloud-init.yaml'].splitlines())
+
     def testsimple_render_small_netplan(self):
         entry = NETWORK_CONFIGS['small']
         files = self._render_and_read(network_config=yaml.load(entry['yaml']))
diff --git a/tests/unittests/test_vmware_config_file.py b/tests/unittests/test_vmware_config_file.py
index 18475f1..03b36d3 100644
--- a/tests/unittests/test_vmware_config_file.py
+++ b/tests/unittests/test_vmware_config_file.py
@@ -7,8 +7,8 @@
 
 import logging
 import sys
-import unittest
 
+from .helpers import CiTestCase
 from cloudinit.sources.helpers.vmware.imc.boot_proto import BootProtoEnum
 from cloudinit.sources.helpers.vmware.imc.config import Config
 from cloudinit.sources.helpers.vmware.imc.config_file import ConfigFile
@@ -17,7 +17,7 @@ logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
 logger = logging.getLogger(__name__)
 
 
-class TestVmwareConfigFile(unittest.TestCase):
+class TestVmwareConfigFile(CiTestCase):
 
     def test_utility_methods(self):
         cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg")
@@ -90,4 +90,32 @@ class TestVmwareConfigFile(unittest.TestCase):
         self.assertEqual('00:50:56:a6:8c:08', nics[0].mac, "mac0")
         self.assertEqual(BootProtoEnum.DHCP, nics[0].bootProto, "bootproto0")
 
+    def test_config_password(self):
+        cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg")
+
+        cf._insertKey("PASSWORD|-PASS", "test-password")
+        cf._insertKey("PASSWORD|RESET", "no")
+
+        conf = Config(cf)
+        self.assertEqual('test-password', conf.admin_password, "password")
+        self.assertFalse(conf.reset_password, "do not reset password")
+
+    def test_config_reset_passwd(self):
+        cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg")
+
+        cf._insertKey("PASSWORD|-PASS", "test-password")
+        cf._insertKey("PASSWORD|RESET", "random")
+
+        conf = Config(cf)
+        with self.assertRaises(ValueError):
+            conf.reset_password()
+
+        cf.clear()
+        cf._insertKey("PASSWORD|RESET", "yes")
+        self.assertEqual(1, len(cf), "insert size")
+
+        conf = Config(cf)
+        self.assertTrue(conf.reset_password, "reset password")
+
+
 # vi: ts=4 expandtab
diff --git a/tox.ini b/tox.ini
index 1140f9b..1e7ca2d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -21,7 +21,11 @@ setenv =
     LC_ALL = en_US.utf-8
 
 [testenv:pylint]
-deps = pylint==1.7.1
+deps =
+    # requirements
+    pylint==1.7.1
+    # test-requirements because unit tests are now present in cloudinit tree
+    -r{toxinidir}/test-requirements.txt
 commands = {envpython} -m pylint {posargs:cloudinit}
 
 [testenv:py3]
@@ -29,7 +33,7 @@ basepython = python3
 deps = -r{toxinidir}/test-requirements.txt
 commands = {envpython} -m nose {posargs:--with-coverage \
            --cover-erase --cover-branches --cover-inclusive \
-           --cover-package=cloudinit tests/unittests}
+           --cover-package=cloudinit tests/unittests cloudinit}
 
 [testenv:py27]
 basepython = python2.7
@@ -98,7 +102,11 @@ deps = pyflakes
 
 [testenv:tip-pylint]
 commands = {envpython} -m pylint {posargs:cloudinit}
-deps = pylint
+deps =
+    # requirements
+    pylint
+    # test-requirements
+    -r{toxinidir}/test-requirements.txt
 
 [testenv:citest]
 basepython = python3