Merge lp:~raharper/ubuntu/xenial/curtin/pkg-sru-revno399 into lp:~smoser/ubuntu/xenial/curtin/pkg

[Ryan Harper]

SRU'ing revno399 into Xenial

[Scott Moser (smoser)]

i also added the debian/new-upstream-snapshot fix.

review: Approve

=== modified file 'curtin/block/__init__.py'
--- curtin/block/__init__.py	2016-05-10 16:13:29 +0000
+++ curtin/block/__init__.py	2016-07-12 16:34:37 +0000
@@ -356,6 +356,7 @@
         cmd.append('--replace-whitespace')
     try:
         (out, err) = util.subp(cmd, capture=True)
+        LOG.debug("scsi_id output raw:\n%s\nerror:\n%s", out, err)
         scsi_wwid = out.rstrip('\n')
         return scsi_wwid
     except util.ProcessExecutionError as e:
@@ -474,9 +475,14 @@
     """
     # Get all volumes in /dev/disk/by-id/ containing the serial string. The
     # string specified can be either in the short or long serial format
-    disks = list(filter(lambda x: serial in x, os.listdir("/dev/disk/by-id/")))
+    # hack, some serials have spaces, udev usually converts ' ' -> '_'
+    serial_udev = serial.replace(' ', '_')
+    LOG.info('Processing serial %s via udev to %s', serial, serial_udev)
+
+    disks = list(filter(lambda x: serial_udev in x,
+                        os.listdir("/dev/disk/by-id/")))
     if not disks or len(disks) < 1:
-        raise ValueError("no disk with serial '%s' found" % serial)
+        raise ValueError("no disk with serial '%s' found" % serial_udev)
 
     # Sort by length and take the shortest path name, as the longer path names
     # will be the partitions on the disk. Then use os.path.realpath to
@@ -486,7 +492,7 @@
 
     if not os.path.exists(path):
         raise ValueError("path '%s' to block device for disk with serial '%s' \
-            does not exist" % (path, serial))
+            does not exist" % (path, serial_udev))
     return path
 
 

=== modified file 'curtin/commands/block_meta.py'
--- curtin/commands/block_meta.py	2016-05-10 16:13:29 +0000
+++ curtin/commands/block_meta.py	2016-07-12 16:34:37 +0000
@@ -379,9 +379,14 @@
         if vol.get('serial'):
             volume_path = block.lookup_disk(vol.get('serial'))
         elif vol.get('path'):
-            volume_path = vol.get('path')
+            # resolve any symlinks to the dev_kname so sys/class/block access
+            # is valid.  ie, there are no udev generated values in sysfs
+            volume_path = os.path.realpath(vol.get('path'))
+        elif vol.get('wwn'):
+            by_wwn = '/dev/disk/by-id/wwn-%s' % vol.get('wwn')
+            volume_path = os.path.realpath(by_wwn)
         else:
-            raise ValueError("serial number or path to block dev must be \
+            raise ValueError("serial, wwn or path to block dev must be \
                 specified to identify disk")
 
     elif vol.get('type') == "lvm_partition":

=== modified file 'curtin/commands/curthooks.py'
--- curtin/commands/curthooks.py	2016-06-03 13:50:09 +0000
+++ curtin/commands/curthooks.py	2016-07-12 16:34:37 +0000
@@ -238,7 +238,7 @@
 
 def install_kernel(cfg, target):
     kernel_cfg = cfg.get('kernel', {'package': None,
-                                    'fallback-package': None,
+                                    'fallback-package': "linux-generic",
                                     'mapping': {}})
     if kernel_cfg is not None:
         kernel_package = kernel_cfg.get('package')
@@ -270,7 +270,7 @@
             LOG.warn("Couldn't detect kernel package to install for %s."
                      % kernel)
             if kernel_fallback is not None:
-                util.install_packages([kernel_fallback])
+                util.install_packages([kernel_fallback], target=target)
             return
 
         package = "linux-{flavor}{map_suffix}".format(
@@ -638,6 +638,21 @@
     LOG.info("Detected multipath devices. Installing support via %s", mppkgs)
 
     util.install_packages(mppkgs, target=target)
+    replace_spaces = True
+    try:
+        # check in-target version
+        pkg_ver = util.get_package_version('multipath-tools', target=target)
+        LOG.debug("get_package_version:\n%s", pkg_ver)
+        LOG.debug("multipath version is %s (major=%s minor=%s micro=%s)",
+                  pkg_ver['semantic_version'], pkg_ver['major'],
+                  pkg_ver['minor'], pkg_ver['micro'])
+        # multipath-tools versions < 0.5.0 do _NOT_ want whitespace replaced
+        # i.e. 0.4.X in Trusty.
+        if pkg_ver['semantic_version'] < 500:
+            replace_spaces = False
+    except Exception as e:
+        LOG.warn("failed reading multipath-tools version, "
+                 "assuming it wants no spaces in wwids: %s", e)
 
     multipath_cfg_path = os.path.sep.join([target, '/etc/multipath.conf'])
     multipath_bind_path = os.path.sep.join([target, '/etc/multipath/bindings'])
@@ -658,7 +673,8 @@
     if mpbindings or not os.path.isfile(multipath_bind_path):
         # we do assume that get_devices_for_mp()[0] is /
         target_dev = block.get_devices_for_mp(target)[0]
-        wwid = block.get_scsi_wwid(target_dev)
+        wwid = block.get_scsi_wwid(target_dev,
+                                   replace_whitespace=replace_spaces)
         blockdev, partno = block.get_blockdev_for_partition(target_dev)
 
         mpname = "mpath0"

=== modified file 'curtin/commands/net_meta.py'
--- curtin/commands/net_meta.py	2016-02-12 21:54:46 +0000
+++ curtin/commands/net_meta.py	2016-07-12 16:34:37 +0000
@@ -20,6 +20,7 @@
 import sys
 
 from curtin import net
+from curtin.log import LOG
 import curtin.util as util
 import curtin.config as config
 
@@ -53,23 +54,20 @@
         raise ValueError("'%s' is not an alias: %s", alias, DEVNAME_ALIASES)
 
 
-def interfaces_basic_dhcp(devices):
-    content = '\n'.join(
-        [("# This file describes the network interfaces available on "
-         "your system"),
-         "# and how to activate them. For more information see interfaces(5).",
-         "",
-         "# The loopback network interface",
-         "auto lo",
-         "iface lo inet loopback",
-         ])
-
-    for d in devices:
-        content += '\n'.join(("", "", "auto %s" % d,
-                              "iface %s inet dhcp" % d,))
-    content += "\n"
-
-    return content
+def interfaces_basic_dhcp(devices, macs=None):
+    # return network configuration that says to dhcp on provided devices
+    if macs is None:
+        macs = {}
+        for dev in devices:
+            macs[dev] = net.get_interface_mac(dev)
+
+    config = []
+    for dev in devices:
+        config.append({
+            'type': 'physical', 'name': dev, 'mac_address': macs.get(dev),
+            'subnets': [{'type': 'dhcp4'}]})
+
+    return {'network': {'version': 1, 'config': config}}
 
 
 def interfaces_custom(args):
@@ -81,7 +79,7 @@
         raise Exception("network configuration is required by mode '%s' "
                         "but not provided in the config file" % 'custom')
 
-    return config.dump_config({'network': network_config})
+    return {'network': network_config}
 
 
 def net_meta(args):
@@ -124,6 +122,9 @@
             else:
                 devices.append(dev)
 
+    LOG.debug("net-meta mode is '%s'.  devices=%s", args.mode, devices)
+
+    output_network_config = os.environ.get("OUTPUT_NETWORK_CONFIG", "")
     if args.mode == "copy":
         if not args.target:
             raise argparse.ArgumentTypeError("mode 'copy' requires --target")
@@ -131,21 +132,31 @@
         t_eni = os.path.sep.join((args.target, "etc/network/interfaces",))
         with open(t_eni, "r") as fp:
             content = fp.read()
+        LOG.warn("net-meta mode is 'copy', static network interfaces files"
+                 "can be brittle.  Copied interfaces: %s", content)
+        target = args.output
 
     elif args.mode == "dhcp":
-        content = interfaces_basic_dhcp(devices)
+        target = output_network_config
+        content = config.dump_config(interfaces_basic_dhcp(devices))
+
     elif args.mode == 'custom':
-        content = interfaces_custom(args)
-        # if we have a config, write it out to OUTPUT_NETWORK_CONFIG
-        output_network_config = os.environ.get("OUTPUT_NETWORK_CONFIG", "")
-        if output_network_config:
-            with open(output_network_config, "w") as fp:
-                fp.write(content)
-
-    if args.output == "-":
+        target = output_network_config
+        content = config.dump_config(interfaces_custom(args))
+
+    else:
+        raise Exception("Unexpected network config mode '%s'." % args.mode)
+
+    if not target:
+        raise Exception(
+            "No target given for mode = '%s'.  No where to write content: %s" %
+            (args.mode, content))
+
+    LOG.debug("writing to file %s with network config: %s", target, content)
+    if target == "-":
         sys.stdout.write(content)
     else:
-        with open(args.output, "w") as fp:
+        with open(target, "w") as fp:
             fp.write(content)
 
     sys.exit(0)

=== modified file 'curtin/net/__init__.py'
--- curtin/net/__init__.py	2016-05-10 16:13:29 +0000
+++ curtin/net/__init__.py	2016-07-12 16:34:37 +0000
@@ -331,7 +331,14 @@
 
 
 # TODO: switch to valid_map for attrs
-def iface_add_attrs(iface):
+def iface_add_attrs(iface, index):
+    # If the index is non-zero, this is an alias interface. Alias interfaces
+    # represent additional interface addresses, and should not have additional
+    # attributes. (extra attributes here are almost always either incorrect,
+    # or are applied to the parent interface.) So if this is an alias, stop
+    # right here.
+    if index != 0:
+        return ""
     content = ""
     ignore_map = [
         'control',
@@ -423,22 +430,30 @@
                 iface['index'] = index
                 iface['mode'] = subnet['type']
                 iface['control'] = subnet.get('control', 'auto')
+                subnet_inet = 'inet'
                 if iface['mode'].endswith('6'):
-                    iface['inet'] += '6'
+                    # This is a request for DHCPv6.
+                    subnet_inet += '6'
                 elif iface['mode'] == 'static' and ":" in subnet['address']:
-                    iface['inet'] += '6'
+                    # This is a static IPv6 address.
+                    subnet_inet += '6'
+                iface['inet'] = subnet_inet
                 if iface['mode'].startswith('dhcp'):
                     iface['mode'] = 'dhcp'
 
                 content += iface_start_entry(iface, index)
                 content += iface_add_subnet(iface, subnet)
-                content += iface_add_attrs(iface)
+                content += iface_add_attrs(iface, index)
+                if len(subnets) > 1 and index == 0:
+                    for i in range(1, len(subnets)):
+                        content += "    post-up ifup %s:%s\n" % (iface['name'],
+                                                                 i)
         else:
             # ifenslave docs say to auto the slave devices
             if 'bond-master' in iface:
                 content += "auto {name}\n".format(**iface)
             content += "iface {name} {inet} {mode}\n".format(**iface)
-            content += iface_add_attrs(iface)
+            content += iface_add_attrs(iface, index)
 
     for route in network_state.get('routes'):
         content += render_route(route)
@@ -469,4 +484,9 @@
     LOG.info('Writing ' + cc_disable)
     util.write_file(cc_disable, content='network: {config: disabled}\n')
 
+
+def get_interface_mac(ifname):
+    """Returns the string value of an interface's MAC Address"""
+    return read_sys_net(ifname, "address", enoent=False)
+
 # vi: ts=4 expandtab syntax=python

=== modified file 'curtin/reporter/handlers.py'
--- curtin/reporter/handlers.py	2016-05-10 16:13:29 +0000
+++ curtin/reporter/handlers.py	2016-07-12 16:34:37 +0000
@@ -23,7 +23,7 @@
 
 
 class LogHandler(ReportingHandler):
-    """Publishes events to the cloud-init log at the ``INFO`` log level."""
+    """Publishes events to the curtin log at the ``DEBUG`` log level."""
 
     def __init__(self, level="DEBUG"):
         super(LogHandler, self).__init__()
@@ -39,9 +39,9 @@
         self.level = level
 
     def publish_event(self, event):
-        """Publish an event to the ``INFO`` log level."""
+        """Publish an event to the ``DEBUG`` log level."""
         logger = logging.getLogger(
-            '.'.join(['cloudinit', 'reporting', event.event_type, event.name]))
+            '.'.join(['curtin', 'reporting', event.event_type, event.name]))
         logger.log(self.level, event.as_string())
 
 
@@ -55,7 +55,7 @@
 class WebHookHandler(ReportingHandler):
     def __init__(self, endpoint, consumer_key=None, token_key=None,
                  token_secret=None, consumer_secret=None, timeout=None,
-                 retries=None, level="INFO"):
+                 retries=None, level="DEBUG"):
         super(WebHookHandler, self).__init__()
 
         self.oauth_helper = url_helper.OauthUrlHelper(
@@ -72,15 +72,6 @@
         self.headers = {'Content-Type': 'application/json'}
 
     def publish_event(self, event):
-        if isinstance(event.level, int):
-            ev_level = event.level
-        else:
-            try:
-                ev_level = getattr(logging, event.level.upper())
-            except:
-                ev_level = logging.INFO
-        if ev_level < self.level:
-            return
         try:
             return self.oauth_helper.geturl(
                 url=self.endpoint, data=event.as_dict(),

=== modified file 'curtin/util.py'
--- curtin/util.py	2016-06-03 13:50:09 +0000
+++ curtin/util.py	2016-07-12 16:34:37 +0000
@@ -499,6 +499,62 @@
         return False
 
 
+def parse_dpkg_version(raw, name=None, semx=None):
+    """Parse a dpkg version string into various parts and calcualate a
+       numerical value of the version for use in comparing package versions
+
+       returns a dictionary with the results
+    """
+    if semx is None:
+        semx = (10000, 100, 1)
+
+    upstream = raw.split('-')[0]
+    toks = upstream.split(".", 2)
+    if len(toks) == 3:
+        major, minor, micro = toks
+    elif len(toks) == 2:
+        major, minor, micro = (toks[0], toks[1], 0)
+    elif len(toks) == 1:
+        major, minor, micro = (toks[0], 0, 0)
+
+    version = {
+        'major': major,
+        'minor': minor,
+        'micro': micro,
+        'raw': raw,
+        'upstream': upstream,
+    }
+    if name:
+        version['name'] = name
+
+    if semx:
+        try:
+            version['semantic_version'] = int(
+                int(major) * semx[0] + int(minor) * semx[1] +
+                int(micro) * semx[2])
+        except (ValueError, IndexError):
+            version['semantic_version'] = None
+
+    return version
+
+
+def get_package_version(pkg, target=None, semx=None):
+    """Use dpkg-query to extract package pkg's version string
+       and parse the version string into a dictionary
+    """
+    chroot = []
+    if target is not None:
+        chroot = ['chroot', target]
+    try:
+        out, _ = subp(chroot + ['dpkg-query', '--show', '--showformat',
+                                '${Version}', pkg],
+                      capture=True)
+        raw = out.rstrip()
+        return parse_dpkg_version(raw, name=pkg, semx=semx)
+    except ProcessExecutionError:
+        return None
+
+
 def find_newer(src, files):
     mtime = os.stat(src).st_mtime
     return [f for f in files if

=== modified file 'debian/changelog'
--- debian/changelog	2016-06-03 14:35:22 +0000
+++ debian/changelog	2016-07-12 16:34:37 +0000
@@ -1,3 +1,22 @@
+curtin (0.1.0~bzr399-0ubuntu1~16.04.1) xenial; urgency=medium
+
+  * sru current curtin
+    - curtin/net: fix inet value for subnets, don't add interface attributes
+      to alias (LP: #1588547)
+    - improve net-meta network configuration (LP: #1592149)
+    - reporting: set webhook handler level to DEBUG, no filtering
+      (LP: #1590846)
+    - tests/vmtests: add yakkety, remove vivid
+    - curtin/net: use post-up for interface alias, resolve 120 second time out
+      on Trusty when using interface aliases
+    - vmtest: provide info on images used
+    - fix multipath configuration and add multipath tests (LP: #1551937)
+    - tools/launch and tools/xkvm: whitespace cleanup and bash -x
+    - tools/launch: boot by root=LABEL=cloudimg-rootfs
+    - Initial vmtest power8 support and TestSimple test.
+
+ -- Ryan Harper <[e-mail]>  Tue, 12 Jul 2016 11:29:30 -0500
+
 curtin (0.1.0~bzr389-0ubuntu1~16.04.1) xenial-proposed; urgency=medium
 
   * New upstream snapshot.

=== modified file 'examples/tests/basic_network.yaml'
--- examples/tests/basic_network.yaml	2016-06-03 13:50:09 +0000
+++ examples/tests/basic_network.yaml	2016-07-12 16:34:37 +0000
@@ -16,6 +16,12 @@
               - type: static
                 address: 10.0.2.100/24
               - type: static
+                address: 10.0.3.100/24
+              - type: static
+                address: 10.0.4.100/24
+              - type: static
+                address: 10.0.5.100/24
+              - type: static
                 address: 10.0.2.200/24
                 dns_nameservers:
                   - 8.8.8.8

=== added file 'examples/tests/basic_scsi.yaml'
--- examples/tests/basic_scsi.yaml	1970-01-01 00:00:00 +0000
+++ examples/tests/basic_scsi.yaml	2016-07-12 16:34:37 +0000
@@ -0,0 +1,72 @@
+showtrace: true
+storage:
+    version: 1
+    config:
+      - id: sda
+        type: disk
+        ptable: msdos
+        wwn: '0x39cc071e72c64cc4'
+        name: main_disk
+        wipe: superblock
+        grub_device: true
+      - id: sda1
+        type: partition
+        number: 1
+        size: 3GB
+        device: sda
+        flag: boot
+      - id: sda2
+        type: partition
+        number: 2
+        size: 1GB
+        device: sda
+      - id: sda1_root
+        type: format
+        fstype: ext4
+        volume: sda1
+      - id: sda2_home
+        type: format
+        fstype: ext4
+        volume: sda2
+      - id: sda1_mount
+        type: mount
+        path: /
+        device: sda1_root
+      - id: sda2_mount
+        type: mount
+        path: /home
+        device: sda2_home
+      - id: sparedisk_id
+        type: disk
+        wwn: '0x080258d13ea95ae5'
+        name: sparedisk
+        wipe: superblock
+      - id: btrfs_disk_id
+        type: disk
+        wwn: '0x22dc58dc023c7008'
+        name: btrfs_volume
+        wipe: superblock
+      - id: btrfs_disk_fmt_id
+        type: format
+        fstype: btrfs
+        volume: btrfs_disk_id
+      - id: btrfs_disk_mnt_id
+        type: mount
+        path: /btrfs
+        device: btrfs_disk_fmt_id
+      - id: pnum_disk
+        type: disk
+        wwn: '0x550a270c3a5811c5'
+        name: pnum_disk
+        wipe: superblock
+        ptable: gpt
+      - id: pnum_disk_p1
+        type: partition
+        number: 1
+        size: 1GB
+        device: pnum_disk
+      - id: pnum_disk_p2
+        type: partition
+        number: 10
+        size: 1GB
+        device: pnum_disk

=== modified file 'examples/tests/bonding_network.yaml'
--- examples/tests/bonding_network.yaml	2016-06-03 13:50:09 +0000
+++ examples/tests/bonding_network.yaml	2016-07-12 16:34:37 +0000
@@ -26,6 +26,8 @@
           subnets:
               - type: static
                 address: 10.23.23.2/24
+              - type: static
+                address: 10.23.24.2/24
 
 curthooks_commands:
     # use curtin to disable open-iscsi ifupdown hooks for precise; they're

=== added file 'examples/tests/multipath.yaml'
--- examples/tests/multipath.yaml	1970-01-01 00:00:00 +0000
+++ examples/tests/multipath.yaml	2016-07-12 16:34:37 +0000
@@ -0,0 +1,38 @@
+showtrace: true
+storage:
+    version: 1
+    config:
+      - id: sda
+        type: disk
+        ptable: msdos
+        serial: 'IPR-0 1234567890'
+        name: mpath_a
+        wipe: superblock
+        grub_device: true
+      - id: sda1
+        type: partition
+        number: 1
+        size: 3GB
+        device: sda
+        flag: boot
+      - id: sda2
+        type: partition
+        number: 2
+        size: 1GB
+        device: sda
+      - id: sda1_root
+        type: format
+        fstype: ext4
+        volume: sda1
+      - id: sda2_home
+        type: format
+        fstype: ext4
+        volume: sda2
+      - id: sda1_mount
+        type: mount
+        path: /
+        device: sda1_root
+      - id: sda2_mount
+        type: mount
+        path: /home
+        device: sda2_home

=== added file 'examples/tests/simple.yaml'
--- examples/tests/simple.yaml	1970-01-01 00:00:00 +0000
+++ examples/tests/simple.yaml	2016-07-12 16:34:37 +0000
@@ -0,0 +1,3 @@
+# This pushes curtin through a automatic installation
+# where no storage configuration is necessary.
+placeholder_simple_install: unused

=== modified file 'helpers/common'
--- helpers/common	2016-05-10 16:13:29 +0000
+++ helpers/common	2016-07-12 16:34:37 +0000
@@ -356,15 +356,17 @@
     local cmd="" isblk=false
     [ -b "$target" ] && isblk=true
 
+    local pprep="1" proot="2"
     wipedev "$target" ||
         { error "failed to clear $target"; return 1; }
 
     cmd=(
         sgdisk
-           --new "1::+8M"  --typecode=1:4100
-           --new "2::$end" --typecode=2:8300
+           --new "${pprep}::+8M"  "--typecode=${pprep}:4100"
+           --new "${proot}::$end" "--typecode=${proot}:8300"
            "$target"
     )
+    debug 1 "partitioning '$target' with ${cmd[*]}"
     "${cmd[@]}" ||
         fail "Failed to create GPT partitions (${cmd[*]})"
 
@@ -374,13 +376,13 @@
     if $isblk; then
         blockdev --rereadpt "$target"
         udevadm settle
-        assert_partitions "$target" 1 2 ||
+        assert_partitions "$target" "${proot}" "${pprep}"  ||
             { error "$target missing partitions: $_RET"; return 1; }
         # wipe the full prep partition
-        wipe_partitions --full "$target" 1 ||
+        wipe_partitions --full "$target" "${pprep}" ||
             { error "$target: failed to wipe full PReP partition"; return 1;}
-        wipe_partitions "$target" 2 ||
-            { error "$target: failed to wipe partition 2"; return 1;}
+        wipe_partitions "$target" "${proot}" ||
+            { error "$target: failed to wipe partition ${proot}"; return 1;}
     fi
 
     return 0
@@ -695,6 +697,12 @@
             "${grub_name}" "${grub_target}" "$nvram" </dev/null ||
             { error "failed to install grub!"; return 1; }
     else
+        # Note: dpkg-reconfigure calls grub-install on ppc64
+        # this means that using '--no-nvram' below ends up
+        # failing very oddly.  This is because grub's post-inst
+        # runs grub-install with no target.  That ends up
+        # updating nvram badly, and then the grub-install would
+        # not fix it because of the no-nvram there.
         debug 1 "installing ${grub_name} to: ${grubdevs[*]}"
         chroot "$mp" env DEBIAN_FRONTEND=noninteractive sh -ec '
             pkg=$1; shift;

=== modified file 'tests/unittests/test_net.py'
--- tests/unittests/test_net.py	2016-06-03 13:50:09 +0000
+++ tests/unittests/test_net.py	2016-07-12 16:34:37 +0000
@@ -473,6 +473,7 @@
 
             auto eth0
             iface eth0 inet dhcp
+                post-up ifup eth0:1
 
             auto eth0:1
             iface eth0:1 inet static
@@ -516,6 +517,11 @@
                 bond-mode active-backup
                 hwaddress 52:54:00:12:34:06
                 bond-slaves none
+                post-up ifup bond0:1
+
+            auto bond0:1
+            iface bond0:1 inet static
+                address 10.23.24.2/24
 
             source /etc/network/interfaces.d/*.cfg
             """)
@@ -545,11 +551,11 @@
                 address 192.168.14.2/24
                 gateway 192.168.14.1
                 mtu 1492
+                post-up ifup interface1:1
 
             auto interface1:1
             iface interface1:1 inet static
                 address 192.168.14.4/24
-                mtu 1492
 
             allow-hotplug interface2
             iface interface2 inet static
@@ -566,4 +572,46 @@
         self.assertEqual(sorted(ifaces.split('\n')),
                          sorted(net_ifaces.split('\n')))
 
+    def test_render_interfaces_ipv6_aliases(self):
+        ipv6_aliases_config = '''
+# YAML example of a simple network config
+network:
+    version: 1
+    config:
+        # Physical interfaces.
+        - type: physical
+          name: eth0
+          mac_address: "c0:d6:9f:2c:e8:80"
+          subnets:
+              - type: static
+                address: fde9:8f83:4a81:1:0:1:0:6/64
+              - type: static
+                address: 192.168.0.1/24
+'''
+
+        ns = self.get_net_state(ipv6_aliases_config)
+        ifaces = dedent("""\
+            auto lo
+            iface lo inet loopback
+
+            auto eth0
+            iface eth0 inet6 static
+                address fde9:8f83:4a81:1:0:1:0:6/64
+                post-up ifup eth0:1
+
+            auto eth0:1
+            iface eth0:1 inet static
+                address 192.168.0.1/24
+
+            source /etc/network/interfaces.d/*.cfg
+            """)
+        net_ifaces = net.render_interfaces(ns.network_state)
+        print("\n".join(sorted(ifaces.split('\n'))))
+        print("\n^^ LOCAL -- RENDER vv")
+        print("\n".join(sorted(net_ifaces.split('\n'))))
+        print(ns.network_state.get('interfaces'))
+        self.assertEqual(sorted(ifaces.split('\n')),
+                         sorted(net_ifaces.split('\n')))
+
+
 # vi: ts=4 expandtab syntax=python

=== modified file 'tests/unittests/test_reporter.py'
--- tests/unittests/test_reporter.py	2016-05-10 16:13:29 +0000
+++ tests/unittests/test_reporter.py	2016-07-12 16:34:37 +0000
@@ -128,7 +128,8 @@
     def test_webhook_handler(self, mock_url_helper):
         event = events.ReportingEvent(events.START_EVENT_TYPE, 'test_event',
                                       'test event', level='INFO')
-        webhook_handler = handlers.WebHookHandler('127.0.0.1:8000')
+        webhook_handler = handlers.WebHookHandler('127.0.0.1:8000',
+                                                  level='INFO')
         webhook_handler.publish_event(event)
         webhook_handler.oauth_helper.geturl.assert_called_with(
             url='127.0.0.1:8000', data=event.as_dict(),
@@ -136,7 +137,6 @@
         event.level = 'DEBUG'
         webhook_handler.oauth_helper.geturl.called = False
         webhook_handler.publish_event(event)
-        self.assertFalse(webhook_handler.oauth_helper.geturl.called)
         webhook_handler = handlers.WebHookHandler('127.0.0.1:8000',
                                                   level="INVALID")
         self.assertEquals(webhook_handler.level, 30)
@@ -194,3 +194,24 @@
                              base64.b64encode(test_data).decode())
         finally:
             os.remove(tmp[1])
+
+    @patch('curtin.url_helper.OauthUrlHelper')
+    def test_webhook_handler_post_files(self, mock_url_helper):
+        test_data = b'abcdefg'
+        tmp = tempfile.mkstemp()
+        tmpfname = tmp[1]
+        try:
+            with open(tmpfname, 'wb') as fp:
+                fp.write(test_data)
+            event = events.FinishReportingEvent('test_event_name',
+                                                'test event description',
+                                                post_files=[tmpfname],
+                                                level='INFO')
+            webhook_handler = handlers.WebHookHandler('127.0.0.1:8000',
+                                                      level='INFO')
+            webhook_handler.publish_event(event)
+            webhook_handler.oauth_helper.geturl.assert_called_with(
+                url='127.0.0.1:8000', data=event.as_dict(),
+                headers=webhook_handler.headers, retries=None)
+        finally:
+            os.remove(tmpfname)

=== modified file 'tests/vmtests/__init__.py'
--- tests/vmtests/__init__.py	2016-06-03 13:50:09 +0000
+++ tests/vmtests/__init__.py	2016-07-12 16:34:37 +0000
@@ -48,6 +48,7 @@
 
 
 DEFAULT_BRIDGE = os.environ.get("CURTIN_VMTEST_BRIDGE", "user")
+OUTPUT_DISK_NAME = 'output_disk.img'
 
 _TOPDIR = None
 
@@ -223,6 +224,7 @@
                          "Expected=%s" % (found, expected))
     for item in results:
         ftypes[item['ftype']] = item['item_url']
+        last_item = item
 
     missing = [(ftype, path) for ftype, path in ftypes.items()
                if not os.path.exists(path)]
@@ -230,7 +232,11 @@
     if len(missing):
         raise ValueError("missing files for ftypes: %s" % missing)
 
-    return ftypes
+    # trusty amd64/hwe-p 20150101
+    version_info = ('%(release)s %(arch)s/%(subarch)s %(version_name)s' %
+                    last_item)
+
+    return version_info, ftypes
 
 
 class ImageStore:
@@ -256,16 +262,17 @@
         self.url = pathlib.Path(self.base_dir).as_uri()
 
     def get_image(self, release, arch, krel=None):
-        """Return local path for root image, kernel and initrd, tarball."""
+        """Return tuple of version info, and paths for root image,
+           kernel, initrd, tarball."""
         if krel is None:
             krel = release
-        ftypes = get_images(
+        ver_info, ftypes = get_images(
             self.source_url, self.base_dir, release, arch, krel, self.sync)
         root_image_path = ftypes['vmtest.root-image']
         kernel_path = ftypes['boot-kernel']
         initrd_path = ftypes['boot-initrd']
         tarball = ftypes['vmtest.root-tgz']
-        return (root_image_path, kernel_path, initrd_path, tarball)
+        return ver_info, (root_image_path, kernel_path, initrd_path, tarball)
 
 
 class TempDir(object):
@@ -327,7 +334,7 @@
 
         # create output disk, mount ro
         logger.debug('Creating output disk')
-        self.output_disk = os.path.join(self.boot, "output_disk.img")
+        self.output_disk = os.path.join(self.boot, OUTPUT_DISK_NAME)
         subprocess.check_call(["qemu-img", "create", "-f", TARGET_IMAGE_FORMAT,
                               self.output_disk, "10M"],
                               stdout=DEVNULL, stderr=subprocess.STDOUT)
@@ -344,20 +351,23 @@
 class VMBaseClass(TestCase):
     __test__ = False
     arch_skip = []
+    boot_timeout = 300
+    collect_scripts = []
+    conf_file = "examples/tests/basic.yaml"
     disk_block_size = 512
+    disk_driver = 'virtio-blk'
     disk_to_check = {}
+    extra_disks = []
+    extra_kern_args = None
     fstab_expected = {}
-    extra_kern_args = None
-    recorded_errors = 0
-    recorded_failures = 0
     image_store_class = ImageStore
-    collect_scripts = []
+    install_timeout = 3000
     interactive = False
-    conf_file = "examples/tests/basic.yaml"
-    extra_disks = []
+    multipath = False
+    multipath_num_paths = 2
     nvme_disks = []
-    boot_timeout = 300
-    install_timeout = 3000
+    recorded_errors = 0
+    recorded_failures = 0
     uefi = False
 
     # these get set from base_vm_classes
@@ -380,15 +390,17 @@
         # Disable sync if env var is set.
         image_store.sync = get_env_var_bool('CURTIN_VMTEST_IMAGE_SYNC', False)
         logger.debug("Image sync = %s", image_store.sync)
-        (boot_img, boot_kernel, boot_initrd, tarball) = image_store.get_image(
-            cls.release, cls.arch, cls.krel)
-
+        img_verstr, (boot_img, boot_kernel, boot_initrd, tarball) = (
+            image_store.get_image(cls.release, cls.arch, cls.krel))
+        logger.debug("Image %s\n  boot=%s\n  kernel=%s\n  initrd=%s\n"
+                     "  tarball=%s\n", img_verstr, boot_img, boot_kernel,
+                     boot_initrd, tarball)
         # set up tempdir
         cls.td = TempDir(
             name=cls.__name__,
             user_data=generate_user_data(collect_scripts=cls.collect_scripts))
-        logger.info('Using tempdir: {}'.format(cls.td.tmpdir))
-
+        logger.info('Using tempdir: %s , Image: %s', cls.td.tmpdir,
+                    img_verstr)
         cls.install_log = os.path.join(cls.td.logs, 'install-serial.log')
         cls.boot_log = os.path.join(cls.td.logs, 'boot-serial.log')
         logger.debug('Install console log: {}'.format(cls.install_log))
@@ -438,21 +450,51 @@
             netdevs.extend(["--netdev=" + DEFAULT_BRIDGE])
 
         # build disk arguments
-        # --disk source:size:driver:block_size
-        extra_disks = []
+        disks = []
+        sc = util.load_file(cls.conf_file)
+        storage_config = yaml.load(sc).get('storage', {}).get('config', {})
+        cls.disk_wwns = ["wwn=%s" % x.get('wwn') for x in storage_config
+                         if 'wwn' in x]
+        cls.disk_serials = ["serial=%s" % x.get('serial')
+                            for x in storage_config if 'serial' in x]
+
+        target_disk = "{}:{}:{}:{}:".format(cls.td.target_disk,
+                                            "",
+                                            cls.disk_driver,
+                                            cls.disk_block_size)
+        if len(cls.disk_wwns):
+            target_disk += cls.disk_wwns[0]
+
+        if len(cls.disk_serials):
+            target_disk += cls.disk_serials[0]
+
+        disks.extend(['--disk', target_disk])
+
+        # --disk source:size:driver:block_size:devopts
         for (disk_no, disk_sz) in enumerate(cls.extra_disks):
             dpath = os.path.join(cls.td.disks, 'extra_disk_%d.img' % disk_no)
-            extra_disks.extend(
-                ['--disk', '{}:{}:{}:{}'.format(dpath, disk_sz, "",
-                                                cls.disk_block_size)])
+            extra_disk = '{}:{}:{}:{}:'.format(dpath, disk_sz,
+                                               cls.disk_driver,
+                                               cls.disk_block_size)
+            if len(cls.disk_wwns):
+                w_index = disk_no + 1
+                if w_index < len(cls.disk_wwns):
+                    extra_disk += cls.disk_wwns[w_index]
+
+            if len(cls.disk_serials):
+                w_index = disk_no + 1
+                if w_index < len(cls.disk_serials):
+                    extra_disk += cls.disk_serials[w_index]
+
+            disks.extend(['--disk', extra_disk])
 
         # build nvme disk args if needed
-        nvme_disks = []
         for (disk_no, disk_sz) in enumerate(cls.nvme_disks):
             dpath = os.path.join(cls.td.disks, 'nvme_disk_%d.img' % disk_no)
-            nvme_disks.extend(
-                ['--disk', '{}:{}:nvme:{}'.format(dpath, disk_sz,
-                                                  cls.disk_block_size)])
+            nvme_disk = '{}:{}:nvme:{}:{}'.format(dpath, disk_sz,
+                                                  cls.disk_block_size,
+                                                  "serial=nvme-%d" % disk_no)
+            disks.extend(['--disk', nvme_disk])
 
         # proxy config
         configs = [cls.conf_file]
@@ -477,11 +519,10 @@
             shutil.copy(OVMF_VARS, nvram)
             cmd.extend(["--uefi", nvram])
 
-        # --disk source:size:driver:block_size
-        target_disk = "{}:{}:{}:{}".format(cls.td.target_disk, "", "",
-                                           cls.disk_block_size)
-        cmd.extend(netdevs + ["--disk", target_disk] +
-                   extra_disks + nvme_disks +
+        if cls.multipath:
+            disks = disks * cls.multipath_num_paths
+
+        cmd.extend(netdevs + disks +
                    [boot_img, "--kernel=%s" % boot_kernel, "--initrd=%s" %
                     boot_initrd, "--", "curtin", "-vv", "install"] +
                    ["--config=%s" % f for f in configs] +
@@ -510,8 +551,8 @@
         logger.debug('')
         try:
             if os.path.exists(cls.install_log):
-                with open(cls.install_log) as l:
-                    install_log = l.read()
+                with open(cls.install_log, 'rb') as l:
+                    install_log = l.read().decode('utf-8', errors='replace')
                 errmsg, errors = check_install_log(install_log)
                 if errmsg:
                     for e in errors:
@@ -527,39 +568,58 @@
             cls.tearDownClass()
             raise
 
-        # drop the size parameter if present in extra_disks
-        extra_disks = [x if ":" not in x else x.split(':')[0]
-                       for x in extra_disks]
         # create --disk params for nvme disks
         bsize_args = "logical_block_size={}".format(cls.disk_block_size)
         bsize_args += ",physical_block_size={}".format(cls.disk_block_size)
         bsize_args += ",min_io_size={}".format(cls.disk_block_size)
-        disk_driver = "virtio-blk"
 
         target_disks = []
-        for (disk_no, disk) in enumerate([cls.td.target_disk,
-                                          cls.td.output_disk]):
-            d = '--disk={},driver={},format={},{}'.format(disk, disk_driver,
-                                                          TARGET_IMAGE_FORMAT,
-                                                          bsize_args)
-            target_disks.extend([d])
+        for (disk_no, disk) in enumerate([cls.td.target_disk]):
+            disk = '--disk={},driver={},format={},{}'.format(
+                disk, cls.disk_driver, TARGET_IMAGE_FORMAT, bsize_args)
+            if len(cls.disk_wwns):
+                disk += ",%s" % cls.disk_wwns[0]
+            if len(cls.disk_serials):
+                disk += ",%s" % cls.disk_serials[0]
+
+            target_disks.extend([disk])
 
         extra_disks = []
         for (disk_no, disk_sz) in enumerate(cls.extra_disks):
             dpath = os.path.join(cls.td.disks, 'extra_disk_%d.img' % disk_no)
-            d = '--disk={},driver={},format={},{}'.format(dpath, disk_driver,
-                                                          TARGET_IMAGE_FORMAT,
-                                                          bsize_args)
-            extra_disks.extend([d])
+            disk = '--disk={},driver={},format={},{}'.format(
+                dpath, cls.disk_driver, TARGET_IMAGE_FORMAT, bsize_args)
+            if len(cls.disk_wwns):
+                w_index = disk_no + 1
+                if w_index < len(cls.disk_wwns):
+                    disk += ",%s" % cls.disk_wwns[w_index]
+
+            if len(cls.disk_serials):
+                w_index = disk_no + 1
+                if w_index < len(cls.disk_serials):
+                    disk += ",%s" % cls.disk_serials[w_index]
+
+            extra_disks.extend([disk])
 
         nvme_disks = []
         disk_driver = 'nvme'
         for (disk_no, disk_sz) in enumerate(cls.nvme_disks):
             dpath = os.path.join(cls.td.disks, 'nvme_disk_%d.img' % disk_no)
-            d = '--disk={},driver={},format={},{}'.format(dpath, disk_driver,
-                                                          TARGET_IMAGE_FORMAT,
-                                                          bsize_args)
-            nvme_disks.extend([d])
+            disk = '--disk={},driver={},format={},{}'.format(
+                dpath, disk_driver, TARGET_IMAGE_FORMAT, bsize_args)
+            nvme_disks.extend([disk])
+
+        if cls.multipath:
+            target_disks = target_disks * cls.multipath_num_paths
+            extra_disks = extra_disks * cls.multipath_num_paths
+            nvme_disks = nvme_disks * cls.multipath_num_paths
+
+        # output disk is always virtio-blk, with serial of output_disk.img
+        output_disk = '--disk={},driver={},format={},{},{}'.format(
+            cls.td.output_disk, 'virtio-blk',
+            TARGET_IMAGE_FORMAT, bsize_args,
+            'serial=%s' % os.path.basename(cls.td.output_disk))
+        target_disks.extend([output_disk])
 
         # create xkvm cmd
         cmd = (["tools/xkvm", "-v", dowait] + netdevs +
@@ -788,10 +848,13 @@
         self.base_dir = base_dir
 
     def get_image(self, release, arch, krel=None):
-        """Return local path for root image, kernel and initrd, tarball."""
+        """Return tuple of version info, and paths for root image,
+           kernel, initrd, tarball."""
         names = ['psuedo-root-image', 'psuedo-kernel', 'psuedo-initrd',
                  'psuedo-tarball']
-        return [os.path.join(self.base_dir, release, arch, f) for f in names]
+        return (
+            "psuedo-%s %s/hwe-P 20160101" % (release, arch),
+            [os.path.join(self.base_dir, release, arch, f) for f in names])
 
 
 class PsuedoVMBaseClass(VMBaseClass):
@@ -940,9 +1003,9 @@
               'content': yaml.dump(base_cloudconfig, indent=1)},
              {'type': 'text/cloud-config', 'content': ssh_keys}]
 
+    output_dir = '/mnt/output'
     output_dir_macro = 'OUTPUT_COLLECT_D'
-    output_dir = '/mnt/output'
-    output_device = '/dev/vdb'
+    output_device = '/dev/disk/by-id/virtio-%s' % OUTPUT_DISK_NAME
 
     collect_prep = textwrap.dedent("mkdir -p " + output_dir)
     collect_post = textwrap.dedent(
@@ -950,7 +1013,7 @@
 
     # failsafe poweroff runs on precise only, where power_state does
     # not exist.
-    precise_poweroff = textwrap.dedent("""#!/bin/sh
+    precise_poweroff = textwrap.dedent("""#!/bin/sh -x
         [ "$(lsb_release -sc)" = "precise" ] || exit 0;
         shutdown -P now "Shutting down on precise"
         """)
@@ -960,7 +1023,7 @@
 
     for part in scripts:
         if not part.startswith("#!"):
-            part = "#!/bin/sh\n" + part
+            part = "#!/bin/sh -x\n" + part
         part = part.replace(output_dir_macro, output_dir)
         logger.debug('Cloud config archive content (pre-json):' + part)
         parts.append({'content': part, 'type': 'text/x-shellscript'})

=== modified file 'tests/vmtests/helpers.py'
--- tests/vmtests/helpers.py	2016-02-12 21:54:46 +0000
+++ tests/vmtests/helpers.py	2016-07-12 16:34:37 +0000
@@ -117,3 +117,172 @@
                 if getattr(test_case, 'release', ''):
                     releases.add(getattr(test_case, 'release'))
     return sorted(releases)
+
+
+def _parse_ifconfig_xenial(ifconfig_out):
+    """Parse ifconfig output from xenial or earlier and return a dictionary.
+    given content like below, return:
+    {'eth0': {'address': '10.8.1.78', 'broadcast': '10.8.1.255',
+              'inet6': [{'address': 'fe80::216:3eff:fe63:c05d',
+                         'prefixlen': '64', 'scope': 'Link'},
+                        {'address': 'fdec:2922:2f07:0:216:3eff:fe63:c05d',
+                         'prefixlen': '64', 'scope': 'Global'}],
+              'interface': 'eth0', 'link_encap': 'Ethernet',
+              'mac_address': '00:16:3e:63:c0:5d', 'mtu': 1500,
+              'multicast': True, 'netmask': '255.255.255.0',
+              'running': True, 'up': True}}
+
+    eth0  Link encap:Ethernet  HWaddr 00:16:3e:63:c0:5d
+          inet addr:10.8.1.78  Bcast:10.8.1.255  Mask:255.255.255.0
+          inet6 addr: fe80::216:3eff:fe63:c05d/64 Scope:Link
+          inet6 addr: fdec:2922:2f07:0:216:3eff:fe63:c05d/64 Scope:Global
+          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
+          RX packets:21503 errors:0 dropped:0 overruns:0 frame:0
+          TX packets:11346 errors:0 dropped:0 overruns:0 carrier:0
+          collisions:0 txqueuelen:1000
+          RX bytes:31556357 (31.5 MB)  TX bytes:870943 (870.9 KB)
+    """
+    ifaces = {}
+    combined_fields = {'addr': 'address', 'Bcast': 'broadcast',
+                       'Mask': 'netmask', 'MTU': 'mtu',
+                       'encap': 'link_encap'}
+    boolmap = {'RUNNING': 'running', 'UP': 'up', 'MULTICAST': 'multicast'}
+
+    for line in ifconfig_out.splitlines():
+        if not line:
+            continue
+        if not line.startswith(" "):
+            cur_iface = line.split()[0].rstrip(":")
+            cur_data = {'inet6': [], 'interface': cur_iface}
+            for t in boolmap.values():
+                cur_data[t] = False
+            ifaces[cur_iface] = cur_data
+
+        toks = line.split()
+
+        if toks[0] == "inet6":
+            cidr = toks[2]
+            address, prefixlen = cidr.split("/")
+            scope = toks[3].split(":")[1]
+            cur_ipv6 = {'address': address, 'scope': scope,
+                        'prefixlen': prefixlen}
+            cur_data['inet6'].append(cur_ipv6)
+            continue
+
+        for i in range(0, len(toks)):
+            cur_tok = toks[i]
+            try:
+                next_tok = toks[i+1]
+            except IndexError:
+                next_tok = None
+
+            if cur_tok == "HWaddr":
+                cur_data['mac_address'] = next_tok
+            elif ":" in cur_tok:
+                key, _colon, val = cur_tok.partition(":")
+                if key in combined_fields:
+                    cur_data[combined_fields[key]] = val
+            elif cur_tok in boolmap:
+                cur_data[boolmap[cur_tok]] = True
+
+        if 'mtu' in cur_data:
+            cur_data['mtu'] = int(cur_data['mtu'])
+
+    return ifaces
+
+
+def _parse_ifconfig_yakkety(ifconfig_out):
+    """Parse ifconfig output from yakkety or later(?) and return a dictionary.
+
+    given ifconfig output like below, return:
+    {'ens2': {'address': '10.5.0.78',
+              'broadcast': '10.5.255.255',
+              'broadcast_flag': True,
+              'inet6': [{'address': 'fe80::f816:3eff:fe05:9673',
+                         'prefixlen': '64', 'scopeid': '0x20<link>'},
+                        {'address': 'fe80::f816:3eff:fe05:9673',
+                         'prefixlen': '64', 'scopeid': '0x20<link>'}],
+              'interface': 'ens2', 'link_encap': 'Ethernet',
+              'mac_address': 'fa:16:3e:05:96:73', 'mtu': 1500,
+              'multicast': True, 'netmask': '255.255.0.0',
+              'running': True, 'up': True}}
+
+    ens2: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
+            inet 10.5.0.78  netmask 255.255.0.0  broadcast 10.5.255.255
+            inet6 fe80::f816:3eff:fe05:9673  prefixlen 64  scopeid 0x20<link>
+            inet6 fe80::f816:3eff:fe05:9673  prefixlen 64  scopeid 0x20<link>
+            ether fa:16:3e:05:96:73  txqueuelen 1000  (Ethernet)
+            RX packets 33196  bytes 48916947 (48.9 MB)
+            RX errors 0  dropped 0  overruns 0  frame 0
+            TX packets 5458  bytes 411486 (411.4 KB)
+            TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
+    """
+    fmap = {'mtu': 'mtu', 'inet': 'address',
+            'netmask': 'netmask', 'broadcast': 'broadcast',
+            'ether': 'mac_address'}
+    boolmap = {'RUNNING': 'running', 'UP': 'up', 'MULTICAST': 'multicast',
+               'BROADCAST': 'broadcast_flag'}
+
+    ifaces = {}
+    for line in ifconfig_out.splitlines():
+        if not line:
+            continue
+        if not line.startswith(" "):
+            cur_iface = line.split()[0].rstrip(":")
+            cur_data = {'inet6': [], 'interface': cur_iface}
+            for t in boolmap.values():
+                cur_data[t] = False
+            ifaces[cur_iface] = cur_data
+
+        toks = line.split()
+        if toks[0] == "inet6":
+            cur_ipv6 = {'address': toks[1]}
+            cur_data['inet6'].append(cur_ipv6)
+
+        for i in range(0, len(toks)):
+            cur_tok = toks[i]
+            try:
+                next_tok = toks[i+1]
+            except IndexError:
+                next_tok = None
+            if cur_tok in fmap:
+                cur_data[fmap[cur_tok]] = next_tok
+            elif cur_tok in ('prefixlen', 'scopeid'):
+                cur_ipv6[cur_tok] = next_tok
+                cur_data['inet6'].append
+            elif cur_tok.startswith("flags="):
+                # flags=4163<UP,BROADCAST,RUNNING,MULTICAST>
+                flags = cur_tok[cur_tok.find("<") + 1:
+                                cur_tok.rfind(">")].split(",")
+                for flag in flags:
+                    if flag in boolmap:
+                        cur_data[boolmap[flag]] = True
+            elif cur_tok == "(Ethernet)":
+                cur_data['link_encap'] = 'Ethernet'
+
+        if 'mtu' in cur_data:
+            cur_data['mtu'] = int(cur_data['mtu'])
+
+    return ifaces
+
+
+def ifconfig_to_dict(ifconfig_a):
+    # if the first token of the first line ends in a ':' then assume yakkety
+    # parse ifconfig output and return a dictionary.
+    #
+    # return a dictionary of network information like:
+    #  {'ens2': {'address': '10.5.0.78', 'broadcast': '10.5.255.255',
+    #         'broadcast_flag': True,
+    #         'inet6': [{'address': 'fe80::f816:3eff:fe05:9673',
+    #                    'prefixlen': '64', 'scopeid': '0x20<link>'},
+    #                   {'address': 'fe80::f816:3eff:fe05:9673',
+    #                    'prefixlen': '64', 'scopeid': '0x20<link>'}],
+    #         'interface': 'ens2', 'link_encap': 'Ethernet',
+    #         'mac_address': 'fa:16:3e:05:96:73', 'mtu': 1500,
+    #         'multicast': True, 'netmask': '255.255.0.0',
+    #         'running': True, 'up': True}}
+    line = ifconfig_a.lstrip().splitlines()[0]
+    if line.split()[0].endswith(":"):
+        return _parse_ifconfig_yakkety(ifconfig_a)
+    else:
+        return _parse_ifconfig_xenial(ifconfig_a)

=== modified file 'tests/vmtests/image_sync.py'
--- tests/vmtests/image_sync.py	2016-05-10 16:13:29 +0000
+++ tests/vmtests/image_sync.py	2016-07-12 16:34:37 +0000
@@ -29,6 +29,8 @@
 VMTEST_CONTENT_ID = 'com.ubuntu.maas:daily:v2:download'
 VMTEST_JSON_PATH = "streams/v1/vmtest.json"
 
+DEFAULT_OUTPUT_FORMAT = (
+    "%(release)-7s %(arch)s/%(subarch)s %(version_name)-10s %(item_name)s")
 
 DEFAULT_ARCHES = {
     'i386': ['i386'],
@@ -397,7 +399,19 @@
     results = query(args.mirror_url, args.max_items, args.filters,
                     verbosity=vlevel)
     try:
-        print(util.json_dumps(results).decode())
+        if args.output_format == FORMAT_JSON:
+            print(util.json_dumps(results).decode())
+        else:
+            output = []
+            for item in results:
+                try:
+                    output.append(args.output_format % item)
+                except KeyError as e:
+                    sys.stderr.write("output format failed (%s) for: %s\n" %
+                                     (e, item))
+                    sys.exit(1)
+            for line in sorted(output):
+                print(line)
     except IOError as e:
         if e.errno == errno.EPIPE:
             sys.exit(0x80 | signal.SIGPIPE)
@@ -440,7 +454,7 @@
 
     fmt_group = query_p.add_mutually_exclusive_group()
     fmt_group.add_argument('--output-format', '-o', action='store',
-                           dest='output_format', default=None,
+                           dest='output_format', default=DEFAULT_OUTPUT_FORMAT,
                            help="specify output format per python str.format")
     fmt_group.add_argument('--json', action='store_const',
                            const=FORMAT_JSON, dest='output_format',

=== modified file 'tests/vmtests/releases.py'
--- tests/vmtests/releases.py	2016-05-10 16:13:29 +0000
+++ tests/vmtests/releases.py	2016-07-12 16:34:37 +0000
@@ -46,6 +46,10 @@
     release = "xenial"
 
 
+class _YakketyBase(_ReleaseBase):
+    release = "yakkety"
+
+
 class _Releases(object):
     precise = _PreciseBase
     precise_hwe_t = _PreciseHWET
@@ -56,6 +60,7 @@
     vivid = _VividBase
     wily = _WilyBase
     xenial = _XenialBase
+    yakkety = _YakketyBase
 
 base_vm_classes = _Releases
 

=== modified file 'tests/vmtests/test_basic.py'
--- tests/vmtests/test_basic.py	2016-06-03 13:50:09 +0000
+++ tests/vmtests/test_basic.py	2016-07-12 16:34:37 +0000
@@ -223,13 +223,130 @@
     __test__ = False
 
 
-class VividTestBasic(relbase.vivid, TestBasicAbs):
-    __test__ = True
-
-
 class WilyTestBasic(relbase.wily, TestBasicAbs):
     __test__ = True
 
 
 class XenialTestBasic(relbase.xenial, TestBasicAbs):
     __test__ = True
+
+
+class YakketyTestBasic(relbase.yakkety, TestBasicAbs):
+    __test__ = True
+
+
+class TestBasicScsiAbs(TestBasicAbs):
+    conf_file = "examples/tests/basic_scsi.yaml"
+    disk_driver = 'scsi-hd'
+    extra_disks = ['128G', '128G', '4G']
+    nvme_disks = ['4G']
+    collect_scripts = [textwrap.dedent("""
+        cd OUTPUT_COLLECT_D
+        blkid -o export /dev/sda > blkid_output_sda
+        blkid -o export /dev/sda1 > blkid_output_sda1
+        blkid -o export /dev/sda2 > blkid_output_sda2
+        btrfs-show-super /dev/sdc > btrfs_show_super_sdc
+        cat /proc/partitions > proc_partitions
+        ls -al /dev/disk/by-uuid/ > ls_uuid
+        ls -al /dev/disk/by-id/ > ls_disk_id
+        cat /etc/fstab > fstab
+        mkdir -p /dev/disk/by-dname
+        ls /dev/disk/by-dname/ > ls_dname
+        find /etc/network/interfaces.d > find_interfacesd
+
+        v=""
+        out=$(apt-config shell v Acquire::HTTP::Proxy)
+        eval "$out"
+        echo "$v" > apt-proxy
+        """)]
+
+    def test_output_files_exist(self):
+        self.output_files_exist(
+            ["blkid_output_sda", "blkid_output_sda1", "blkid_output_sda2",
+             "btrfs_show_super_sdc", "fstab", "ls_dname", "ls_uuid",
+             "ls_disk_id", "proc_partitions"])
+
+    def test_ptable(self):
+        blkid_info = self.get_blkid_data("blkid_output_sda")
+        self.assertEquals(blkid_info["PTTYPE"], "dos")
+
+    def test_partition_numbers(self):
+        # vde should have partitions 1 and 10
+        disk = "sdd"
+        proc_partitions_path = os.path.join(self.td.collect,
+                                            'proc_partitions')
+        self.assertTrue(os.path.exists(proc_partitions_path))
+        found = []
+        with open(proc_partitions_path, 'r') as fp:
+            for line in fp.readlines():
+                if disk in line:
+                    found.append(line.split()[3])
+        # /proc/partitions should have 3 lines with 'vde' in them.
+        expected = [disk + s for s in ["", "1", "10"]]
+        self.assertEqual(found, expected)
+
+    def test_partitions(self):
+        with open(os.path.join(self.td.collect, "fstab")) as fp:
+            fstab_lines = fp.readlines()
+        print("\n".join(fstab_lines))
+        # Test that vda1 is on /
+        blkid_info = self.get_blkid_data("blkid_output_sda1")
+        fstab_entry = None
+        for line in fstab_lines:
+            if blkid_info['UUID'] in line:
+                fstab_entry = line
+                break
+        self.assertIsNotNone(fstab_entry)
+        self.assertEqual(fstab_entry.split(' ')[1], "/")
+
+        # Test that vda2 is on /home
+        blkid_info = self.get_blkid_data("blkid_output_sda2")
+        fstab_entry = None
+        for line in fstab_lines:
+            if blkid_info['UUID'] in line:
+                fstab_entry = line
+                break
+        self.assertIsNotNone(fstab_entry)
+        self.assertEqual(fstab_entry.split(' ')[1], "/home")
+
+        # Test whole disk sdc is mounted at /btrfs
+        fstab_entry = None
+        for line in fstab_lines:
+            if "/dev/sdc" in line:
+                fstab_entry = line
+                break
+        self.assertIsNotNone(fstab_entry)
+        self.assertEqual(fstab_entry.split(' ')[1], "/btrfs")
+
+    def test_whole_disk_format(self):
+        # confirm the whole disk format is the expected device
+        with open(os.path.join(self.td.collect,
+                  "btrfs_show_super_sdc"), "r") as fp:
+            btrfs_show_super = fp.read()
+
+        with open(os.path.join(self.td.collect, "ls_uuid"), "r") as fp:
+            ls_uuid = fp.read()
+
+        # extract uuid from btrfs superblock
+        btrfs_fsid = [line for line in btrfs_show_super.split('\n')
+                      if line.startswith('fsid\t\t')]
+        self.assertEqual(len(btrfs_fsid), 1)
+        btrfs_uuid = btrfs_fsid[0].split()[1]
+        self.assertTrue(btrfs_uuid is not None)
+
+        # extract uuid from /dev/disk/by-uuid on /dev/sdc
+        # parsing ls -al output on /dev/disk/by-uuid:
+        # lrwxrwxrwx 1 root root   9 Dec  4 20:02
+        #  d591e9e9-825a-4f0a-b280-3bfaf470b83c -> ../../vdg
+        uuid = [line.split()[8] for line in ls_uuid.split('\n')
+                if 'sdc' in line]
+        self.assertEqual(len(uuid), 1)
+        uuid = uuid.pop()
+        self.assertTrue(uuid is not None)
+
+        # compare them
+        self.assertEqual(uuid, btrfs_uuid)
+
+
+class XenialTestScsiBasic(relbase.xenial, TestBasicScsiAbs):
+    __test__ = True

=== modified file 'tests/vmtests/test_bcache_basic.py'
--- tests/vmtests/test_bcache_basic.py	2016-06-03 13:50:09 +0000
+++ tests/vmtests/test_bcache_basic.py	2016-07-12 16:34:37 +0000
@@ -50,3 +50,7 @@
 
 class XenialBcacheBasic(relbase.xenial, TestBcacheBasic):
     __test__ = True
+
+
+class YakketyBcacheBasic(relbase.yakkety, TestBcacheBasic):
+    __test__ = True

=== modified file 'tests/vmtests/test_bonding.py'
--- tests/vmtests/test_bonding.py	2016-06-03 13:50:09 +0000
+++ tests/vmtests/test_bonding.py	2016-07-12 16:34:37 +0000
@@ -1,4 +1,4 @@
-from . import VMBaseClass, logger
+from . import VMBaseClass, logger, helpers
 from .releases import base_vm_classes as relbase
 
 import ipaddress
@@ -8,44 +8,6 @@
 import yaml
 
 
-def iface_extract(input):
-    mo = re.search(r'^(?P<interface>\w+|\w+:\d+)\s+' +
-                   r'Link encap:(?P<link_encap>\S+)\s+' +
-                   r'(HWaddr\s+(?P<mac_address>\S+))?' +
-                   r'(\s+inet addr:(?P<address>\S+))?' +
-                   r'(\s+Bcast:(?P<broadcast>\S+)\s+)?' +
-                   r'(Mask:(?P<netmask>\S+)\s+)?',
-                   input, re.MULTILINE)
-
-    mtu = re.search(r'(\s+MTU:(?P<mtu>\d+)\s+)\s+', input, re.MULTILINE)
-    mtu_info = mtu.groupdict('')
-    mtu_info['mtu'] = int(mtu_info['mtu'])
-
-    if mo:
-        info = mo.groupdict('')
-        info['running'] = False
-        info['up'] = False
-        info['multicast'] = False
-        if 'RUNNING' in input:
-            info['running'] = True
-        if 'UP' in input:
-            info['up'] = True
-        if 'MULTICAST' in input:
-            info['multicast'] = True
-        info.update(mtu_info)
-        return info
-    return {}
-
-
-def ifconfig_to_dict(ifconfig):
-    interfaces = {}
-    for iface in [iface_extract(iface) for iface in ifconfig.split('\n\n')
-                  if iface.strip()]:
-        interfaces[iface['interface']] = iface
-
-    return interfaces
-
-
 class TestNetworkAbs(VMBaseClass):
     interactive = False
     conf_file = "examples/tests/bonding_network.yaml"
@@ -96,7 +58,7 @@
             ifconfig_a = fp.read()
             logger.debug('ifconfig -a:\n{}'.format(ifconfig_a))
 
-        ifconfig_dict = ifconfig_to_dict(ifconfig_a)
+        ifconfig_dict = helpers.ifconfig_to_dict(ifconfig_a)
         logger.debug('parsed ifcfg dict:\n{}'.format(
             yaml.dump(ifconfig_dict, default_flow_style=False, indent=4)))
 
@@ -230,13 +192,13 @@
     __test__ = True
 
 
-class VividTestBonding(relbase.vivid, TestNetworkAbs):
-    __test__ = True
-
-
 class WilyTestBonding(relbase.wily, TestNetworkAbs):
     __test__ = True
 
 
 class XenialTestBonding(relbase.xenial, TestNetworkAbs):
     __test__ = True
+
+
+class YakketyTestBonding(relbase.yakkety, TestNetworkAbs):
+    __test__ = True

=== modified file 'tests/vmtests/test_lvm.py'
--- tests/vmtests/test_lvm.py	2016-06-03 13:50:09 +0000
+++ tests/vmtests/test_lvm.py	2016-07-12 16:34:37 +0000
@@ -63,13 +63,13 @@
         print("test_dname does not work for Trusty")
 
 
-class VividTestLvm(relbase.vivid, TestLvmAbs):
-    __test__ = True
-
-
 class WilyTestLvm(relbase.wily, TestLvmAbs):
     __test__ = True
 
 
 class XenialTestLvm(relbase.xenial, TestLvmAbs):
     __test__ = True
+
+
+class YakketyTestLvm(relbase.yakkety, TestLvmAbs):
+    __test__ = True

=== modified file 'tests/vmtests/test_mdadm_bcache.py'
--- tests/vmtests/test_mdadm_bcache.py	2016-06-03 13:50:09 +0000
+++ tests/vmtests/test_mdadm_bcache.py	2016-07-12 16:34:37 +0000
@@ -130,10 +130,6 @@
     __test__ = True
 
 
-class VividTestMdadmBcache(relbase.vivid, TestMdadmBcacheAbs):
-    __test__ = True
-
-
 class WilyTestMdadmBcache(relbase.wily, TestMdadmBcacheAbs):
     __test__ = True
 
@@ -142,6 +138,10 @@
     __test__ = True
 
 
+class YakketyTestMdadmBcache(relbase.yakkety, TestMdadmBcacheAbs):
+    __test__ = True
+
+
 class TestMirrorbootAbs(TestMdadmAbs):
     # alternative config for more complex setup
     conf_file = "examples/tests/mirrorboot.yaml"
@@ -170,10 +170,6 @@
     __test__ = True
 
 
-class VividTestMirrorboot(relbase.vivid, TestMirrorbootAbs):
-    __test__ = True
-
-
 class WilyTestMirrorboot(relbase.wily, TestMirrorbootAbs):
     __test__ = True
 
@@ -182,6 +178,10 @@
     __test__ = True
 
 
+class YakketyTestMirrorboot(relbase.yakkety, TestMirrorbootAbs):
+    __test__ = True
+
+
 class TestRaid5bootAbs(TestMdadmAbs):
     # alternative config for more complex setup
     conf_file = "examples/tests/raid5boot.yaml"
@@ -211,10 +211,6 @@
     __test__ = True
 
 
-class VividTestRaid5boot(relbase.vivid, TestRaid5bootAbs):
-    __test__ = True
-
-
 class WilyTestRaid5boot(relbase.wily, TestRaid5bootAbs):
     __test__ = True
 
@@ -223,6 +219,10 @@
     __test__ = True
 
 
+class YakketyTestRaid5boot(relbase.yakkety, TestRaid5bootAbs):
+    __test__ = True
+
+
 class TestRaid6bootAbs(TestMdadmAbs):
     # alternative config for more complex setup
     conf_file = "examples/tests/raid6boot.yaml"
@@ -264,10 +264,6 @@
     __test__ = True
 
 
-class VividTestRaid6boot(relbase.vivid, TestRaid6bootAbs):
-    __test__ = True
-
-
 class WilyTestRaid6boot(relbase.wily, TestRaid6bootAbs):
     __test__ = True
 
@@ -276,6 +272,10 @@
     __test__ = True
 
 
+class YakketyTestRaid6boot(relbase.yakkety, TestRaid6bootAbs):
+    __test__ = True
+
+
 class TestRaid10bootAbs(TestMdadmAbs):
     # alternative config for more complex setup
     conf_file = "examples/tests/raid10boot.yaml"
@@ -305,10 +305,6 @@
     __test__ = True
 
 
-class VividTestRaid10boot(relbase.vivid, TestRaid10bootAbs):
-    __test__ = True
-
-
 class WilyTestRaid10boot(relbase.wily, TestRaid10bootAbs):
     __test__ = True
 
@@ -317,6 +313,10 @@
     __test__ = True
 
 
+class YakketyTestRaid10boot(relbase.yakkety, TestRaid10bootAbs):
+    __test__ = True
+
+
 class TestAllindataAbs(TestMdadmAbs):
     # more complex, needs more time
     # alternative config for more complex setup
@@ -403,13 +403,13 @@
     __test__ = False  # lukes=no does not disable mounting of device
 
 
-class VividTestAllindata(relbase.vivid, TestAllindataAbs):
-    __test__ = True
-
-
 class WilyTestAllindata(relbase.wily, TestAllindataAbs):
     __test__ = True
 
 
 class XenialTestAllindata(relbase.xenial, TestAllindataAbs):
     __test__ = True
+
+
+class YakketyTestAllindata(relbase.yakkety, TestAllindataAbs):
+    __test__ = True

=== added file 'tests/vmtests/test_multipath.py'
--- tests/vmtests/test_multipath.py	1970-01-01 00:00:00 +0000
+++ tests/vmtests/test_multipath.py	2016-07-12 16:34:37 +0000
@@ -0,0 +1,63 @@
+from . import VMBaseClass
+from .releases import base_vm_classes as relbase
+
+import os
+import textwrap
+
+
+class TestMultipathBasicAbs(VMBaseClass):
+    conf_file = "examples/tests/multipath.yaml"
+    multipath = True
+    disk_driver = 'scsi-hd'
+    extra_disks = []
+    nvme_disks = []
+    collect_scripts = [textwrap.dedent("""
+        cd OUTPUT_COLLECT_D
+        blkid -o export /dev/sda > blkid_output_sda
+        blkid -o export /dev/sda1 > blkid_output_sda1
+        blkid -o export /dev/sda2 > blkid_output_sda2
+        blkid -o export /dev/sdb > blkid_output_sdb
+        blkid -o export /dev/sdb1 > blkid_output_sdb1
+        blkid -o export /dev/sdb2 > blkid_output_sdb2
+        dmsetup ls > dmsetup_ls
+        dmsetup info > dmsetup_info
+        cat /proc/partitions > proc_partitions
+        multipath -ll > multipath_ll
+        multipath -v3 -ll > multipath_v3_ll
+        multipath -r > multipath_r
+        cp -a /etc/multipath* .
+        ls -al /dev/disk/by-uuid/ > ls_uuid
+        ls -al /dev/disk/by-id/ > ls_disk_id
+        readlink -f /sys/class/block/sda/holders/dm-0 > holders_sda
+        readlink /sys/class/block/sdb/holders/dm-0 > holders_sdb
+        cat /etc/fstab > fstab
+        mkdir -p /dev/disk/by-dname
+        ls /dev/disk/by-dname/ > ls_dname
+        find /etc/network/interfaces.d > find_interfacesd
+        """)]
+
+    def test_multipath_disks_match(self):
+        sda = os.path.join(self.td.collect, 'holders_sda')
+        sdb = os.path.join(self.td.collect, 'holders_sdb')
+        self.assertTrue(os.path.exists(sda))
+        self.assertTrue(os.path.exists(sdb))
+        with open(sda, 'r') as fp:
+            sda_data = fp.read()
+            print('sda holders:\n%s' % sda_data)
+        with open(sda, 'r') as fp:
+            sdb_data = fp.read()
+            print('sdb holders:\n%s' % sda_data)
+
+        self.assertEqual(sda_data, sdb_data)
+
+
+class TrustyTestMultipathBasic(relbase.trusty, TestMultipathBasicAbs):
+    __test__ = True
+
+
+class XenialTestMultipathBasic(relbase.xenial, TestMultipathBasicAbs):
+    __test__ = True
+
+
+class YakketyTestMultipathBasic(relbase.yakkety, TestMultipathBasicAbs):
+    __test__ = True

=== modified file 'tests/vmtests/test_network.py'
--- tests/vmtests/test_network.py	2016-06-03 13:50:09 +0000
+++ tests/vmtests/test_network.py	2016-07-12 16:34:37 +0000
@@ -1,4 +1,4 @@
-from . import VMBaseClass, logger
+from . import VMBaseClass, logger, helpers
 from .releases import base_vm_classes as relbase
 
 import ipaddress
@@ -9,44 +9,6 @@
 import yaml
 
 
-def iface_extract(input):
-    mo = re.search(r'^(?P<interface>\w+|\w+:\d+|\w+\.\d+)\s+' +
-                   r'Link encap:(?P<link_encap>\S+)\s+' +
-                   r'(HWaddr\s+(?P<mac_address>\S+))?' +
-                   r'(\s+inet addr:(?P<address>\S+))?' +
-                   r'(\s+Bcast:(?P<broadcast>\S+)\s+)?' +
-                   r'(Mask:(?P<netmask>\S+)\s+)?',
-                   input, re.MULTILINE)
-
-    mtu = re.search(r'(\s+MTU:(?P<mtu>\d+)\s+)\s+', input, re.MULTILINE)
-    mtu_info = mtu.groupdict('')
-    mtu_info['mtu'] = int(mtu_info['mtu'])
-
-    if mo:
-        info = mo.groupdict('')
-        info['running'] = False
-        info['up'] = False
-        info['multicast'] = False
-        if 'RUNNING' in input:
-            info['running'] = True
-        if 'UP' in input:
-            info['up'] = True
-        if 'MULTICAST' in input:
-            info['multicast'] = True
-        info.update(mtu_info)
-        return info
-    return {}
-
-
-def ifconfig_to_dict(ifconfig):
-    interfaces = {}
-    for iface in [iface_extract(iface) for iface in ifconfig.split('\n\n')
-                  if iface.strip()]:
-        interfaces[iface['interface']] = iface
-
-    return interfaces
-
-
 class TestNetworkAbs(VMBaseClass):
     interactive = False
     conf_file = "examples/tests/basic_network.yaml"
@@ -132,7 +94,7 @@
             ifconfig_a = fp.read()
             logger.debug('ifconfig -a:\n{}'.format(ifconfig_a))
 
-        ifconfig_dict = ifconfig_to_dict(ifconfig_a)
+        ifconfig_dict = helpers.ifconfig_to_dict(ifconfig_a)
         logger.debug('parsed ifcfg dict:\n{}'.format(
             yaml.dump(ifconfig_dict, default_flow_style=False, indent=4)))
 
@@ -340,7 +302,7 @@
             ifconfig_a = fp.read()
             logger.debug('ifconfig -a:\n{}'.format(ifconfig_a))
 
-        ifconfig_dict = ifconfig_to_dict(ifconfig_a)
+        ifconfig_dict = helpers.ifconfig_to_dict(ifconfig_a)
         logger.debug('parsed ifconfig dict:\n{}'.format(
             yaml.dump(ifconfig_dict, default_flow_style=False, indent=4)))
         print('parsed ifconfig dict:\n{}'.format(
@@ -411,14 +373,6 @@
     __test__ = False
 
 
-class VividTestNetwork(relbase.vivid, TestNetworkAbs):
-    __test__ = True
-
-
-class VividTestNetworkStatic(relbase.vivid, TestNetworkStaticAbs):
-    __test__ = True
-
-
 class WilyTestNetwork(relbase.wily, TestNetworkAbs):
     __test__ = True
 
@@ -435,6 +389,14 @@
     __test__ = True
 
 
+class YakketyTestNetwork(relbase.yakkety, TestNetworkAbs):
+    __test__ = True
+
+
+class YakketyTestNetworkStatic(relbase.yakkety, TestNetworkStaticAbs):
+    __test__ = True
+
+
 class PreciseTestNetworkVlan(relbase.precise, TestNetworkVlanAbs):
     __test__ = True
 
@@ -455,10 +417,6 @@
     __test__ = True
 
 
-class VividTestNetworkVlan(relbase.vivid, TestNetworkVlanAbs):
-    __test__ = True
-
-
 class WilyTestNetworkVlan(relbase.wily, TestNetworkVlanAbs):
     __test__ = True
 
@@ -467,6 +425,10 @@
     __test__ = True
 
 
+class YakketyTestNetworkVlan(relbase.yakkety, TestNetworkVlanAbs):
+    __test__ = True
+
+
 class PreciseTestNetworkENISource(relbase.precise, TestNetworkENISource):
     __test__ = False
     # not working, still debugging though; possible older ifupdown doesn't
@@ -477,13 +439,13 @@
     __test__ = True
 
 
-class VividTestNetworkENISource(relbase.vivid, TestNetworkENISource):
-    __test__ = True
-
-
 class WilyTestNetworkENISource(relbase.wily, TestNetworkENISource):
     __test__ = True
 
 
 class XenialTestNetworkENISource(relbase.xenial, TestNetworkENISource):
     __test__ = True
+
+
+class YakketyTestNetworkENISource(relbase.yakkety, TestNetworkENISource):
+    __test__ = True

=== modified file 'tests/vmtests/test_nvme.py'
--- tests/vmtests/test_nvme.py	2016-06-03 13:50:09 +0000
+++ tests/vmtests/test_nvme.py	2016-07-12 16:34:37 +0000
@@ -74,13 +74,13 @@
         print("test_ptable does not work for Trusty")
 
 
-class VividTestNvme(relbase.vivid, TestNvmeAbs):
-    __test__ = True
-
-
 class WilyTestNvme(relbase.wily, TestNvmeAbs):
     __test__ = True
 
 
 class XenialTestNvme(relbase.xenial, TestNvmeAbs):
     __test__ = True
+
+
+class YakketyTestNvme(relbase.yakkety, TestNvmeAbs):
+    __test__ = True

=== modified file 'tests/vmtests/test_raid5_bcache.py'
--- tests/vmtests/test_raid5_bcache.py	2016-06-03 13:50:09 +0000
+++ tests/vmtests/test_raid5_bcache.py	2016-07-12 16:34:37 +0000
@@ -93,9 +93,13 @@
     __test__ = False
 
 
-class VividTestRaid5Bcache(relbase.vivid, TestMdadmBcacheAbs):
-    __test__ = True
-
-
 class WilyTestRaid5Bcache(relbase.wily, TestMdadmBcacheAbs):
     __test__ = True
+
+
+class XenialTestRaid5Bcache(relbase.xenial, TestMdadmBcacheAbs):
+    __test__ = True
+
+
+class YakketyTestRaid5Bcache(relbase.yakkety, TestMdadmBcacheAbs):
+    __test__ = True

=== added file 'tests/vmtests/test_simple.py'
--- tests/vmtests/test_simple.py	1970-01-01 00:00:00 +0000
+++ tests/vmtests/test_simple.py	2016-07-12 16:34:37 +0000
@@ -0,0 +1,40 @@
+from . import VMBaseClass
+from .releases import base_vm_classes as relbase
+
+import textwrap
+
+
+class TestSimple(VMBaseClass):
+    # Test that curtin with no config does the right thing
+    conf_file = "examples/tests/simple.yaml"
+    extra_disks = []
+    extra_nics = []
+    collect_scripts = [textwrap.dedent("""
+        cd OUTPUT_COLLECT_D
+        sfdisk --list > sfdisk_list
+        for d in /dev/[sv]d[a-z] /dev/xvd?; do
+            [ -b "$d" ] || continue
+            echo == $d ==
+            sgdisk --print $d
+        done > sgdisk_list
+        blkid > blkid
+        cat /proc/partitions > proc_partitions
+        cp /etc/network/interfaces interfaces
+        if [ -f /var/log/cloud-init-output.log ]; then
+           cp /var/log/cloud-init-output.log .
+        fi
+        cp /var/log/cloud-init.log .
+        find /etc/network/interfaces.d > find_interfacesd
+        """)]
+
+    def test_output_files_exist(self):
+        self.output_files_exist(["sfdisk_list", "blkid",
+                                 "proc_partitions", "interfaces"])
+
+
+class TrustyTestSimple(relbase.trusty, TestSimple):
+    __test__ = True
+
+
+class XenialTestSimple(relbase.xenial, TestSimple):
+    __test__ = True

=== modified file 'tests/vmtests/test_uefi_basic.py'
--- tests/vmtests/test_uefi_basic.py	2016-06-03 13:50:09 +0000
+++ tests/vmtests/test_uefi_basic.py	2016-07-12 16:34:37 +0000
@@ -108,14 +108,14 @@
     __test__ = True
 
 
-class VividUefiTestBasic(relbase.vivid, TestBasicAbs):
-    __test__ = True
-
-
 class XenialUefiTestBasic(relbase.xenial, TestBasicAbs):
     __test__ = True
 
 
+class YakketyUefiTestBasic(relbase.yakkety, TestBasicAbs):
+    __test__ = True
+
+
 class PreciseUefiTestBasic4k(PreciseUefiTestBasic):
     disk_block_size = 4096
 
@@ -124,13 +124,13 @@
     disk_block_size = 4096
 
 
-class VividUefiTestBasic4k(VividUefiTestBasic):
-    disk_block_size = 4096
-
-
 class WilyUefiTestBasic4k(WilyUefiTestBasic):
     disk_block_size = 4096
 
 
 class XenialUefiTestBasic4k(XenialUefiTestBasic):
     disk_block_size = 4096
+
+
+class YakketyUefiTestBasic4k(YakketyUefiTestBasic):
+    disk_block_size = 4096

=== modified file 'tools/jenkins-runner'
--- tools/jenkins-runner	2016-02-12 21:54:46 +0000
+++ tools/jenkins-runner	2016-07-12 16:34:37 +0000
@@ -7,6 +7,7 @@
 export CURTIN_VMTEST_KEEP_DATA_FAIL=$fkeep
 export CURTIN_VMTEST_TOPDIR="$topdir"
 export CURTIN_VMTEST_LOG=${CURTIN_VMTEST_LOG:-"$topdir/debug.log"}
+export IMAGE_DIR=${IMAGE_DIR:-/srv/images}
 
 fail() { echo "$@" 1>&2; exit 1; }
 
@@ -24,6 +25,15 @@
 for v in ${!CURTIN_*}; do
    echo "$v=${!v}"
 done
+
+# avoid LOG info by running python3 tests/vmtests/image_sync.py
+# rather than python3 -m tests.vmtests.image_sync (LP: #1594465)
+echo "Working with images in $IMAGE_DIR"
+fmt="  %(release)-7s %(arch)s/%(subarch)s %(version_name)-10s"
+PYTHONPATH="$PWD" python3 tests/vmtests/image_sync.py query \
+    --output-format="$fmt" "$IMAGE_DIR" ftype=root-image.gz ||
+    { echo "WARNING: error querying images in $IMAGE_DIR" 1>&2; }
+
 echo "$(date -R): vmtest start: nosetests3 $*"
 nosetests3 "$@"
 ret=$?

=== modified file 'tools/launch'
--- tools/launch	2016-05-10 16:13:29 +0000
+++ tools/launch	2016-07-12 16:34:37 +0000
@@ -8,6 +8,7 @@
 HTTP_PORT_MIN=${HTTP_PORT_MIN:-12000}
 HTTP_PORT_MAX=${HTTP_PORT_MAX:-65500}
 MY_D=$(dirname "$0")
+DEFAULT_ROOT_ARG="root=LABEL=cloudimg-rootfs"
 
 error() { echo "$@" 1>&2; }
 
@@ -15,19 +16,24 @@
     cat <<EOF
 Usage: ${0##*/} [ options ] boot-image curtin install [args]
 
-   boot the image 'boot-image', so that it will run 
+   boot the image 'boot-image', so that it will run
       curtin install [args]
    after booting. 'curtin install [args]' can be any command'
-  
+
    options:
            --add  F[:T] add file 'F' to the curtin archive at T
       -a | --append     append args to kernel cmdline (--kernel)
       -A | --arch   A   assume guest kernel architecture A
-      -d | --disk   D   add a disk 'D' format (path[:size][:driver][:bsize])
+      -d | --disk   D   add a disk 'D' format
+                        (path[:size][:driver][:bsize][:devopts])
                         driver can be specified without size using path::driver
                         driver defaults to virtio-blk
                         bsize <logical>[,<physical>][,<min_io_size>]
                         bsize defaults to 512b sector size
+                        opts is a comma delimitted list of property=value
+                        elements. Examine qemu-kvm -device scsi-hd,? for
+                        details.
+           --vnc D      use -vnc D (mutually exclusive with --silent)
            --uefi   N   enable uefi boot method, store nvram at N
       -h | --help       show this message
       -i | --initrd F   use initramfs F
@@ -40,6 +46,8 @@
                         directly through to qemu-system.
                         Note, qemu adds 5900 to port numbers. (:0 = port 5900)
            --serial-log F  : log to F (default 'serial.log')
+           --root-arg X pass 'X' through as the root= param when booting a
+                        kernel.  default: $DEFAULT_ROOT_PARAM
       -v | --verbose    be more verbose
            --no-install-deps  do not install insert '--install-deps'
                               on curtin command invocations
@@ -188,7 +196,7 @@
     # starts a web service at 'port' that serves files in 'pubdir'
     # waits until it is verified to be lisenting at ip
     # if port is not provided, '$tries' random ports are tried.
-    # 
+    #
     # sets HTTP_PID and returns in _RET the port selected.
     local pubdir="$1" ip="$2" port="$3" tries="${4:-5}" i=""
     [ -z "$ip" ] && ip="localhost"
@@ -212,7 +220,7 @@
         [ "$port" = "random" ] && port=$(($pmin+($RANDOM%($pmax+1-$pmin))))
         debug 2 "trying http server $ip:$port"
         _start_http "$pubdir" "$ip" "$port" &&
-            HTTP_PID="$_RET" && _RET="$port" && 
+            HTTP_PID="$_RET" && _RET="$port" &&
             debug 1 "serving $pubdir at http://$ip:$port/ in pid $HTTP_PID" &&
             return 0
         ret=$?
@@ -241,7 +249,7 @@
 
 main() {
     local short_opts="a:A:d:h:i:k:n:p:v"
-    local long_opts="add:,append:,arch:,bios:,disk:,dowait,help,initrd:,kernel:,mem:,netdev:,no-dowait,power:,publish:,silent,serial-log:,uefi:,verbose,vnc:"
+    local long_opts="add:,append:,arch:,bios:,disk:,dowait,help,initrd:,kernel:,mem:,netdev:,no-dowait,power:,publish:,root-arg:,silent,serial-log:,uefi:,verbose,vnc:"
     local getopt_out=""
     getopt_out=$(getopt --name "${0##*/}" \
         --options "${short_opts}" --long "${long_opts}" -- "$@") &&
@@ -258,6 +266,7 @@
     local netdevs="" install_deps="--install-deps"
     local arch_hint=""
     local video="-curses -vga std" serial_log="serial.log"
+    local root_arg="$DEFAULT_ROOT_ARG"
     # dowait: run xkvm with a '&' and then 'wait' on the pid.
     #  the reason to do this or not do this has to do with interactivity
     #  if detached with &, then user input will not go to xkvm.
@@ -280,13 +289,17 @@
         case "$cur" in
                --add) addfiles[${#addfiles[@]}]="$next"; shift;;
             -a|--append) uappend="$next"; shift;;
-            -A|--arch) arch_hint="$next"; shift;;	       
+            -A|--arch) arch_hint="$next"; shift;;
+               --bios) bios="$2"; shift;;
             -d|--disk) disks[${#disks[@]}]="$next"; shift;;
+               --dowait) pt[${#pt[@]}]="$cur"; dowait=true;;
             -h|--help) Usage ; exit 0;;
             -i|--initrd) initrd="$next"; shift;;
-               --no-install-deps) install_deps="";;
             -k|--kernel) kernel="$next"; shift;;
                --mem) mem="$next"; shift;;
+            -n|--netdev) netdevs[${#netdevs[@]}]="$next"; shift;;
+               --no-dowait) pt[${#pt[@]}]="$cur"; dowait=false;;
+               --no-install-deps) install_deps="";;
                --power)
                 case "$next" in
                     off) pstate="poweroff";;
@@ -295,19 +308,16 @@
                     *) error "Invalid power state, must be: off, on, reboot";;
                 esac
                 shift;;
-            -n|--netdev) netdevs[${#netdevs[@]}]="$next"; shift;;
             -p|--publish) pubs[${#pub[@]}]="$next"; shift;;
+               --root-arg) root_arg="$next";;
+               --serial-log) serial_log="$next"; shift;;
+               --silent) video="-nographic";;
                --uefi) uefi="$2"; shift;;
-               --bios) bios="$2"; shift;;
-               --silent) video="-nographic";;
+            -v|--verbose) VERBOSITY=$((${VERBOSITY}+1));;
                --vnc)
                     video="-vnc $next"
                     debug 1 "VNC requested - $next"
                     shift;;
-               --serial-log) serial_log="$next"; shift;;
-            -v|--verbose) VERBOSITY=$((${VERBOSITY}+1));;
-               --dowait) pt[${#pt[@]}]="$cur"; dowait=true;;
-               --no-dowait) pt[${#pt[@]}]="$cur"; dowait=false;;
             --) shift; break;;
         esac
         shift;
@@ -336,10 +346,10 @@
     if [ -n "$bios" ]; then
         bios_opts=(  -drive "if=pflash,format=raw,file=$bios" )
     elif [ -n "$uefi" ]; then
-        case `lsb_release -sc` in 
+        case `lsb_release -sc` in
             precise|trusty|vivid)
                 # for non-split UEFI firmware, the code and
-                # var space are in the same file.  We must 
+                # var space are in the same file.  We must
                 # make a copy so we can retain modifications.
                 local ovmf_code="/usr/share/ovmf/OVMF.fd"
                 local ovmf_var=$ovmf_code
@@ -388,21 +398,22 @@
         { error "failed to get dir for $0"; return 1; }
 
     local disk="" src="" size="" fmt="" out="" id="" driver="" if=""
-    local split_input=""
+    local split_input="" serial=""
     disk_args=( )
     id=1
     for disk in "${disks[@]}"; do
         ((id++))
-
         # 1=src
         # 2=src:size
         # 3=src:size:driver
         # 4=src:size:driver:bsize
+        # 5=src:size:driver:bsize:devopts
         src=$(echo $disk | awk -F: '{print $1}')
         size=$(echo $disk | awk -F: '{print $2}')
         driver=$(echo $disk | awk -F: '{print $3}')
         bsize=$(echo $disk | awk -F: '{print $4}')
-        
+        devopts=$(echo $disk | awk -F: '{print $5}')
+
         if [ -z "${src}" ]; then
             error "Failed to provide disk source"
             exit 1
@@ -415,7 +426,7 @@
         if [ -z "${driver}" ]; then
             driver="virtio-blk"
         fi
-        
+
         if [ -z "${bsize}" ]; then
             bsize="512"
         fi
@@ -430,6 +441,11 @@
                 { error "failed to determine format of $src"; return 1; }
         fi
 
+        # prepend comma if passing devopts
+        if [ -n "${devopts}" ]; then
+            devopts=",${devopts}"
+        fi
+
         # set logical/physical size blocksz is logical:phys
         local logbs=$(round_up ${bsize%%:*})
         local phybs=$(round_up ${bsize##*:})
@@ -441,8 +457,8 @@
             "file=${src},if=none,cache=unsafe,format=$fmt,id=drv${id},index=$id" )
 
         disk_args=( "${disk_args[@]}" "-device"
-            "${driver},drive=drv${id},serial=dev${id},${bs_args}" )
-        
+            "${driver},drive=drv${id},${bs_args}${devopts}" )
+
     done
 
     get_my_ip || { error "failed to get your ip. set IP_ADDR"; return 1; }
@@ -554,17 +570,24 @@
     seedargs=()
     if [ -n "$kernel" ]; then
         local append="" root=""
-        # if this is a partition image, root=/dev/vda. else root=/dev/vda1
-        # this hack is necessary because LABEL even UUID  might be the same
-        # in the boot image and the target (if re-using target)
-        if tmp=$(blkid "$bootimg_dist" -ovalue -s UUID) && [ -n "$tmp" ]; then
-            root="/dev/vda"
-        else
-            root="/dev/vda1"
+        # Note: root_arg is by default done by LABEL.  This assumes
+        # a.) our root device is not multipath
+        # b.) no other disks attached will have this LABEL
+        # c.) the LABEL is in fact correct.
+        # all of these assumptions are true under vmtest.
+        if [ -z "$root_arg" ]; then
+            debug 1 "WARN: root_arg is empty with kernel."
         fi
-        append="root=$root ds=nocloud-net;seedfrom=$burl"
-        if [ "${arch_hint}" != "s390x" ]; then
-            append="${append} console=ttyS0"
+        append="${root_arg:+${root_arg} }ds=nocloud-net;seedfrom=$burl"
+
+        local console_name=""
+        case "${arch_hint}" in
+            s390x) console_name="";;
+            ppc64*) console_name="hvc0";;
+            *) console_name="ttyS0";;
+        esac
+        if [ -n "$console_name" ]; then
+            append="${append} console=${console_name}"
         fi
         append="${append} $uappend"
         seedargs=( "${seedargs[@]}" -kernel "$kernel" )
@@ -586,18 +609,20 @@
     local cmd serial_args="" chardev_arg=""
     [ "${serial_log}" = "none" ] && serial_log=""
     if [ -n "${serial_log}" ]; then
-	if [ "${arch_hint}" = "s390x" ]; then
-	    if [ "${serial_log}" = "stdio" ]; then
-		chardev_arg="stdio"
-	    else
-		chardev_arg="file,path=${serial_log}"
-	    fi
-	    serial_args="-nodefaults -chardev ${chardev_arg},id=charconsole0 -device sclpconsole,chardev=charconsole0,id=console0"
-	else
-	    serial_args="-serial file:${serial_log}"
-	fi
+        if [ "${arch_hint}" = "s390x" ]; then
+            if [ "${serial_log}" = "stdio" ]; then
+                chardev_arg="stdio"
+            else
+                chardev_arg="file,path=${serial_log}"
+            fi
+                serial_args="-nodefaults -chardev ${chardev_arg},id=charconsole0 -device sclpconsole,chardev=charconsole0,id=console0"
+        else
+            serial_args="-serial file:${serial_log}"
+            #debug mode serial_args="-serial ${serial_log} -monitor stdio"
+        fi
     fi
-    cmd=( 
+    # -monitor stdio
+    cmd=(
         xkvm "${pt[@]}" "${netargs[@]}" --
         "${bios_opts[@]}"
         -m ${mem} ${serial_args} ${video}
@@ -625,6 +650,11 @@
     return $ret
 }
 
+random_wwn() {
+    # wwn must be a int64, less than (1 << 63) - 1
+    # we achieve this by combining 4 (1 << 15) ints
+    printf "0x%04x%04x%04x%04x" $RANDOM $RANDOM $RANDOM $RANDOM
+}
 
 round_up() {
     local size="${1}"

=== modified file 'tools/vmtest-sync-images'
--- tools/vmtest-sync-images	2016-05-10 16:13:29 +0000
+++ tools/vmtest-sync-images	2016-07-12 16:34:37 +0000
@@ -34,7 +34,11 @@
                         print(" removing vmtest file %s" % fpath)
                         os.unlink(fpath)
 
-    releases = find_releases()
+    arg_releases = [r for r in sys.argv[1:] if r != "--clean"]
+    if len(arg_releases):
+        releases = arg_releases
+    else:
+        releases = find_releases()
     release_filter = 'release~{}'.format('|'.join(releases))
     my_filters = ['arch=' + DEFAULT_ARCH, release_filter] + ITEM_NAME_FILTERS
     # Sync images.

=== modified file 'tools/xkvm'
--- tools/xkvm	2016-05-10 16:13:29 +0000
+++ tools/xkvm	2016-07-12 16:34:37 +0000
@@ -77,14 +77,14 @@
 
    NETDEV:
     Above, 'NETDEV' is a comma delimited string
-    The first field must be 
+    The first field must be
      * bridge name: (br0 or virbr0): attach a device to this bridge
      * literal 'user': use qemu user networking
-    
+
     Additional fields are optional, and can be anything that is acceptable
     to kvm either as:
       * '-device virtio-net-pci' option (see 'kvm -device virtio-net-pci,?')
-      * '-net [user|tap]' option 
+      * '-net [user|tap]' option
 
    Example:
      * xkvm --netdev br0,macaddr=:05 -- -drive file=disk.img,if=virtio -curses
@@ -345,7 +345,7 @@
                 if=sd|if=mtd|floppy) fail "do not know what to do with $tok on $cur";;
                 id=*) id=$val;;
                 file=*) file=$val;;
-                format=*) fmt=$val;;
+                fmt=*|format=*) fmt=$val;;
                 serial=*) serial=$val;;
                 bus=*) bus=$val;;
                 unit=*) unit=$val;;
@@ -357,6 +357,8 @@
             out=$(LANG=C qemu-img info "$file") &&
                 fmt=$(echo "$out" | awk '$0 ~ /^file format:/ { print $3 }') ||
                 { error "failed to determine format of $file"; return 1; }
+        else
+            fmt=raw
         fi
         if [ -z "$driver" ]; then
             driver="$def_disk_driver"
@@ -379,7 +381,8 @@
         devopts="$driver,drive=$id${serial:+,serial=${serial}}"
         for tok in "$@"; do
             case "$tok" in
-                id=*|if=*|driver=*|$file) continue;;
+                id=*|if=*|driver=*|$file|file=*) continue;;
+                fmt=*|format=*) continue;;
                 serial=*|bus=*|unit=*|index=*) continue;;
             esac
             isdevopt "$driver" "$tok" && devopts="${devopts},$tok" ||
@@ -529,7 +532,7 @@
 
     netargs=()
     for((i=0;i<${#device_args[@]};i++)); do
-        netargs=( "${netargs[@]}" -device "${device_args[$i]}" 
+        netargs=( "${netargs[@]}" -device "${device_args[$i]}"
                   -netdev "${netdev_args[$i]}")
     done