Merge ~raharper/cloud-init:fix/sync-xkvm into cloud-init:master

[Ryan Harper]

cloud_tests: updates and fixes

- Update paramiko and cryptography module versions (2.4.2) to
  address issues with algo and deprecation warnings.
- Modify ssh keypair generation to work with updated paramiko
- tools/xkvm sync with newer version from curtin
- Update NoCloudKvm instance.py to work with updated xkvm
  - pass -name to instance, useful for debugging on shared host
  - Add cache_mode platform config; default to cache=none,aio=native
- Switch to yaml.safe_load() in platforms.py

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:afd093ba08ba24cd27271de252c5c499e196651d
https://jenkins.ubuntu.com/server/job/cloud-init-ci/797/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Unit & Style Tests
    SUCCESS: Ubuntu LTS: Build
    SUCCESS: Ubuntu LTS: Integration
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/797/rebuild

review: Approve (continuous-integration)

[Joshua Powers (powersj)]

LGTM thanks for the clean up!

review: Approve

diff --git a/integration-requirements.txt b/integration-requirements.txt
index 880d988..fe5ad45 100644
--- a/integration-requirements.txt
+++ b/integration-requirements.txt
@@ -10,7 +10,8 @@ unittest2
 boto3==1.5.9
 
 # ssh communication
-paramiko==2.4.1
+paramiko==2.4.2
+cryptography==2.4.2
 
 
 # lxd backend
diff --git a/tests/cloud_tests/platforms.yaml b/tests/cloud_tests/platforms.yaml
index 448aa98..652a705 100644
--- a/tests/cloud_tests/platforms.yaml
+++ b/tests/cloud_tests/platforms.yaml
@@ -66,5 +66,6 @@ platforms:
                 {{ config_get("user.vendor-data", properties.default) }}
     nocloud-kvm:
         enabled: true
+        cache_mode: cache=none,aio=native
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/platforms/nocloudkvm/instance.py b/tests/cloud_tests/platforms/nocloudkvm/instance.py
index 33ff3f2..96185b7 100644
--- a/tests/cloud_tests/platforms/nocloudkvm/instance.py
+++ b/tests/cloud_tests/platforms/nocloudkvm/instance.py
@@ -74,6 +74,8 @@ class NoCloudKVMInstance(Instance):
         self.pid_file = None
         self.console_file = None
         self.disk = image_path
+        self.cache_mode = platform.config.get('cache_mode',
+                                              'cache=none,aio=native')
         self.meta_data = meta_data
 
     def shutdown(self, wait=True):
@@ -113,7 +115,10 @@ class NoCloudKVMInstance(Instance):
                 pass
 
         if self.pid_file:
-            os.remove(self.pid_file)
+            try:
+                os.remove(self.pid_file)
+            except Exception:
+                pass
 
         self.pid = None
         self._ssh_close()
@@ -160,13 +165,13 @@ class NoCloudKVMInstance(Instance):
         self.ssh_port = self.get_free_port()
 
         cmd = ['./tools/xkvm',
-               '--disk', '%s,cache=unsafe' % self.disk,
-               '--disk', '%s,cache=unsafe' % seed,
+               '--disk', '%s,%s' % (self.disk, self.cache_mode),
+               '--disk', '%s' % seed,
                '--netdev', ','.join(['user',
                                      'hostfwd=tcp::%s-:22' % self.ssh_port,
                                      'dnssearch=%s' % CI_DOMAIN]),
                '--', '-pidfile', self.pid_file, '-vnc', 'none',
-               '-m', '2G', '-smp', '2', '-nographic',
+               '-m', '2G', '-smp', '2', '-nographic', '-name', self.name,
                '-serial', 'file:' + self.console_file]
         subprocess.Popen(cmd,
                          close_fds=True,
diff --git a/tests/cloud_tests/platforms/platforms.py b/tests/cloud_tests/platforms/platforms.py
index abbfebb..bebdf1c 100644
--- a/tests/cloud_tests/platforms/platforms.py
+++ b/tests/cloud_tests/platforms/platforms.py
@@ -48,7 +48,7 @@ class Platform(object):
         if os.path.exists(filename):
             c_util.del_file(filename)
 
-        c_util.subp(['ssh-keygen', '-t', 'rsa', '-b', '4096',
+        c_util.subp(['ssh-keygen', '-m', 'PEM', '-t', 'rsa', '-b', '4096',
                      '-f', filename, '-P', '',
                      '-C', 'ubuntu@cloud_test'],
                     capture=True)
diff --git a/tests/cloud_tests/setup_image.py b/tests/cloud_tests/setup_image.py
index 39f4517..a8aaba1 100644
--- a/tests/cloud_tests/setup_image.py
+++ b/tests/cloud_tests/setup_image.py
@@ -222,7 +222,8 @@ def setup_image(args, image):
              for name, func, desc in handlers if getattr(args, name, None)]
 
     try:
-        data = yaml.load(image.read_data("/etc/cloud/build.info", decode=True))
+        data = yaml.safe_load(
+            image.read_data("/etc/cloud/build.info", decode=True))
         info = ' '.join(["%s=%s" % (k, data.get(k))
                          for k in ("build_name", "serial") if k in data])
     except Exception as e:
diff --git a/tools/xkvm b/tools/xkvm
index a30ba91..8d44cad 100755
--- a/tools/xkvm
+++ b/tools/xkvm
@@ -1,4 +1,6 @@
 #!/bin/bash
+# This file is part of cloud-init.
+# See LICENSE file for copyright and license info.
 
 set -f
 
@@ -11,6 +13,8 @@ TAPDEVS=( )
 # OVS_CLEANUP gets populated with bridge:devname pairs used with ovs
 OVS_CLEANUP=( )
 MAC_PREFIX="52:54:00:12:34"
+# allow this to be set externally.
+_QEMU_SUPPORTS_FILE_LOCKING="${_QEMU_SUPPORTS_FILE_LOCKING}"
 KVM="kvm"
 declare -A KVM_DEVOPTS
 
@@ -119,6 +123,21 @@ isdevopt() {
     return 1
 }
 
+qemu_supports_file_locking() {
+    # hackily check if qemu has file.locking in -drive params (LP: #1716028)
+    if [ -z "$_QEMU_SUPPORTS_FILE_LOCKING" ]; then
+        # The only way we could find to check presense of file.locking is
+        # qmp (query-qmp-schema).  Simply checking if the virtio-blk driver
+        # supports 'share-rw' is expected to be equivalent and simpler.
+        isdevopt virtio-blk share-rw &&
+            _QEMU_SUPPORTS_FILE_LOCKING=true ||
+            _QEMU_SUPPORTS_FILE_LOCKING=false
+        debug 1 "qemu supports file locking = ${_QEMU_SUPPORTS_FILE_LOCKING}"
+    fi
+    [ "$_QEMU_SUPPORTS_FILE_LOCKING" = "true" ]
+    return
+}
+
 padmac() {
     # return a full mac, given a subset.
     # assume whatever is input is the last portion to be
@@ -367,7 +386,7 @@ main() {
     [ ${#netdevs[@]} -eq 0 ] && netdevs=( "${DEF_BRIDGE}" )
     pt=( "$@" )
 
-    local kvm_pkg="" virtio_scsi_bus="virtio-scsi-pci"
+    local kvm_pkg="" virtio_scsi_bus="virtio-scsi-pci" virtio_rng_device="virtio-rng-pci"
     [ -n "$kvm" ] && kvm_pkg="none"
     case $(uname -m) in
         i?86)
@@ -382,7 +401,10 @@ main() {
             [ -n "$kvm" ] ||
                 { kvm="qemu-system-s390x"; kvm_pkg="qemu-system-misc"; }
             def_netmodel=${DEF_NETMODEL:-"virtio-net-ccw"}
+            # disable virtio-scsi-bus
             virtio_scsi_bus="virtio-scsi-ccw"
+            virtio_blk_bus="virtio-blk-ccw"
+            virtio_rng_device="virtio-rng-ccw"
             ;;
         ppc64*)
             [ -n "$kvm" ] ||
@@ -408,7 +430,7 @@ main() {
     bios_opts=( "${_RET[@]}" )
 
     local out="" fmt="" bus="" unit="" index="" serial="" driver="" devopts=""
-    local busorindex="" driveopts="" cur="" val="" file=""
+    local busorindex="" driveopts="" cur="" val="" file="" wwn=""
     for((i=0;i<${#diskdevs[@]};i++)); do
         cur=${diskdevs[$i]}
         IFS=","; set -- $cur; IFS="$oifs"
@@ -420,6 +442,7 @@ main() {
         unit=""
         index=""
         serial=""
+        wwn=""
         for tok in "$@"; do
             [ "${tok#*=}" = "${tok}" -a -f "${tok}" -a -z "$file" ] && file="$tok"
             val=${tok#*=}
@@ -433,6 +456,7 @@ main() {
                 file=*) file=$val;;
                 fmt=*|format=*) fmt=$val;;
                 serial=*) serial=$val;;
+                wwn=*) wwn=$val;;
                 bus=*) bus=$val;;
                 unit=*) unit=$val;;
                 index=*) index=$val;;
@@ -443,14 +467,19 @@ main() {
             out=$(LANG=C qemu-img info "$file") &&
                 fmt=$(echo "$out" | awk '$0 ~ /^file format:/ { print $3 }') ||
                 { error "failed to determine format of $file"; return 1; }
-        else
+        elif [ -z "$fmt" ]; then
             fmt=raw
         fi
         if [ -z "$driver" ]; then
             driver="$def_disk_driver"
         fi
         if [ -z "$serial" ]; then
-            serial="${file##*/}"
+            # use filename as serial if not provided a wwn
+            if [ -n "$wwn" ]; then
+                serial="$wwn"
+            else
+                serial="${file##*/}"
+            fi
         fi
 
         # make sure we add either bus= or index=
@@ -470,11 +499,21 @@ main() {
                 id=*|if=*|driver=*|$file|file=*) continue;;
                 fmt=*|format=*) continue;;
                 serial=*|bus=*|unit=*|index=*) continue;;
+                file.locking=*)
+                    qemu_supports_file_locking || {
+                        debug 2 "qemu has no file locking." \
+                            "Dropping '$tok' from: $cur"
+                        continue
+                    };;
             esac
             isdevopt "$driver" "$tok" && devopts="${devopts},$tok" ||
                 diskopts="${diskopts},${tok}"
         done
-
+        case $driver in
+            virtio-blk-ccw)
+                # disable scsi when using virtio-blk-ccw
+                devopts="${devopts},scsi=off";;
+        esac
         diskargs=( "${diskargs[@]}" -drive "$diskopts" -device "$devopts" )
     done
 
@@ -623,10 +662,16 @@ main() {
     done
 
     local bus_devices
-    bus_devices=( -device "$virtio_scsi_bus,id=virtio-scsi-xkvm" )
-    cmd=( "${kvmcmd[@]}" "${archopts[@]}" 
+    if [ -n "${virtio_scsi_bus}" ]; then
+        bus_devices=( -device "$virtio_scsi_bus,id=virtio-scsi-xkvm" )
+    fi
+    local rng_devices
+    rng_devices=( -object "rng-random,filename=/dev/urandom,id=objrng0"
+                  -device "$virtio_rng_device,rng=objrng0,id=rng0" )
+    cmd=( "${kvmcmd[@]}" "${archopts[@]}"
           "${bios_opts[@]}"
           "${bus_devices[@]}"
+          "${rng_devices[@]}"
           "${netargs[@]}"
           "${diskargs[@]}" "${pt[@]}" )
     local pcmd=$(quote_cmd "${cmd[@]}")
@@ -661,4 +706,4 @@ else
     main "$@"
 fi
 
-# vi: ts=4 expandtab
+# vi: ts=4 expandtab syntax=sh