Merge ~smoser/cloud-init:cleanup/test-more-files-collect into cloud-init:master

[Scott Moser]

tests: Collect script output as binary, collect systemd journal, fix lxd.

This adds collection a gzip compressed systemd journal on systemd systems.
The file can later be reviewed with:
  zcat system.journal.gz > system.journal
  journalctl --file=system.journal [-o short-monotonic ..]

To support this:
  * modify test harness infrastructure to not assume content is utf-8.
  * fix lxd platform to support make '_execute' return bytes rather
    than a string. https://github.com/lxc/pylxd/issues/268

Also switched the base collectors to use /bin/sh as others already did.

see commit message

[Ryan Harper (raharper)]

Add a comment that you switched to using /bin/sh instead of bash in several scripts. Some comments inline.

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:08639703c83e19242059c2d73df373a05e89746a
https://jenkins.ubuntu.com/server/job/cloud-init-ci/720/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Unit & Style Tests
    SUCCESS: Ubuntu LTS: Build
    SUCCESS: Ubuntu LTS: Integration
    SUCCESS: MAAS Compatability Testing
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/720/rebuild

review: Approve (continuous-integration)

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:b9a418a6795730bb7b88f1273d12ab5cafa29cc6
https://jenkins.ubuntu.com/server/job/cloud-init-ci/724/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Unit & Style Tests
    SUCCESS: Ubuntu LTS: Build
    SUCCESS: Ubuntu LTS: Integration
    SUCCESS: MAAS Compatability Testing
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/724/rebuild

review: Approve (continuous-integration)

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:4bcbc7bd9dea686a62576b28bc8b854a992c3221
https://jenkins.ubuntu.com/server/job/cloud-init-ci/726/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Unit & Style Tests
    SUCCESS: Ubuntu LTS: Build
    SUCCESS: Ubuntu LTS: Integration
    SUCCESS: MAAS Compatability Testing
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/726/rebuild

review: Approve (continuous-integration)

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:e975f1b176ab722a09abd387bff00efc2c40358c
https://jenkins.ubuntu.com/server/job/cloud-init-ci/729/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Unit & Style Tests
    SUCCESS: Ubuntu LTS: Build
    SUCCESS: Ubuntu LTS: Integration
    SUCCESS: MAAS Compatability Testing
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/729/rebuild

review: Approve (continuous-integration)

[Ryan Harper (raharper)]

review: Approve

[Chad Smith (chad.smith)]

+1 on this thanks.

review: Approve

diff --git a/tests/cloud_tests/collect.py b/tests/cloud_tests/collect.py
index 33acbb1..5ea88e5 100644
--- a/tests/cloud_tests/collect.py
+++ b/tests/cloud_tests/collect.py
@@ -24,6 +24,13 @@ def collect_script(instance, base_dir, script, script_name):
     (out, err, exit) = instance.run_script(
         script.encode(), rcs=False,
         description='collect: {}'.format(script_name))
+    if err:
+        LOG.debug("collect script %s had stderr: %s", script_name, err)
+    if not isinstance(out, bytes):
+        raise util.PlatformError(
+            "Collection of '%s' returned type %s, expected bytes: %s" %
+            (script_name, type(out), out))
+
     c_util.write_file(os.path.join(base_dir, script_name), out)
 
 
diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py
index 0d697c0..d2d2a1f 100644
--- a/tests/cloud_tests/platforms/lxd/instance.py
+++ b/tests/cloud_tests/platforms/lxd/instance.py
@@ -6,6 +6,8 @@ import os
 import shutil
 from tempfile import mkdtemp
 
+from cloudinit.util import subp, ProcessExecutionError
+
 from ..instances import Instance
 
 
@@ -29,6 +31,7 @@ class LXDInstance(Instance):
             platform, name, properties, config, features)
         self.tmpd = mkdtemp(prefix="%s-%s" % (type(self).__name__, name))
         self._setup_console_log()
+        self.name = name
 
     @property
     def pylxd_container(self):
@@ -55,33 +58,25 @@ class LXDInstance(Instance):
         if env is None:
             env = {}
 
-        if stdin is not None:
-            # pylxd does not support input to execute.
-            # https://github.com/lxc/pylxd/issues/244
-            #
-            # The solution here is write a tmp file in the container
-            # and then execute a shell that sets it standard in to
-            # be from that file, removes it, and calls the comand.
-            tmpf = self.tmpfile()
-            self.write_data(tmpf, stdin)
-            ncmd = 'exec <"{tmpf}"; rm -f "{tmpf}"; exec "$@"'
-            command = (['sh', '-c', ncmd.format(tmpf=tmpf), 'stdinhack'] +
-                       list(command))
+        env_args = []
+        if env:
+            env_args = ['env'] + ["%s=%s" for k, v in env.items()]
 
         # ensure instance is running and execute the command
         self.start()
-        # execute returns a ContainerExecuteResult, named tuple
-        # (exit_code, stdout, stderr)
-        res = self.pylxd_container.execute(command, environment=env)
-
-        # get out, exit and err from pylxd return
-        if not hasattr(res, 'exit_code'):
-            # pylxd 2.1.3 and earlier only return out and err, no exit
-            raise RuntimeError(
-                "No 'exit_code' in pylxd.container.execute return.\n"
-                "pylxd > 2.2 is required.")
-
-        return res.stdout, res.stderr, res.exit_code
+
+        # Use cmdline client due to https://github.com/lxc/pylxd/issues/268
+        exit_code = 0
+        try:
+            stdout, stderr = subp(
+                ['lxc', 'exec', self.name, '--'] + env_args + list(command),
+                data=stdin, decode=False)
+        except ProcessExecutionError as e:
+            exit_code = e.exit_code
+            stdout = e.stdout
+            stderr = e.stderr
+
+        return stdout, stderr, exit_code
 
     def read_data(self, remote_path, decode=False):
         """Read data from instance filesystem.
diff --git a/tests/cloud_tests/testcases.yaml b/tests/cloud_tests/testcases.yaml
index 7183e01..8e0fb62 100644
--- a/tests/cloud_tests/testcases.yaml
+++ b/tests/cloud_tests/testcases.yaml
@@ -7,22 +7,37 @@ base_test_data:
         #cloud-config
     collect_scripts:
         cloud-init.log: |
-            #!/bin/bash
+            #!/bin/sh
             cat /var/log/cloud-init.log
         cloud-init-output.log: |
-            #!/bin/bash
+            #!/bin/sh
             cat /var/log/cloud-init-output.log
         instance-id: |
-            #!/bin/bash
+            #!/bin/sh
             cat /run/cloud-init/.instance-id
         result.json: |
-            #!/bin/bash
+            #!/bin/sh
             cat /run/cloud-init/result.json
         status.json: |
-            #!/bin/bash
+            #!/bin/sh
             cat /run/cloud-init/status.json
         cloud-init-version: |
-            #!/bin/bash
+            #!/bin/sh
             dpkg-query -W -f='${Version}' cloud-init
+        system.journal.gz: |
+            #!/bin/sh
+            [ -d /run/systemd ] || { echo "not systemd."; exit 0; }
+            fail() { echo "ERROR:" "$@" 1>&2; exit 1; }
+            journal=""
+            for d in /run/log/journal /var/log/journal; do
+                for f in $d/*/system.journal; do
+                    [ -f "$f" ] || continue
+                    [ -z "$journal" ] ||
+                        fail "multiple journal found: $f $journal."
+                    journal="$f"
+                done
+            done
+            [ -f "$journal" ] || fail "no journal file found."
+            gzip --to-stdout "$journal"
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py
index 1c5b540..20e9595 100644
--- a/tests/cloud_tests/testcases/base.py
+++ b/tests/cloud_tests/testcases/base.py
@@ -30,12 +30,14 @@ class CloudTestCase(unittest.TestCase):
             raise AssertionError('Key "{}" not in cloud config'.format(name))
         return self.cloud_config[name]
 
-    def get_data_file(self, name):
+    def get_data_file(self, name, decode=True):
         """Get data file failing test if it is not present."""
         if name not in self.data:
             raise AssertionError('File "{}" missing from collect data'
                                  .format(name))
-        return self.data[name]
+        if not decode:
+            return self.data[name]
+        return self.data[name].decode('utf-8')
 
     def get_instance_id(self):
         """Get recorded instance id."""
diff --git a/tests/cloud_tests/verify.py b/tests/cloud_tests/verify.py
index fc1efcf..2a9fd52 100644
--- a/tests/cloud_tests/verify.py
+++ b/tests/cloud_tests/verify.py
@@ -29,7 +29,7 @@ def verify_data(base_dir, tests):
         data = {}
         test_dir = os.path.join(base_dir, test_name)
         for script_name in os.listdir(test_dir):
-            with open(os.path.join(test_dir, script_name), 'r') as fp:
+            with open(os.path.join(test_dir, script_name), 'rb') as fp:
                 data[script_name] = fp.read()
 
         # get test suite and launch tests