Merge ~wesley-wiedenmeier/cloud-init:integration-testing-tmpdir into cloud-init:master

[Wesley Wiedenmeier]

Integration Testing: improve handling of collected data during run

Previous behavior for run was to collect data into a temporary directory which
was always deleted when tests passed. This adds a command line option
'--preserve-data' that ensures that collected data will be left after tests
run. This also allows the directory to store collected data in during the run
command to be specified using '--data-dir'.

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:d730a7cb5f4fa25331837d48910f5d192871223a
https://jenkins.ubuntu.com/server/job/cloud-init-ci/141/
Executed test runs:
    SUCCESS: https://jenkins.ubuntu.com/server/job/cloud-init-ci/nodes=metal-amd64/141
    SUCCESS: https://jenkins.ubuntu.com/server/job/cloud-init-ci/nodes=metal-arm64/141
    SUCCESS: https://jenkins.ubuntu.com/server/job/cloud-init-ci/nodes=metal-ppc64el/141
    SUCCESS: https://jenkins.ubuntu.com/server/job/cloud-init-ci/nodes=metal-s390x/141
    SUCCESS: https://jenkins.ubuntu.com/server/job/cloud-init-ci/nodes=vm-i386/141

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/141/rebuild

review: Approve (continuous-integration)

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:735e7bc095d9f1a4ca373b70288d89bdfd713b92
https://jenkins.ubuntu.com/server/job/cloud-init-ci/193/
Executed test runs:
    SUCCESS: https://jenkins.ubuntu.com/server/job/cloud-init-ci/nodes=metal-amd64/193
    SUCCESS: https://jenkins.ubuntu.com/server/job/cloud-init-ci/nodes=metal-arm64/193
    SUCCESS: https://jenkins.ubuntu.com/server/job/cloud-init-ci/nodes=metal-ppc64el/193
    SUCCESS: https://jenkins.ubuntu.com/server/job/cloud-init-ci/nodes=metal-s390x/193
    SUCCESS: https://jenkins.ubuntu.com/server/job/cloud-init-ci/nodes=vm-i386/193

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/193/rebuild

review: Approve (continuous-integration)

[Scott Moser (smoser)]

I'm going to mark this 'work in progress'. Josh has integrated this branch and others into his merge proposal at https://code.launchpad.net/~powersj/cloud-init/+git/cloud-init/+merge/324136 . We do plan to pull that all soon.

Thanks for your work, Wesley!

[Scott Moser (smoser)]

Hi.
I've marked this 'merged' as I think it is now in trunk under 76d58265e34851b78e952a7f275340863c90a9f5.
If you disagree, please feel free to re-open.

diff --git a/doc/rtd/topics/tests.rst b/doc/rtd/topics/tests.rst
index 0663811..7142370 100644
--- a/doc/rtd/topics/tests.rst
+++ b/doc/rtd/topics/tests.rst
@@ -193,7 +193,8 @@ explaining how to run one or the other independently.
     $ git clone https://git.launchpad.net/cloud-init
     $ cd cloud-init
     $ python3 -m tests.cloud_tests run -v -n trusty -n xenial \
-        --deb cloud-init_0.7.8~my_patch_all.deb
+        --deb cloud-init_0.7.8~my_patch_all.deb --preserve-data
+        --data-dir /tmp/collection
 
 The above command will do the following:
 
@@ -208,8 +209,19 @@ The above command will do the following:
 * ``--deb cloud-init_0.7.8~patch_all.deb`` use this deb as the version of
   cloud-init to run with
 
+* ``--preserve-data`` always preserve collected data, do not remove data after
+  successful test run
+
+* ``--data-dir /tmp/collection`` write collected data into `/tmp/collection`,
+  rather than using a temporary directory
+
 For a more detailed explanation of each option see below.
 
+.. note::
+    By default, data collected by the run command will be written into a
+    temporary directory, to be deleted after a successful. If you would
+    like to preserve this data, please use the option ``--preserve-data``.
+
 Collect
 -------
 
diff --git a/tests/cloud_tests/__main__.py b/tests/cloud_tests/__main__.py
index ef7d187..c0d34d8 100644
--- a/tests/cloud_tests/__main__.py
+++ b/tests/cloud_tests/__main__.py
@@ -2,11 +2,9 @@
 
 import argparse
 import logging
-import shutil
 import sys
-import tempfile
 
-from tests.cloud_tests import (args, collect, manage, verify)
+from tests.cloud_tests import (args, collect, manage, util, verify)
 from tests.cloud_tests import LOG
 
 
@@ -24,23 +22,19 @@ def configure_log(args):
 
 def run(args):
     """
-    run full test suite
+    run test suite
     """
     failed = 0
-    args.data_dir = tempfile.mkdtemp(prefix='cloud_test_data_')
-    LOG.debug('using tmpdir %s', args.data_dir)
-    try:
+    tmpdir = util.TempDir(tmpdir=args.data_dir, preserve=args.preserve_data)
+
+    with tmpdir as data_dir:
+        args.data_dir = data_dir
         failed += collect.collect(args)
         failed += verify.verify(args)
-    except Exception:
-        failed += 1
-        raise
-    finally:
-        # TODO: make this configurable via environ or cmdline
         if failed:
-            LOG.warn('some tests failed, leaving data in %s', args.data_dir)
-        else:
-            shutil.rmtree(args.data_dir)
+            tmpdir.preserve = True
+            LOG.warning('some tests failed')
+
     return failed
 
 
diff --git a/tests/cloud_tests/args.py b/tests/cloud_tests/args.py
index b68cc98..e673277 100644
--- a/tests/cloud_tests/args.py
+++ b/tests/cloud_tests/args.py
@@ -46,6 +46,13 @@ ARG_SETS = {
         (('-r', '--result'),
          {'help': 'file to write results to',
           'action': 'store', 'metavar': 'FILE'}),),
+    'RUN_OUTPUT': (
+        (('-d', '--data-dir'),
+         {'help': 'directory to store test data in',
+          'action': 'store', 'metavar': 'DIR', 'required': False}),
+        (('--preserve-data',),
+         {'help': 'do not remove collected data after successful run',
+          'action': 'store_true', 'default': False, 'required': False}),),
     'SETUP': (
         (('--deb',),
          {'help': 'install deb', 'metavar': 'FILE', 'action': 'store'}),
@@ -69,7 +76,8 @@ SUBCMDS = {
     'collect': ('collect test data',
                 ('COLLECT', 'INTERFACE', 'OUTPUT', 'RESULT', 'SETUP')),
     'create': ('create new test case', ('CREATE', 'INTERFACE')),
-    'run': ('run test suite', ('COLLECT', 'INTERFACE', 'RESULT', 'SETUP')),
+    'run': ('run test suite',
+            ('COLLECT', 'INTERFACE', 'RESULT', 'RUN_OUTPUT', 'SETUP')),
     'verify': ('verify test data', ('INTERFACE', 'OUTPUT', 'RESULT')),
 }
 
@@ -215,6 +223,7 @@ NORMALIZERS = {
     'INTERFACE': _empty_normalizer,
     'OUTPUT': normalize_output_args,
     'RESULT': _empty_normalizer,
+    'RUN_OUTPUT': _empty_normalizer,
     'SETUP': normalize_setup_args,
 }
 
diff --git a/tests/cloud_tests/util.py b/tests/cloud_tests/util.py
index 64a8667..006d708 100644
--- a/tests/cloud_tests/util.py
+++ b/tests/cloud_tests/util.py
@@ -3,6 +3,7 @@
 import glob
 import os
 import random
+import shutil
 import string
 import tempfile
 import yaml
@@ -160,4 +161,39 @@ def write_file(*args, **kwargs):
     """
     c_util.write_file(*args, **kwargs)
 
+
+class TempDir(object):
+    """
+    temporary directory like tempfile.TemporaryDirectory, but configurable
+    """
+
+    def __init__(self, tmpdir=None, preserve=False, prefix='cloud_test_data_'):
+        """
+        tmpdir: directory to use as tempdir
+        preserve: if true, always preserve data on exit
+        prefix: prefix to use for tempfile name
+        """
+        self.tmpdir = tmpdir
+        self.preserve = preserve
+        self.prefix = prefix
+
+    def __enter__(self):
+        """
+        create tempdir
+        return_value: tempdir path
+        """
+        if not self.tmpdir:
+            self.tmpdir = tempfile.mkdtemp(prefix=self.prefix)
+        LOG.debug('using tmpdir: %s', self.tmpdir)
+        return self.tmpdir
+
+    def __exit__(self, etype, value, trace):
+        """
+        destroy tempdir if no errors occurred
+        """
+        if etype or self.preserve:
+            LOG.info('leaving data in %s', self.tmpdir)
+        else:
+            shutil.rmtree(self.tmpdir)
+
 # vi: ts=4 expandtab