Merge ~smoser/cloud-init:bug/run_test_net into cloud-init:master

[Scott Moser]

tests: remove executable bit on test_net, so it runs, and fix it.

The test_user_data_normalize and test_net files had gotten
the executable bit set, and thus are skipped by nose by default.
We could set run with the --exe flag, but they should not have
gotten this way.

Other changes here:
 * replace TempDirTestCase with CiTestCase, which has some nice
   tmp_dir() and tmp_path() functions. Going forward the intent
   is to have CiTestCase be the base test case for tests.
 * test_net: switch to CiTestCase and fix usage that was
   silently broken, because of exe bit.
 * populate_dir: return the list of files that it writes rather
   than having no return value.
 * CiTestCase:
    * support tmp_path("foo") that returns a full path to 'foo'
      under a tmpdir.
    * add tmp_dir() to get a temp dir and clean up.

diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py
index a093346..90e2431 100644
--- a/tests/unittests/helpers.py
+++ b/tests/unittests/helpers.py
@@ -2,6 +2,7 @@
 
 from __future__ import print_function
 
+import functools
 import os
 import shutil
 import sys
@@ -82,6 +83,28 @@ class TestCase(unittest2.TestCase):
     pass
 
 
+class CiTestCase(TestCase):
+    """This is the preferred test case base class unless user
+       needs other test case classes below."""
+    def tmp_dir(self, dir=None, cleanup=True):
+        # return a full path to a temporary directory that will be cleaned up.
+        if dir is None:
+            tmpd = tempfile.mkdtemp(
+                prefix="ci-%s." % self.__class__.__name__)
+        else:
+            tmpd = tempfile.mkdtemp(dir=dir)
+        self.addCleanup(functools.partial(shutil.rmtree, tmpd))
+        return tmpd
+
+    def tmp_path(self, path, dir=None):
+        # return an absolute path to 'path' under dir.
+        # if dir is None, one will be created with tmp_dir()
+        # the file is not created or modified.
+        if dir is None:
+            dir = self.tmp_dir()
+        return os.path.normpath(os.path.abspath(os.path.join(dir, path)))
+
+
 class ResourceUsingTestCase(TestCase):
     def setUp(self):
         super(ResourceUsingTestCase, self).setUp()
@@ -227,29 +250,10 @@ class HttprettyTestCase(TestCase):
         super(HttprettyTestCase, self).tearDown()
 
 
-class TempDirTestCase(TestCase):
-    # provide a tempdir per class, not per test.
-    @classmethod
-    def setUpClass(cls):
-        cls.tmpd = tempfile.mkdtemp(prefix="ci-%s." % cls.__name__)
-        return TestCase.setUpClass()
-
-    @classmethod
-    def tearDownClass(cls):
-        shutil.rmtree(cls.tmpd)
-        return TestCase.tearDownClass()
-
-    def tmp_path(self, path):
-        # if absolute path (starts with /), then make ./path
-        if path.startswith(os.path.sep):
-            path = "." + path
-
-        return os.path.normpath(os.path.join(self.tmpd, path))
-
-
 def populate_dir(path, files):
     if not os.path.exists(path):
         os.makedirs(path)
+    ret = []
     for (name, content) in files.items():
         p = os.path.join(path, name)
         util.ensure_dir(os.path.dirname(p))
@@ -259,6 +263,9 @@ def populate_dir(path, files):
             else:
                 fp.write(content.encode('utf-8'))
             fp.close()
+        ret.append(p)
+
+    return ret
 
 
 def dir2dict(startdir, prefix=None):
diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py
index e6f4c31..781f6d5 100644
--- a/tests/unittests/test__init__.py
+++ b/tests/unittests/test__init__.py
@@ -12,7 +12,7 @@ from cloudinit import settings
 from cloudinit import url_helper
 from cloudinit import util
 
-from .helpers import TestCase, TempDirTestCase, ExitStack, mock
+from .helpers import TestCase, CiTestCase, ExitStack, mock
 
 
 class FakeModule(handlers.Handler):
@@ -172,7 +172,7 @@ class TestHandlerHandlePart(TestCase):
             self.data, self.ctype, self.filename, self.payload)
 
 
-class TestCmdlineUrl(TempDirTestCase):
+class TestCmdlineUrl(CiTestCase):
     def test_parse_cmdline_url_nokey_raises_keyerror(self):
         self.assertRaises(
             KeyError, main.parse_cmdline_url, 'root=foo bar single')
@@ -189,7 +189,7 @@ class TestCmdlineUrl(TempDirTestCase):
         cmdline = "ro %s=%s bar=1" % (key, url)
         m_read.return_value = url_helper.StringResponse(b"unexpected blob")
 
-        fpath = self.tmp_path("test_valid")
+        fpath = self.tmp_path("ccfile")
         lvl, msg = main.attempt_cmdline_url(
             fpath, network=True, cmdline=cmdline)
         self.assertEqual(logging.WARN, lvl)
@@ -203,7 +203,7 @@ class TestCmdlineUrl(TempDirTestCase):
         cmdline = "ro %s=%s bar=1" % ('cloud-config-url', url)
 
         m_read.return_value = url_helper.StringResponse(payload)
-        fpath = self.tmp_path("test_valid")
+        fpath = self.tmp_path("ccfile")
         lvl, msg = main.attempt_cmdline_url(
             fpath, network=True, cmdline=cmdline)
         self.assertEqual(util.load_file(fpath, decode=False), payload)
@@ -213,7 +213,7 @@ class TestCmdlineUrl(TempDirTestCase):
     @mock.patch('cloudinit.cmd.main.util.read_file_or_url')
     def test_no_key_found(self, m_read):
         cmdline = "ro mykey=http://example.com/foo root=foo"
-        fpath = self.tmp_path("test_no_key_found")
+        fpath = self.tmp_path("ccpath")
         lvl, msg = main.attempt_cmdline_url(
             fpath, network=True, cmdline=cmdline)
 
@@ -225,7 +225,7 @@ class TestCmdlineUrl(TempDirTestCase):
     def test_exception_warns(self, m_read):
         url = "http://example.com/foo"
         cmdline = "ro cloud-config-url=%s root=LABEL=bar" % url
-        fpath = self.tmp_path("test_no_key_found")
+        fpath = self.tmp_path("ccfile")
         m_read.side_effect = url_helper.UrlError(
             cause="Unexpected Error", url="http://example.com/foo")
 
diff --git a/tests/unittests/test_atomic_helper.py b/tests/unittests/test_atomic_helper.py
index e170c7c..515919d 100644
--- a/tests/unittests/test_atomic_helper.py
+++ b/tests/unittests/test_atomic_helper.py
@@ -6,10 +6,10 @@ import stat
 
 from cloudinit import atomic_helper
 
-from . import helpers
+from .helpers import CiTestCase
 
 
-class TestAtomicHelper(helpers.TempDirTestCase):
+class TestAtomicHelper(CiTestCase):
     def test_basic_usage(self):
         """write_file takes bytes if no omode."""
         path = self.tmp_path("test_basic_usage")
diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py
 old mode 100755
 new mode 100644
index 88746e0..88746e0
--- a/tests/unittests/test_distros/test_user_data_normalize.py
+++ b/tests/unittests/test_distros/test_user_data_normalize.py
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
 old mode 100755
 new mode 100644
index 1090282..2c2bde9
--- a/tests/unittests/test_net.py
+++ b/tests/unittests/test_net.py
@@ -8,11 +8,10 @@ from cloudinit.net import sysconfig
 from cloudinit.sources.helpers import openstack
 from cloudinit import util
 
+from .helpers import CiTestCase
 from .helpers import dir2dict
 from .helpers import mock
 from .helpers import populate_dir
-from .helpers import TempDirTestCase
-from .helpers import TestCase
 
 import base64
 import copy
@@ -20,8 +19,6 @@ import gzip
 import io
 import json
 import os
-import shutil
-import tempfile
 import textwrap
 import yaml
 
@@ -478,7 +475,7 @@ def _setup_test(tmp_dir, mock_get_devicelist, mock_read_sys_net,
     mock_sys_dev_path.side_effect = sys_dev_path
 
 
-class TestSysConfigRendering(TestCase):
+class TestSysConfigRendering(CiTestCase):
 
     @mock.patch("cloudinit.net.sys_dev_path")
     @mock.patch("cloudinit.net.read_sys_net")
@@ -486,8 +483,7 @@ class TestSysConfigRendering(TestCase):
     def test_default_generation(self, mock_get_devicelist,
                                 mock_read_sys_net,
                                 mock_sys_dev_path):
-        tmp_dir = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, tmp_dir)
+        tmp_dir = self.tmp_dir()
         _setup_test(tmp_dir, mock_get_devicelist,
                     mock_read_sys_net, mock_sys_dev_path)
 
@@ -518,9 +514,7 @@ USERCTL=no
             self.assertEqual(expected_content, content)
 
     def test_openstack_rendering_samples(self):
-        tmp_dir = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, tmp_dir)
-        render_dir = os.path.join(tmp_dir, "render")
+        render_dir = self.tmp_dir()
         for os_sample in OS_SAMPLES:
             ex_input = os_sample['in_data']
             ex_mac_addrs = os_sample['in_macs']
@@ -535,7 +529,7 @@ USERCTL=no
                     self.assertEqual(expected_content, fh.read())
 
 
-class TestEniNetRendering(TestCase):
+class TestEniNetRendering(CiTestCase):
 
     @mock.patch("cloudinit.net.sys_dev_path")
     @mock.patch("cloudinit.net.read_sys_net")
@@ -543,8 +537,7 @@ class TestEniNetRendering(TestCase):
     def test_default_generation(self, mock_get_devicelist,
                                 mock_read_sys_net,
                                 mock_sys_dev_path):
-        tmp_dir = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, tmp_dir)
+        tmp_dir = self.tmp_dir()
         _setup_test(tmp_dir, mock_get_devicelist,
                     mock_read_sys_net, mock_sys_dev_path)
 
@@ -576,7 +569,7 @@ iface eth1000 inet dhcp
         self.assertEqual(expected.lstrip(), contents.lstrip())
 
 
-class TestEniNetworkStateToEni(TestCase):
+class TestEniNetworkStateToEni(CiTestCase):
     mycfg = {
         'config': [{"type": "physical", "name": "eth0",
                     "mac_address": "c0:d6:9f:2c:e8:80",
@@ -607,7 +600,7 @@ class TestEniNetworkStateToEni(TestCase):
         self.assertNotIn("hwaddress", rendered)
 
 
-class TestCmdlineConfigParsing(TestCase):
+class TestCmdlineConfigParsing(CiTestCase):
     simple_cfg = {
         'config': [{"type": "physical", "name": "eth0",
                     "mac_address": "c0:d6:9f:2c:e8:80",
@@ -665,7 +658,7 @@ class TestCmdlineConfigParsing(TestCase):
         self.assertEqual(found, self.simple_cfg)
 
 
-class TestCmdlineReadKernelConfig(TempDirTestCase):
+class TestCmdlineReadKernelConfig(CiTestCase):
     macs = {
         'eth0': '14:02:ec:42:48:00',
         'eno1': '14:02:ec:42:48:01',
@@ -673,8 +666,7 @@ class TestCmdlineReadKernelConfig(TempDirTestCase):
 
     def test_ip_cmdline_read_kernel_cmdline_ip(self):
         content = {'net-eth0.conf': DHCP_CONTENT_1}
-        populate_dir(self.tmp, content)
-        files = [os.path.join(self.tmp, k) for k in content.keys()]
+        files = sorted(populate_dir(self.tmp_dir(), content))
         found = cmdline.read_kernel_cmdline_config(
             files=files, cmdline='foo ip=dhcp', mac_addrs=self.macs)
         exp1 = copy.deepcopy(DHCP_EXPECTED_1)
@@ -684,8 +676,7 @@ class TestCmdlineReadKernelConfig(TempDirTestCase):
 
     def test_ip_cmdline_read_kernel_cmdline_ip6(self):
         content = {'net6-eno1.conf': DHCP6_CONTENT_1}
-        populate_dir(self.tmp, content)
-        files = [os.path.join(self.tmp, k) for k in content.keys()]
+        files = sorted(populate_dir(self.tmp_dir(), content))
         found = cmdline.read_kernel_cmdline_config(
             files=files, cmdline='foo ip6=dhcp root=/dev/sda',
             mac_addrs=self.macs)
@@ -701,8 +692,7 @@ class TestCmdlineReadKernelConfig(TempDirTestCase):
     def test_ip_cmdline_read_kernel_cmdline_none(self):
         # if there is no ip= or ip6= on cmdline, return value should be None
         content = {'net6-eno1.conf': DHCP6_CONTENT_1}
-        populate_dir(self.tmp, content)
-        files = [os.path.join(self.tmp, k) for k in content.keys()]
+        files = sorted(populate_dir(self.tmp_dir(), content))
         found = cmdline.read_kernel_cmdline_config(
             files=files, cmdline='foo root=/dev/sda', mac_addrs=self.macs)
         self.assertEqual(found, None)
@@ -710,8 +700,7 @@ class TestCmdlineReadKernelConfig(TempDirTestCase):
     def test_ip_cmdline_both_ip_ip6(self):
         content = {'net-eth0.conf': DHCP_CONTENT_1,
                    'net6-eth0.conf': DHCP6_CONTENT_1.replace('eno1', 'eth0')}
-        populate_dir(self.tmp, content)
-        files = [os.path.join(self.tmp, k) for k in sorted(content.keys())]
+        files = sorted(populate_dir(self.tmp_dir(), content))
         found = cmdline.read_kernel_cmdline_config(
             files=files, cmdline='foo ip=dhcp ip6=dhcp', mac_addrs=self.macs)
 
@@ -725,14 +714,12 @@ class TestCmdlineReadKernelConfig(TempDirTestCase):
         self.assertEqual(found['config'], expected)
 
 
-class TestEniRoundTrip(TestCase):
-    def setUp(self):
-        super(TestCase, self).setUp()
-        self.tmp_dir = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, self.tmp_dir)
-
+class TestEniRoundTrip(CiTestCase):
     def _render_and_read(self, network_config=None, state=None, eni_path=None,
-                         links_prefix=None, netrules_path=None):
+                         links_prefix=None, netrules_path=None, dir=None):
+        if dir is None:
+            dir = self.tmp_dir()
+
         if network_config:
             ns = network_state.parse_net_config_data(network_config)
         elif state:
@@ -747,8 +734,8 @@ class TestEniRoundTrip(TestCase):
             config={'eni_path': eni_path, 'links_path_prefix': links_prefix,
                     'netrules_path': netrules_path})
 
-        renderer.render_network_state(self.tmp_dir, ns)
-        return dir2dict(self.tmp_dir)
+        renderer.render_network_state(dir, ns)
+        return dir2dict(dir)
 
     def testsimple_convert_and_render(self):
         network_config = eni.convert_eni_data(EXAMPLE_ENI)