Merge lp:~thomir-deactivatedaccount/autopilot/fix-autopilot-import-issues into lp:autopilot

[Thomi Richards]

Fix autopilot test import behavior to be more convenient, and work around an odd bug inside the unittest module.

This branch changes the autopilot behavior in a couple of important ways:

First, it adds the current working directory to the python module search path (i.e.- sys.path). This means that in order to run a particular set of tests, you now just need to: cd into the directory that contains the tests, and run 'autopilot run testpackagename' - no mode mucking about with PYTHONPATH variables.

Second, it's reasonably common to have several instances of the same test suite on a developers machine. Autopilot now tells you where it's loading tests from for both the 'list' and 'run' commands.

The existing tests around the 'autopilot list' functionality have been modified to cope with this new feature.

[jenkins (martin-mrazik+qa)]

PASSED: Continuous integration, rev:68
http://s-jenkins:8080/job/autopilot-ci/75/
Executed test runs:
    SUCCESS: http://s-jenkins:8080/job/autopilot-ci/./distribution=precise,flavor=amd64/75/console
    SUCCESS: http://s-jenkins:8080/job/autopilot-ci/./distribution=precise,flavor=i386/75/console

review: Approve (continuous-integration)

[lp:~thomir-deactivatedaccount/autopilot/fix-autopilot-import-issues]

69. By Thomi Richards on 2012-09-10

Fixed code.

70. By Thomi Richards on 2012-09-10

more cleanups.

[jenkins (martin-mrazik+qa)]

PASSED: Continuous integration, rev:70
http://s-jenkins:8080/job/autopilot-ci/76/
Executed test runs:
    SUCCESS: http://s-jenkins:8080/job/autopilot-ci/./distribution=precise,flavor=amd64/76/console
    SUCCESS: http://s-jenkins:8080/job/autopilot-ci/./distribution=precise,flavor=i386/76/console

review: Approve (continuous-integration)

[Christopher Lee (veebers)]

review: Approve

=== modified file 'autopilot/tests/test_autopilot_functional.py'
--- autopilot/tests/test_autopilot_functional.py	2012-08-28 02:23:46 +0000
+++ autopilot/tests/test_autopilot_functional.py	2012-09-10 05:14:26 +0000
@@ -79,7 +79,7 @@
 
         environ = os.environ
         environ.update(dict(
-                PYTHONPATH='%s:%s' % (self.base_path, ap_base_path),
+                PYTHONPATH=ap_base_path,
                 DISPLAY=':0'))
 
         process = subprocess.Popen(
@@ -100,6 +100,27 @@
 
         return (retcode, stdout, stderr)
 
+    def assertTestsInOutput(self, tests, output):
+        """Asserts that 'tests' are all present in 'output'."""
+
+        if type(tests) is not list:
+            raise TypeError("tests must be a list, not %r" % type(tests))
+        if not isinstance(output, basestring):
+            raise TypeError("output must be a string, not %r" % type(output))
+
+        expected = '''\
+Loading tests from: %s
+
+%s
+
+ %d total tests.
+''' % (self.base_path,
+    ''.join(['    %s\n' % t for t in sorted(tests)]),
+    len(tests))
+
+        self.assertThat(output, Equals(expected))
+
+
     def create_test_file(self, name, contents):
         """Create a test file with the given name and contents.
 
@@ -118,11 +139,9 @@
         """Autopilot list must report 0 tests found with an empty test module."""
         code, output, error = self.run_autopilot_list()
 
-        expected_output = '\n\n 0 total tests.\n'
-
         self.assertThat(code, Equals(0))
         self.assertThat(error, Equals(''))
-        self.assertThat(output, Equals(expected_output))
+        self.assertTestsInOutput([], output)
 
     def test_can_list_tests(self):
         """Autopilot must find tests in a file."""
@@ -138,13 +157,6 @@
             """
             ))
 
-        expected_output = '''\
-    tests.test_simple.SimpleTest.test_simple
-
-
- 1 total tests.
-'''
-
         # ideally these would be different tests, but I'm lazy:
         valid_test_specs = [
             'tests',
@@ -156,7 +168,7 @@
             code, output, error = self.run_autopilot_list(test_spec)
             self.assertThat(code, Equals(0))
             self.assertThat(error, Equals(''))
-            self.assertThat(output, Equals(expected_output))
+            self.assertTestsInOutput(['tests.test_simple.SimpleTest.test_simple'], output)
 
     def test_list_tests_with_import_error(self):
         self.create_test_file('test_simple.py', dedent("""\
@@ -173,6 +185,8 @@
             ))
         code, output, error = self.run_autopilot_list()
         expected_regex = '''\
+Loading tests from: %s
+
 Failed to import test module: tests.test_simple
 Traceback \(most recent call last\):
   File "/usr/lib/python2.7/unittest/loader.py", line 252, in _find_tests
@@ -186,7 +200,7 @@
 
 
  0 total tests.
-'''
+''' % self.base_path
         self.assertThat(code, Equals(0))
         self.assertThat(error, Equals(''))
         self.assertThat(output, MatchesRegex(expected_regex, re.MULTILINE))
@@ -206,6 +220,8 @@
             ))
         code, output, error = self.run_autopilot_list()
         expected_regex = '''\
+Loading tests from: %s
+
 Failed to import test module: tests.test_simple
 Traceback \(most recent call last\):
   File "/usr/lib/python2.7/unittest/loader.py", line 252, in _find_tests
@@ -220,7 +236,7 @@
 
 
  0 total tests.
-'''
+''' % self.base_path
         self.assertThat(code, Equals(0))
         self.assertThat(error, Equals(''))
         self.assertThat(output, MatchesRegex(expected_regex, re.MULTILINE))
@@ -244,11 +260,13 @@
             ))
 
         expected_output = '''\
+Loading tests from: %s
+
  *1 tests.test_simple.SimpleTest.test_simple
 
 
  1 total tests.
-'''
+''' % self.base_path
 
         code, output, error = self.run_autopilot_list()
         self.assertThat(code, Equals(0))
@@ -282,12 +300,14 @@
             ))
 
         expected_output = '''\
+Loading tests from: %s
+
  *2 tests.test_simple.SimpleTest.test_simple
  *2 tests.test_simple.SimpleTest.test_simple_two
 
 
  4 total tests.
-'''
+''' % self.base_path
 
         code, output, error = self.run_autopilot_list()
         self.assertThat(code, Equals(0))
@@ -310,15 +330,7 @@
             """
             ))
 
-        expected_output = '''\
-    tests.test_simple.SimpleTest.test_simple
-
-
- 1 total tests.
-'''
-
         code, output, error = self.run_autopilot_list()
         self.assertThat(code, Equals(0))
         self.assertThat(error, Equals(''))
-        self.assertThat(output, Equals(expected_output))
-
+        self.assertTestsInOutput(['tests.test_simple.SimpleTest.test_simple'], output)

=== modified file 'bin/autopilot'
--- bin/autopilot	2012-08-28 02:23:46 +0000
+++ bin/autopilot	2012-09-10 05:14:26 +0000
@@ -13,11 +13,9 @@
 from imp import find_module
 import os
 import os.path
-from os import remove, putenv
+from os import putenv
 from platform import node
-from subprocess import call
 import sys
-from tempfile import NamedTemporaryFile
 from testtools import iterate_tests
 from textwrap import dedent
 from argparse import ArgumentParser
@@ -192,6 +190,7 @@
 
 def load_test_suite_from_name(test_names):
     """Returns a test suite object given a dotted test names."""
+    patch_python_path()
     loader = TestLoader()
     if isinstance(test_names, basestring):
         test_names = list(test_names)
@@ -200,6 +199,7 @@
                         % (type(test_names)))
 
     tests = []
+    test_package_locations = []
     for test_name in test_names:
         top_level_pkg = test_name.split('.')[0]
         package = __import__(top_level_pkg)
@@ -209,9 +209,14 @@
                 '..'
                 )
             )
+        if package_parent_path not in test_package_locations:
+            test_package_locations.append(package_parent_path)
         tests.append(loader.discover(top_level_pkg, top_level_dir=package_parent_path))
     all_tests = TestSuite(tests)
 
+    test_dirs = ", ".join(sorted(test_package_locations))
+    print "Loading tests from: %s\n" % test_dirs
+
     requested_tests = {}
     for test in iterate_tests(all_tests):
         # The test loader returns tests that start with 'unittest.loader' if for
@@ -230,6 +235,15 @@
     return TestSuite(requested_tests.values())
 
 
+def patch_python_path():
+    """Prepend the current directory to sys.path to ensure that we can load & run
+    autopilot tests if the caller is in the parent directory.
+
+    """
+    if os.getcwd() not in sys.path:
+        sys.path.insert(0, os.getcwd())
+
+
 def main():
     args = parse_arguments()
     if args.mode == 'list':