Merge lp:~lifeless/bzr/test-speed into lp:~bzr/bzr/trunk-old

[Robert Collins]

[Robert Collins (lifeless)]

This turns two 13 second tests into a few millisecond-scale tests
testing more precise layers.

-Rob

--

[Martin Pool (mbp)]

I think those upper-layer tests are almost pointless, but fixing the lower layers to work on smaller data is certainly welcome.

review: Approve

=== modified file 'bzrlib/tests/__init__.py'
--- bzrlib/tests/__init__.py	2009-08-20 05:05:59 +0000
+++ bzrlib/tests/__init__.py	2009-08-23 22:35:09 +0000
@@ -3207,6 +3207,7 @@
              starting_with=None,
              runner_class=None,
              suite_decorators=None,
+             stream=None,
              ):
     """Run the whole test suite under the enhanced runner"""
     # XXX: Very ugly way to do this...
@@ -3245,6 +3246,7 @@
                      strict=strict,
                      runner_class=runner_class,
                      suite_decorators=suite_decorators,
+                     stream=stream,
                      )
     finally:
         default_transport = old_transport
 
=== modified file 'bzrlib/tests/blackbox/test_selftest.py'
--- bzrlib/tests/blackbox/test_selftest.py	2009-07-10 07:14:02 +0000
+++ bzrlib/tests/blackbox/test_selftest.py	2009-08-23 22:35:09 +0000
@@ -26,6 +26,7 @@
 import bzrlib
 from bzrlib import (
     osutils,
+    tests,
     )
 from bzrlib.errors import ParamikoNotPresent
 from bzrlib.tests import (
@@ -512,59 +513,55 @@
         return (header,body,footer)
 
     def test_list_only(self):
-        # check that bzr selftest --list-only works correctly
-        out,err = self.run_bzr('selftest selftest --list-only')
-        (header,body,footer) = self._parse_test_list(out.splitlines())
-        num_tests = len(body)
-        self.assertLength(0, header)
-        self.assertLength(0, footer)
-        self.assertEqual('', err)
-
-    def test_list_only_filtered(self):
-        # check that a filtered --list-only works, both include and exclude
-        out_all,err_all = self.run_bzr('selftest --list-only')
-        tests_all = self._parse_test_list(out_all.splitlines())[1]
-        out_incl,err_incl = self.run_bzr('selftest --list-only selftest')
-        tests_incl = self._parse_test_list(out_incl.splitlines())[1]
-        self.assertSubset(tests_incl, tests_all)
-        out_excl,err_excl = self.run_bzr(['selftest', '--list-only',
-                                          '--exclude', 'selftest'])
-        tests_excl = self._parse_test_list(out_excl.splitlines())[1]
-        self.assertSubset(tests_excl, tests_all)
-        set_incl = set(tests_incl)
-        set_excl = set(tests_excl)
-        intersection = set_incl.intersection(set_excl)
-        self.assertEquals(0, len(intersection))
-        self.assertEquals(len(tests_all), len(tests_incl) + len(tests_excl))
-
-    def test_list_only_random(self):
-        # check that --randomize works correctly
-        out_all,err_all = self.run_bzr('selftest --list-only selftest')
-        tests_all = self._parse_test_list(out_all.splitlines())[1]
-        # XXX: It looks like there are some orders for generating tests that
-        # fail as of 20070504 - maybe because of import order dependencies.
-        # So unfortunately this will rarely intermittently fail at the moment.
-        # -- mbp 20070504
-        out_rand,err_rand = self.run_bzr(['selftest', '--list-only',
-                                          'selftest', '--randomize', 'now'])
-        (header_rand,tests_rand,dummy) = self._parse_test_list(
-            out_rand.splitlines(), 1)
-        # XXX: The following line asserts that the randomized order is not the
-        # same as the default order.  It is just possible that they'll get
-        # randomized into the same order and this will falsely fail, but
-        # that's very unlikely in practice because there are thousands of
-        # tests.
-        self.assertNotEqual(tests_all, tests_rand)
-        self.assertEqual(sorted(tests_all), sorted(tests_rand))
-        # Check that the seed can be reused to get the exact same order
-        seed_re = re.compile('Randomizing test order using seed (\w+)')
-        match_obj = seed_re.search(header_rand[-1])
-        seed = match_obj.group(1)
-        out_rand2,err_rand2 = self.run_bzr(['selftest', '--list-only',
-                                            'selftest', '--randomize', seed])
-        (header_rand2,tests_rand2,dummy) = self._parse_test_list(
-            out_rand2.splitlines(), 1)
-        self.assertEqual(tests_rand, tests_rand2)
+        # check that bzr selftest --list-only outputs no ui noise
+        def selftest(*args, **kwargs):
+            """Capture the arguments selftest was run with."""
+            return True
+        def outputs_nothing(cmdline):
+            out,err = self.run_bzr(cmdline)
+            (header,body,footer) = self._parse_test_list(out.splitlines())
+            num_tests = len(body)
+            self.assertLength(0, header)
+            self.assertLength(0, footer)
+            self.assertEqual('', err)
+        # Yes this prevents using threads to run the test suite in parallel,
+        # however we don't have a clean dependency injector for commands, 
+        # and even if we did - we'd still be testing that the glue is wired
+        # up correctly. XXX: TODO: Solve this testing problem.
+        original_selftest = tests.selftest
+        tests.selftest = selftest
+        try:
+            outputs_nothing('selftest --list-only')
+            outputs_nothing('selftest --list-only selftest')
+            outputs_nothing(['selftest', '--list-only', '--exclude', 'selftest'])
+        finally:
+            tests.selftest = original_selftest
+
+    def test_parameters_passed_to_core(self):
+        params = []
+        def selftest(*args, **kwargs):
+            """Capture the arguments selftest was run with."""
+            params.append((args, kwargs))
+            return True
+        # Yes this prevents using threads to run the test suite in parallel,
+        # however we don't have a clean dependency injector for commands, 
+        # and even if we did - we'd still be testing that the glue is wired
+        # up correctly. XXX: TODO: Solve this testing problem.
+        original_selftest = tests.selftest
+        tests.selftest = selftest
+        try:
+            self.run_bzr('selftest --list-only')
+            self.run_bzr('selftest --list-only selftest')
+            self.run_bzr(['selftest', '--list-only', '--exclude', 'selftest'])
+            self.run_bzr(['selftest', '--list-only', 'selftest',
+                '--randomize', 'now'])
+            # list_only should have been passed in each invocation.
+            self.assertTrue("list_only" in params[0][1])
+            self.assertTrue("list_only" in params[1][1])
+            self.assertTrue("list_only" in params[2][1])
+            self.assertSubset(["list_only", "random_seed"], params[2][1])
+        finally:
+            tests.selftest = original_selftest
 
 
 class TestSelftestWithIdList(TestCaseInTempDir):
 
=== modified file 'bzrlib/tests/test_selftest.py'
--- bzrlib/tests/test_selftest.py	2009-08-17 03:47:03 +0000
+++ bzrlib/tests/test_selftest.py	2009-08-23 22:35:09 +0000
@@ -1770,6 +1770,60 @@
             test_suite_factory=factory)
         self.assertEqual([True], factory_called)
 
+    def factory(self):
+        """A test suite factory."""
+        class Test(tests.TestCase):
+            def a(self):
+                pass
+            def b(self):
+                pass
+            def c(self):
+                pass
+        return TestUtil.TestSuite([Test("a"), Test("b"), Test("c")])
+
+    def run_selftest(self, **kwargs):
+        """Run selftest returning its output."""
+        output = StringIO()
+        self.assertEqual(True, tests.selftest(stream=output, **kwargs))
+        output.seek(0)
+        return output
+
+    def test_list_only(self):
+        output = self.run_selftest(test_suite_factory=self.factory,
+            list_only=True)
+        self.assertEqual(3, len(output.readlines()))
+
+    def test_list_only_filtered(self):
+        output = self.run_selftest(test_suite_factory=self.factory,
+            list_only=True, pattern="Test.b")
+        self.assertEndsWith(output.getvalue(), "Test.b\n")
+        self.assertLength(1, output.readlines())
+
+    def test_list_only_excludes(self):
+        output = self.run_selftest(test_suite_factory=self.factory,
+            list_only=True, exclude_pattern="Test.b")
+        self.assertNotContainsRe("Test.b", output.getvalue())
+        self.assertLength(2, output.readlines())
+
+    def test_random(self):
+        # test randomising by listing a number of tests.
+        output_123 = self.run_selftest(test_suite_factory=self.factory,
+            list_only=True, random_seed="123")
+        output_234 = self.run_selftest(test_suite_factory=self.factory,
+            list_only=True, random_seed="234")
+        self.assertNotEqual(output_123, output_234)
+        # "Randominzing test order..\n\n
+        self.assertLength(5, output_123.readlines())
+        self.assertLength(5, output_234.readlines())
+
+    def test_random_reuse_is_same_order(self):
+        # test randomising by listing a number of tests.
+        expected = self.run_selftest(test_suite_factory=self.factory,
+            list_only=True, random_seed="123")
+        repeated = self.run_selftest(test_suite_factory=self.factory,
+            list_only=True, random_seed="123")
+        self.assertEqual(expected.getvalue(), repeated.getvalue())
+
 
 class TestKnownFailure(tests.TestCase):