Merge lp:~lifeless/bzr/test-speed into lp:~bzr/bzr/trunk-old

[Robert Collins]

[Robert Collins (lifeless)]

This turns two 13 second tests into a few millisecond-scale tests
testing more precise layers.

-Rob

--

[Martin Pool (mbp)]

I think those upper-layer tests are almost pointless, but fixing the lower layers to work on smaller data is certainly welcome.

review: Approve

[Robert Collins (lifeless)]

I ended up doing a couple more things while I was there.
< 10 seconds for the selftest tests now.

[Martin Pool (mbp)]

> I ended up doing a couple more things while I was there.
> < 10 seconds for the selftest tests now.

review: Approve

=== modified file 'bzrlib/tests/__init__.py'
--- bzrlib/tests/__init__.py	2009-08-20 05:05:59 +0000
+++ bzrlib/tests/__init__.py	2009-08-24 05:35:11 +0000
@@ -3207,6 +3207,7 @@
              starting_with=None,
              runner_class=None,
              suite_decorators=None,
+             stream=None,
              ):
     """Run the whole test suite under the enhanced runner"""
     # XXX: Very ugly way to do this...
@@ -3245,6 +3246,7 @@
                      strict=strict,
                      runner_class=runner_class,
                      suite_decorators=suite_decorators,
+                     stream=stream,
                      )
     finally:
         default_transport = old_transport
 
=== modified file 'bzrlib/tests/blackbox/test_selftest.py'
--- bzrlib/tests/blackbox/test_selftest.py	2009-07-10 07:14:02 +0000
+++ bzrlib/tests/blackbox/test_selftest.py	2009-08-24 05:35:11 +0000
@@ -26,6 +26,7 @@
 import bzrlib
 from bzrlib import (
     osutils,
+    tests,
     )
 from bzrlib.errors import ParamikoNotPresent
 from bzrlib.tests import (
@@ -42,67 +43,59 @@
 
 class TestOptions(TestCase):
 
-    current_test = None
-
     def test_transport_set_to_sftp(self):
-        # test the --transport option has taken effect from within the
-        # test_transport test
+        # Test that we can pass a transport to the selftest core - sftp
+        # version.
         try:
             import bzrlib.transport.sftp
         except ParamikoNotPresent:
             raise TestSkipped("Paramiko not present")
-        if TestOptions.current_test != "test_transport_set_to_sftp":
-            return
+        params = self.get_params_passed_to_core('selftest --transport=sftp')
         self.assertEqual(bzrlib.transport.sftp.SFTPAbsoluteServer,
-                         bzrlib.tests.default_transport)
+            params[1]["transport"])
 
     def test_transport_set_to_memory(self):
-        # test the --transport option has taken effect from within the
-        # test_transport test
+        # Test that we can pass a transport to the selftest core - memory
+        # version.
         import bzrlib.transport.memory
-        if TestOptions.current_test != "test_transport_set_to_memory":
-            return
+        params = self.get_params_passed_to_core('selftest --transport=memory')
         self.assertEqual(bzrlib.transport.memory.MemoryServer,
-                         bzrlib.tests.default_transport)
-
-    def test_transport(self):
-        # test that --transport=sftp works
-        try:
-            import bzrlib.transport.sftp
-        except ParamikoNotPresent:
-            raise TestSkipped("Paramiko not present")
-        old_transport = bzrlib.tests.default_transport
-        old_root = TestCaseWithMemoryTransport.TEST_ROOT
-        TestCaseWithMemoryTransport.TEST_ROOT = None
-        try:
-            TestOptions.current_test = "test_transport_set_to_sftp"
-            stdout = self.run_bzr(
-                'selftest --transport=sftp test_transport_set_to_sftp')[0]
-            self.assertContainsRe(stdout, 'Ran 1 test')
-            self.assertEqual(old_transport, bzrlib.tests.default_transport)
-
-            TestOptions.current_test = "test_transport_set_to_memory"
-            stdout = self.run_bzr(
-                'selftest --transport=memory test_transport_set_to_memory')[0]
-            self.assertContainsRe(stdout, 'Ran 1 test')
-            self.assertEqual(old_transport, bzrlib.tests.default_transport)
+            params[1]["transport"])
+
+    def get_params_passed_to_core(self, cmdline):
+        params = []
+        def selftest(*args, **kwargs):
+            """Capture the arguments selftest was run with."""
+            params.append((args, kwargs))
+            return True
+        # Yes this prevents using threads to run the test suite in parallel,
+        # however we don't have a clean dependency injector for commands, 
+        # and even if we did - we'd still be testing that the glue is wired
+        # up correctly. XXX: TODO: Solve this testing problem.
+        original_selftest = tests.selftest
+        tests.selftest = selftest
+        try:
+            self.run_bzr(cmdline)
+            return params[0]
         finally:
-            bzrlib.tests.default_transport = old_transport
-            TestOptions.current_test = None
-            TestCaseWithMemoryTransport.TEST_ROOT = old_root
+            tests.selftest = original_selftest
+
+    def test_parameters_passed_to_core(self):
+        params = self.get_params_passed_to_core('selftest --list-only')
+        self.assertTrue("list_only" in params[1])
+        params = self.get_params_passed_to_core('selftest --list-only selftest')
+        self.assertTrue("list_only" in params[1])
+        params = self.get_params_passed_to_core(['selftest', '--list-only',
+            '--exclude', 'selftest'])
+        self.assertTrue("list_only" in params[1])
+        params = self.get_params_passed_to_core(['selftest', '--list-only',
+            'selftest', '--randomize', 'now'])
+        self.assertSubset(["list_only", "random_seed"], params[1])
 
     def test_subunit(self):
-        """Passing --subunit results in subunit output."""
         self.requireFeature(SubUnitFeature)
-        from subunit import ProtocolTestCase
-        stdout = self.run_bzr(
-            'selftest --subunit --no-plugins '
-            'tests.test_selftest.SelftestTests.test_import_tests')[0]
-        stream = StringIO(str(stdout))
-        test = ProtocolTestCase(stream)
-        result = unittest.TestResult()
-        test.run(result)
-        self.assertEqual(1, result.testsRun)
+        params = self.get_params_passed_to_core('selftest --subunit')
+        self.assertEqual(tests.SubUnitBzrRunner, params[1]['runner_class'])
 
 
 class TestRunBzr(ExternalBase):
@@ -512,59 +505,29 @@
         return (header,body,footer)
 
     def test_list_only(self):
-        # check that bzr selftest --list-only works correctly
-        out,err = self.run_bzr('selftest selftest --list-only')
-        (header,body,footer) = self._parse_test_list(out.splitlines())
-        num_tests = len(body)
-        self.assertLength(0, header)
-        self.assertLength(0, footer)
-        self.assertEqual('', err)
-
-    def test_list_only_filtered(self):
-        # check that a filtered --list-only works, both include and exclude
-        out_all,err_all = self.run_bzr('selftest --list-only')
-        tests_all = self._parse_test_list(out_all.splitlines())[1]
-        out_incl,err_incl = self.run_bzr('selftest --list-only selftest')
-        tests_incl = self._parse_test_list(out_incl.splitlines())[1]
-        self.assertSubset(tests_incl, tests_all)
-        out_excl,err_excl = self.run_bzr(['selftest', '--list-only',
-                                          '--exclude', 'selftest'])
-        tests_excl = self._parse_test_list(out_excl.splitlines())[1]
-        self.assertSubset(tests_excl, tests_all)
-        set_incl = set(tests_incl)
-        set_excl = set(tests_excl)
-        intersection = set_incl.intersection(set_excl)
-        self.assertEquals(0, len(intersection))
-        self.assertEquals(len(tests_all), len(tests_incl) + len(tests_excl))
-
-    def test_list_only_random(self):
-        # check that --randomize works correctly
-        out_all,err_all = self.run_bzr('selftest --list-only selftest')
-        tests_all = self._parse_test_list(out_all.splitlines())[1]
-        # XXX: It looks like there are some orders for generating tests that
-        # fail as of 20070504 - maybe because of import order dependencies.
-        # So unfortunately this will rarely intermittently fail at the moment.
-        # -- mbp 20070504
-        out_rand,err_rand = self.run_bzr(['selftest', '--list-only',
-                                          'selftest', '--randomize', 'now'])
-        (header_rand,tests_rand,dummy) = self._parse_test_list(
-            out_rand.splitlines(), 1)
-        # XXX: The following line asserts that the randomized order is not the
-        # same as the default order.  It is just possible that they'll get
-        # randomized into the same order and this will falsely fail, but
-        # that's very unlikely in practice because there are thousands of
-        # tests.
-        self.assertNotEqual(tests_all, tests_rand)
-        self.assertEqual(sorted(tests_all), sorted(tests_rand))
-        # Check that the seed can be reused to get the exact same order
-        seed_re = re.compile('Randomizing test order using seed (\w+)')
-        match_obj = seed_re.search(header_rand[-1])
-        seed = match_obj.group(1)
-        out_rand2,err_rand2 = self.run_bzr(['selftest', '--list-only',
-                                            'selftest', '--randomize', seed])
-        (header_rand2,tests_rand2,dummy) = self._parse_test_list(
-            out_rand2.splitlines(), 1)
-        self.assertEqual(tests_rand, tests_rand2)
+        # check that bzr selftest --list-only outputs no ui noise
+        def selftest(*args, **kwargs):
+            """Capture the arguments selftest was run with."""
+            return True
+        def outputs_nothing(cmdline):
+            out,err = self.run_bzr(cmdline)
+            (header,body,footer) = self._parse_test_list(out.splitlines())
+            num_tests = len(body)
+            self.assertLength(0, header)
+            self.assertLength(0, footer)
+            self.assertEqual('', err)
+        # Yes this prevents using threads to run the test suite in parallel,
+        # however we don't have a clean dependency injector for commands, 
+        # and even if we did - we'd still be testing that the glue is wired
+        # up correctly. XXX: TODO: Solve this testing problem.
+        original_selftest = tests.selftest
+        tests.selftest = selftest
+        try:
+            outputs_nothing('selftest --list-only')
+            outputs_nothing('selftest --list-only selftest')
+            outputs_nothing(['selftest', '--list-only', '--exclude', 'selftest'])
+        finally:
+            tests.selftest = original_selftest
 
 
 class TestSelftestWithIdList(TestCaseInTempDir):
 
=== modified file 'bzrlib/tests/test_selftest.py'
--- bzrlib/tests/test_selftest.py	2009-08-17 03:47:03 +0000
+++ bzrlib/tests/test_selftest.py	2009-08-24 05:35:11 +0000
@@ -50,6 +50,7 @@
     deprecated_method,
     )
 from bzrlib.tests import (
+    SubUnitFeature,
     test_lsprof,
     test_sftp_transport,
     TestUtil,
@@ -1770,6 +1771,99 @@
             test_suite_factory=factory)
         self.assertEqual([True], factory_called)
 
+    def factory(self):
+        """A test suite factory."""
+        class Test(tests.TestCase):
+            def a(self):
+                pass
+            def b(self):
+                pass
+            def c(self):
+                pass
+        return TestUtil.TestSuite([Test("a"), Test("b"), Test("c")])
+
+    def run_selftest(self, **kwargs):
+        """Run selftest returning its output."""
+        output = StringIO()
+        old_transport = bzrlib.tests.default_transport
+        old_root = tests.TestCaseWithMemoryTransport.TEST_ROOT
+        tests.TestCaseWithMemoryTransport.TEST_ROOT = None
+        try:
+            self.assertEqual(True, tests.selftest(stream=output, **kwargs))
+        finally:
+            bzrlib.tests.default_transport = old_transport
+            tests.TestCaseWithMemoryTransport.TEST_ROOT = old_root
+        output.seek(0)
+        return output
+
+    def test_list_only(self):
+        output = self.run_selftest(test_suite_factory=self.factory,
+            list_only=True)
+        self.assertEqual(3, len(output.readlines()))
+
+    def test_list_only_filtered(self):
+        output = self.run_selftest(test_suite_factory=self.factory,
+            list_only=True, pattern="Test.b")
+        self.assertEndsWith(output.getvalue(), "Test.b\n")
+        self.assertLength(1, output.readlines())
+
+    def test_list_only_excludes(self):
+        output = self.run_selftest(test_suite_factory=self.factory,
+            list_only=True, exclude_pattern="Test.b")
+        self.assertNotContainsRe("Test.b", output.getvalue())
+        self.assertLength(2, output.readlines())
+
+    def test_random(self):
+        # test randomising by listing a number of tests.
+        output_123 = self.run_selftest(test_suite_factory=self.factory,
+            list_only=True, random_seed="123")
+        output_234 = self.run_selftest(test_suite_factory=self.factory,
+            list_only=True, random_seed="234")
+        self.assertNotEqual(output_123, output_234)
+        # "Randominzing test order..\n\n
+        self.assertLength(5, output_123.readlines())
+        self.assertLength(5, output_234.readlines())
+
+    def test_random_reuse_is_same_order(self):
+        # test randomising by listing a number of tests.
+        expected = self.run_selftest(test_suite_factory=self.factory,
+            list_only=True, random_seed="123")
+        repeated = self.run_selftest(test_suite_factory=self.factory,
+            list_only=True, random_seed="123")
+        self.assertEqual(expected.getvalue(), repeated.getvalue())
+
+    def test_runner_class(self):
+        self.requireFeature(SubUnitFeature)
+        from subunit import ProtocolTestCase
+        stream = self.run_selftest(runner_class=tests.SubUnitBzrRunner,
+            test_suite_factory=self.factory)
+        test = ProtocolTestCase(stream)
+        result = unittest.TestResult()
+        test.run(result)
+        self.assertEqual(3, result.testsRun)
+
+    def check_transport_set(self, transport_server):
+        captured_transport = []
+        def seen_transport(a_transport):
+            captured_transport.append(a_transport)
+        class Capture(tests.TestCase):
+            def a(self):
+                seen_transport(bzrlib.tests.default_transport)
+        def factory():
+            return TestUtil.TestSuite([Capture("a")])
+        self.run_selftest(transport=transport_server, test_suite_factory=factory)
+        self.assertEqual(transport_server, captured_transport[0])
+
+    def test_transport_sftp(self):
+        try:
+            import bzrlib.transport.sftp
+        except ParamikoNotPresent:
+            raise TestSkipped("Paramiko not present")
+        self.check_transport_set(bzrlib.transport.sftp.SFTPAbsoluteServer)
+
+    def test_transport_memory(self):
+        self.check_transport_set(bzrlib.transport.memory.MemoryServer)
+
 
 class TestKnownFailure(tests.TestCase):