Merge lp:~lifeless/bzr/test-speed into lp:~bzr/bzr/trunk-old
- test-speed
- Merge into trunk-old
Proposed by
Robert Collins
Status: | Superseded |
---|---|
Proposed branch: | lp:~lifeless/bzr/test-speed |
Merge into: | lp:~bzr/bzr/trunk-old |
Diff against target: | 204 lines |
To merge this branch: | bzr merge lp:~lifeless/bzr/test-speed |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Martin Pool | Approve | ||
Review via email:
|
This proposal has been superseded by a proposal from 2009-08-23.
Commit message
Description of the change
To post a comment you must log in.
Revision history for this message
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
Robert Collins (lifeless) wrote : | # |
Revision history for this message
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
Martin Pool (mbp) wrote : | # |
I think those upper-layer tests are almost pointless, but fixing the lower layers to work on smaller data is certainly welcome.
review:
Approve
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'bzrlib/tests/__init__.py' |
2 | --- bzrlib/tests/__init__.py 2009-08-20 05:05:59 +0000 |
3 | +++ bzrlib/tests/__init__.py 2009-08-23 22:35:09 +0000 |
4 | @@ -3207,6 +3207,7 @@ |
5 | starting_with=None, |
6 | runner_class=None, |
7 | suite_decorators=None, |
8 | + stream=None, |
9 | ): |
10 | """Run the whole test suite under the enhanced runner""" |
11 | # XXX: Very ugly way to do this... |
12 | @@ -3245,6 +3246,7 @@ |
13 | strict=strict, |
14 | runner_class=runner_class, |
15 | suite_decorators=suite_decorators, |
16 | + stream=stream, |
17 | ) |
18 | finally: |
19 | default_transport = old_transport |
20 | |
21 | === modified file 'bzrlib/tests/blackbox/test_selftest.py' |
22 | --- bzrlib/tests/blackbox/test_selftest.py 2009-07-10 07:14:02 +0000 |
23 | +++ bzrlib/tests/blackbox/test_selftest.py 2009-08-23 22:35:09 +0000 |
24 | @@ -26,6 +26,7 @@ |
25 | import bzrlib |
26 | from bzrlib import ( |
27 | osutils, |
28 | + tests, |
29 | ) |
30 | from bzrlib.errors import ParamikoNotPresent |
31 | from bzrlib.tests import ( |
32 | @@ -512,59 +513,55 @@ |
33 | return (header,body,footer) |
34 | |
35 | def test_list_only(self): |
36 | - # check that bzr selftest --list-only works correctly |
37 | - out,err = self.run_bzr('selftest selftest --list-only') |
38 | - (header,body,footer) = self._parse_test_list(out.splitlines()) |
39 | - num_tests = len(body) |
40 | - self.assertLength(0, header) |
41 | - self.assertLength(0, footer) |
42 | - self.assertEqual('', err) |
43 | - |
44 | - def test_list_only_filtered(self): |
45 | - # check that a filtered --list-only works, both include and exclude |
46 | - out_all,err_all = self.run_bzr('selftest --list-only') |
47 | - tests_all = self._parse_test_list(out_all.splitlines())[1] |
48 | - out_incl,err_incl = self.run_bzr('selftest --list-only selftest') |
49 | - tests_incl = self._parse_test_list(out_incl.splitlines())[1] |
50 | - self.assertSubset(tests_incl, tests_all) |
51 | - out_excl,err_excl = self.run_bzr(['selftest', '--list-only', |
52 | - '--exclude', 'selftest']) |
53 | - tests_excl = self._parse_test_list(out_excl.splitlines())[1] |
54 | - self.assertSubset(tests_excl, tests_all) |
55 | - set_incl = set(tests_incl) |
56 | - set_excl = set(tests_excl) |
57 | - intersection = set_incl.intersection(set_excl) |
58 | - self.assertEquals(0, len(intersection)) |
59 | - self.assertEquals(len(tests_all), len(tests_incl) + len(tests_excl)) |
60 | - |
61 | - def test_list_only_random(self): |
62 | - # check that --randomize works correctly |
63 | - out_all,err_all = self.run_bzr('selftest --list-only selftest') |
64 | - tests_all = self._parse_test_list(out_all.splitlines())[1] |
65 | - # XXX: It looks like there are some orders for generating tests that |
66 | - # fail as of 20070504 - maybe because of import order dependencies. |
67 | - # So unfortunately this will rarely intermittently fail at the moment. |
68 | - # -- mbp 20070504 |
69 | - out_rand,err_rand = self.run_bzr(['selftest', '--list-only', |
70 | - 'selftest', '--randomize', 'now']) |
71 | - (header_rand,tests_rand,dummy) = self._parse_test_list( |
72 | - out_rand.splitlines(), 1) |
73 | - # XXX: The following line asserts that the randomized order is not the |
74 | - # same as the default order. It is just possible that they'll get |
75 | - # randomized into the same order and this will falsely fail, but |
76 | - # that's very unlikely in practice because there are thousands of |
77 | - # tests. |
78 | - self.assertNotEqual(tests_all, tests_rand) |
79 | - self.assertEqual(sorted(tests_all), sorted(tests_rand)) |
80 | - # Check that the seed can be reused to get the exact same order |
81 | - seed_re = re.compile('Randomizing test order using seed (\w+)') |
82 | - match_obj = seed_re.search(header_rand[-1]) |
83 | - seed = match_obj.group(1) |
84 | - out_rand2,err_rand2 = self.run_bzr(['selftest', '--list-only', |
85 | - 'selftest', '--randomize', seed]) |
86 | - (header_rand2,tests_rand2,dummy) = self._parse_test_list( |
87 | - out_rand2.splitlines(), 1) |
88 | - self.assertEqual(tests_rand, tests_rand2) |
89 | + # check that bzr selftest --list-only outputs no ui noise |
90 | + def selftest(*args, **kwargs): |
91 | + """Capture the arguments selftest was run with.""" |
92 | + return True |
93 | + def outputs_nothing(cmdline): |
94 | + out,err = self.run_bzr(cmdline) |
95 | + (header,body,footer) = self._parse_test_list(out.splitlines()) |
96 | + num_tests = len(body) |
97 | + self.assertLength(0, header) |
98 | + self.assertLength(0, footer) |
99 | + self.assertEqual('', err) |
100 | + # Yes this prevents using threads to run the test suite in parallel, |
101 | + # however we don't have a clean dependency injector for commands, |
102 | + # and even if we did - we'd still be testing that the glue is wired |
103 | + # up correctly. XXX: TODO: Solve this testing problem. |
104 | + original_selftest = tests.selftest |
105 | + tests.selftest = selftest |
106 | + try: |
107 | + outputs_nothing('selftest --list-only') |
108 | + outputs_nothing('selftest --list-only selftest') |
109 | + outputs_nothing(['selftest', '--list-only', '--exclude', 'selftest']) |
110 | + finally: |
111 | + tests.selftest = original_selftest |
112 | + |
113 | + def test_parameters_passed_to_core(self): |
114 | + params = [] |
115 | + def selftest(*args, **kwargs): |
116 | + """Capture the arguments selftest was run with.""" |
117 | + params.append((args, kwargs)) |
118 | + return True |
119 | + # Yes this prevents using threads to run the test suite in parallel, |
120 | + # however we don't have a clean dependency injector for commands, |
121 | + # and even if we did - we'd still be testing that the glue is wired |
122 | + # up correctly. XXX: TODO: Solve this testing problem. |
123 | + original_selftest = tests.selftest |
124 | + tests.selftest = selftest |
125 | + try: |
126 | + self.run_bzr('selftest --list-only') |
127 | + self.run_bzr('selftest --list-only selftest') |
128 | + self.run_bzr(['selftest', '--list-only', '--exclude', 'selftest']) |
129 | + self.run_bzr(['selftest', '--list-only', 'selftest', |
130 | + '--randomize', 'now']) |
131 | + # list_only should have been passed in each invocation. |
132 | + self.assertTrue("list_only" in params[0][1]) |
133 | + self.assertTrue("list_only" in params[1][1]) |
134 | + self.assertTrue("list_only" in params[2][1]) |
135 | + self.assertSubset(["list_only", "random_seed"], params[2][1]) |
136 | + finally: |
137 | + tests.selftest = original_selftest |
138 | |
139 | |
140 | class TestSelftestWithIdList(TestCaseInTempDir): |
141 | |
142 | === modified file 'bzrlib/tests/test_selftest.py' |
143 | --- bzrlib/tests/test_selftest.py 2009-08-17 03:47:03 +0000 |
144 | +++ bzrlib/tests/test_selftest.py 2009-08-23 22:35:09 +0000 |
145 | @@ -1770,6 +1770,60 @@ |
146 | test_suite_factory=factory) |
147 | self.assertEqual([True], factory_called) |
148 | |
149 | + def factory(self): |
150 | + """A test suite factory.""" |
151 | + class Test(tests.TestCase): |
152 | + def a(self): |
153 | + pass |
154 | + def b(self): |
155 | + pass |
156 | + def c(self): |
157 | + pass |
158 | + return TestUtil.TestSuite([Test("a"), Test("b"), Test("c")]) |
159 | + |
160 | + def run_selftest(self, **kwargs): |
161 | + """Run selftest returning its output.""" |
162 | + output = StringIO() |
163 | + self.assertEqual(True, tests.selftest(stream=output, **kwargs)) |
164 | + output.seek(0) |
165 | + return output |
166 | + |
167 | + def test_list_only(self): |
168 | + output = self.run_selftest(test_suite_factory=self.factory, |
169 | + list_only=True) |
170 | + self.assertEqual(3, len(output.readlines())) |
171 | + |
172 | + def test_list_only_filtered(self): |
173 | + output = self.run_selftest(test_suite_factory=self.factory, |
174 | + list_only=True, pattern="Test.b") |
175 | + self.assertEndsWith(output.getvalue(), "Test.b\n") |
176 | + self.assertLength(1, output.readlines()) |
177 | + |
178 | + def test_list_only_excludes(self): |
179 | + output = self.run_selftest(test_suite_factory=self.factory, |
180 | + list_only=True, exclude_pattern="Test.b") |
181 | + self.assertNotContainsRe("Test.b", output.getvalue()) |
182 | + self.assertLength(2, output.readlines()) |
183 | + |
184 | + def test_random(self): |
185 | + # test randomising by listing a number of tests. |
186 | + output_123 = self.run_selftest(test_suite_factory=self.factory, |
187 | + list_only=True, random_seed="123") |
188 | + output_234 = self.run_selftest(test_suite_factory=self.factory, |
189 | + list_only=True, random_seed="234") |
190 | + self.assertNotEqual(output_123, output_234) |
191 | + # "Randominzing test order..\n\n |
192 | + self.assertLength(5, output_123.readlines()) |
193 | + self.assertLength(5, output_234.readlines()) |
194 | + |
195 | + def test_random_reuse_is_same_order(self): |
196 | + # test randomising by listing a number of tests. |
197 | + expected = self.run_selftest(test_suite_factory=self.factory, |
198 | + list_only=True, random_seed="123") |
199 | + repeated = self.run_selftest(test_suite_factory=self.factory, |
200 | + list_only=True, random_seed="123") |
201 | + self.assertEqual(expected.getvalue(), repeated.getvalue()) |
202 | + |
203 | |
204 | class TestKnownFailure(tests.TestCase): |
205 |
This turns two 13 second tests into a few millisecond-scale tests
testing more precise layers.
-Rob
--