Merge lp:~lifeless/bzr/test-speed into lp:~bzr/bzr/trunk-old
- test-speed
- Merge into trunk-old
Proposed by
Robert Collins
Status: | Merged |
---|---|
Merge reported by: | Robert Collins |
Merged at revision: | not available |
Proposed branch: | lp:~lifeless/bzr/test-speed |
Merge into: | lp:~bzr/bzr/trunk-old |
Diff against target: | 332 lines |
To merge this branch: | bzr merge lp:~lifeless/bzr/test-speed |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Martin Pool | Approve | ||
Review via email: mp+10584@code.launchpad.net |
This proposal supersedes a proposal from 2009-08-23.
Commit message
Description of the change
To post a comment you must log in.
Revision history for this message
Robert Collins (lifeless) wrote : Posted in a previous version of this proposal | # |
Revision history for this message
Martin Pool (mbp) wrote : Posted in a previous version of this proposal | # |
I think those upper-layer tests are almost pointless, but fixing the lower layers to work on smaller data is certainly welcome.
review:
Approve
Revision history for this message
Robert Collins (lifeless) wrote : | # |
I ended up doing a couple more things while I was there.
< 10 seconds for the selftest tests now.
Revision history for this message
Martin Pool (mbp) wrote : | # |
> I ended up doing a couple more things while I was there.
> < 10 seconds for the selftest tests now.
review:
Approve
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'bzrlib/tests/__init__.py' |
2 | --- bzrlib/tests/__init__.py 2009-08-20 05:05:59 +0000 |
3 | +++ bzrlib/tests/__init__.py 2009-08-24 05:35:11 +0000 |
4 | @@ -3207,6 +3207,7 @@ |
5 | starting_with=None, |
6 | runner_class=None, |
7 | suite_decorators=None, |
8 | + stream=None, |
9 | ): |
10 | """Run the whole test suite under the enhanced runner""" |
11 | # XXX: Very ugly way to do this... |
12 | @@ -3245,6 +3246,7 @@ |
13 | strict=strict, |
14 | runner_class=runner_class, |
15 | suite_decorators=suite_decorators, |
16 | + stream=stream, |
17 | ) |
18 | finally: |
19 | default_transport = old_transport |
20 | |
21 | === modified file 'bzrlib/tests/blackbox/test_selftest.py' |
22 | --- bzrlib/tests/blackbox/test_selftest.py 2009-07-10 07:14:02 +0000 |
23 | +++ bzrlib/tests/blackbox/test_selftest.py 2009-08-24 05:35:11 +0000 |
24 | @@ -26,6 +26,7 @@ |
25 | import bzrlib |
26 | from bzrlib import ( |
27 | osutils, |
28 | + tests, |
29 | ) |
30 | from bzrlib.errors import ParamikoNotPresent |
31 | from bzrlib.tests import ( |
32 | @@ -42,67 +43,59 @@ |
33 | |
34 | class TestOptions(TestCase): |
35 | |
36 | - current_test = None |
37 | - |
38 | def test_transport_set_to_sftp(self): |
39 | - # test the --transport option has taken effect from within the |
40 | - # test_transport test |
41 | + # Test that we can pass a transport to the selftest core - sftp |
42 | + # version. |
43 | try: |
44 | import bzrlib.transport.sftp |
45 | except ParamikoNotPresent: |
46 | raise TestSkipped("Paramiko not present") |
47 | - if TestOptions.current_test != "test_transport_set_to_sftp": |
48 | - return |
49 | + params = self.get_params_passed_to_core('selftest --transport=sftp') |
50 | self.assertEqual(bzrlib.transport.sftp.SFTPAbsoluteServer, |
51 | - bzrlib.tests.default_transport) |
52 | + params[1]["transport"]) |
53 | |
54 | def test_transport_set_to_memory(self): |
55 | - # test the --transport option has taken effect from within the |
56 | - # test_transport test |
57 | + # Test that we can pass a transport to the selftest core - memory |
58 | + # version. |
59 | import bzrlib.transport.memory |
60 | - if TestOptions.current_test != "test_transport_set_to_memory": |
61 | - return |
62 | + params = self.get_params_passed_to_core('selftest --transport=memory') |
63 | self.assertEqual(bzrlib.transport.memory.MemoryServer, |
64 | - bzrlib.tests.default_transport) |
65 | - |
66 | - def test_transport(self): |
67 | - # test that --transport=sftp works |
68 | - try: |
69 | - import bzrlib.transport.sftp |
70 | - except ParamikoNotPresent: |
71 | - raise TestSkipped("Paramiko not present") |
72 | - old_transport = bzrlib.tests.default_transport |
73 | - old_root = TestCaseWithMemoryTransport.TEST_ROOT |
74 | - TestCaseWithMemoryTransport.TEST_ROOT = None |
75 | - try: |
76 | - TestOptions.current_test = "test_transport_set_to_sftp" |
77 | - stdout = self.run_bzr( |
78 | - 'selftest --transport=sftp test_transport_set_to_sftp')[0] |
79 | - self.assertContainsRe(stdout, 'Ran 1 test') |
80 | - self.assertEqual(old_transport, bzrlib.tests.default_transport) |
81 | - |
82 | - TestOptions.current_test = "test_transport_set_to_memory" |
83 | - stdout = self.run_bzr( |
84 | - 'selftest --transport=memory test_transport_set_to_memory')[0] |
85 | - self.assertContainsRe(stdout, 'Ran 1 test') |
86 | - self.assertEqual(old_transport, bzrlib.tests.default_transport) |
87 | + params[1]["transport"]) |
88 | + |
89 | + def get_params_passed_to_core(self, cmdline): |
90 | + params = [] |
91 | + def selftest(*args, **kwargs): |
92 | + """Capture the arguments selftest was run with.""" |
93 | + params.append((args, kwargs)) |
94 | + return True |
95 | + # Yes this prevents using threads to run the test suite in parallel, |
96 | + # however we don't have a clean dependency injector for commands, |
97 | + # and even if we did - we'd still be testing that the glue is wired |
98 | + # up correctly. XXX: TODO: Solve this testing problem. |
99 | + original_selftest = tests.selftest |
100 | + tests.selftest = selftest |
101 | + try: |
102 | + self.run_bzr(cmdline) |
103 | + return params[0] |
104 | finally: |
105 | - bzrlib.tests.default_transport = old_transport |
106 | - TestOptions.current_test = None |
107 | - TestCaseWithMemoryTransport.TEST_ROOT = old_root |
108 | + tests.selftest = original_selftest |
109 | + |
110 | + def test_parameters_passed_to_core(self): |
111 | + params = self.get_params_passed_to_core('selftest --list-only') |
112 | + self.assertTrue("list_only" in params[1]) |
113 | + params = self.get_params_passed_to_core('selftest --list-only selftest') |
114 | + self.assertTrue("list_only" in params[1]) |
115 | + params = self.get_params_passed_to_core(['selftest', '--list-only', |
116 | + '--exclude', 'selftest']) |
117 | + self.assertTrue("list_only" in params[1]) |
118 | + params = self.get_params_passed_to_core(['selftest', '--list-only', |
119 | + 'selftest', '--randomize', 'now']) |
120 | + self.assertSubset(["list_only", "random_seed"], params[1]) |
121 | |
122 | def test_subunit(self): |
123 | - """Passing --subunit results in subunit output.""" |
124 | self.requireFeature(SubUnitFeature) |
125 | - from subunit import ProtocolTestCase |
126 | - stdout = self.run_bzr( |
127 | - 'selftest --subunit --no-plugins ' |
128 | - 'tests.test_selftest.SelftestTests.test_import_tests')[0] |
129 | - stream = StringIO(str(stdout)) |
130 | - test = ProtocolTestCase(stream) |
131 | - result = unittest.TestResult() |
132 | - test.run(result) |
133 | - self.assertEqual(1, result.testsRun) |
134 | + params = self.get_params_passed_to_core('selftest --subunit') |
135 | + self.assertEqual(tests.SubUnitBzrRunner, params[1]['runner_class']) |
136 | |
137 | |
138 | class TestRunBzr(ExternalBase): |
139 | @@ -512,59 +505,29 @@ |
140 | return (header,body,footer) |
141 | |
142 | def test_list_only(self): |
143 | - # check that bzr selftest --list-only works correctly |
144 | - out,err = self.run_bzr('selftest selftest --list-only') |
145 | - (header,body,footer) = self._parse_test_list(out.splitlines()) |
146 | - num_tests = len(body) |
147 | - self.assertLength(0, header) |
148 | - self.assertLength(0, footer) |
149 | - self.assertEqual('', err) |
150 | - |
151 | - def test_list_only_filtered(self): |
152 | - # check that a filtered --list-only works, both include and exclude |
153 | - out_all,err_all = self.run_bzr('selftest --list-only') |
154 | - tests_all = self._parse_test_list(out_all.splitlines())[1] |
155 | - out_incl,err_incl = self.run_bzr('selftest --list-only selftest') |
156 | - tests_incl = self._parse_test_list(out_incl.splitlines())[1] |
157 | - self.assertSubset(tests_incl, tests_all) |
158 | - out_excl,err_excl = self.run_bzr(['selftest', '--list-only', |
159 | - '--exclude', 'selftest']) |
160 | - tests_excl = self._parse_test_list(out_excl.splitlines())[1] |
161 | - self.assertSubset(tests_excl, tests_all) |
162 | - set_incl = set(tests_incl) |
163 | - set_excl = set(tests_excl) |
164 | - intersection = set_incl.intersection(set_excl) |
165 | - self.assertEquals(0, len(intersection)) |
166 | - self.assertEquals(len(tests_all), len(tests_incl) + len(tests_excl)) |
167 | - |
168 | - def test_list_only_random(self): |
169 | - # check that --randomize works correctly |
170 | - out_all,err_all = self.run_bzr('selftest --list-only selftest') |
171 | - tests_all = self._parse_test_list(out_all.splitlines())[1] |
172 | - # XXX: It looks like there are some orders for generating tests that |
173 | - # fail as of 20070504 - maybe because of import order dependencies. |
174 | - # So unfortunately this will rarely intermittently fail at the moment. |
175 | - # -- mbp 20070504 |
176 | - out_rand,err_rand = self.run_bzr(['selftest', '--list-only', |
177 | - 'selftest', '--randomize', 'now']) |
178 | - (header_rand,tests_rand,dummy) = self._parse_test_list( |
179 | - out_rand.splitlines(), 1) |
180 | - # XXX: The following line asserts that the randomized order is not the |
181 | - # same as the default order. It is just possible that they'll get |
182 | - # randomized into the same order and this will falsely fail, but |
183 | - # that's very unlikely in practice because there are thousands of |
184 | - # tests. |
185 | - self.assertNotEqual(tests_all, tests_rand) |
186 | - self.assertEqual(sorted(tests_all), sorted(tests_rand)) |
187 | - # Check that the seed can be reused to get the exact same order |
188 | - seed_re = re.compile('Randomizing test order using seed (\w+)') |
189 | - match_obj = seed_re.search(header_rand[-1]) |
190 | - seed = match_obj.group(1) |
191 | - out_rand2,err_rand2 = self.run_bzr(['selftest', '--list-only', |
192 | - 'selftest', '--randomize', seed]) |
193 | - (header_rand2,tests_rand2,dummy) = self._parse_test_list( |
194 | - out_rand2.splitlines(), 1) |
195 | - self.assertEqual(tests_rand, tests_rand2) |
196 | + # check that bzr selftest --list-only outputs no ui noise |
197 | + def selftest(*args, **kwargs): |
198 | + """Capture the arguments selftest was run with.""" |
199 | + return True |
200 | + def outputs_nothing(cmdline): |
201 | + out,err = self.run_bzr(cmdline) |
202 | + (header,body,footer) = self._parse_test_list(out.splitlines()) |
203 | + num_tests = len(body) |
204 | + self.assertLength(0, header) |
205 | + self.assertLength(0, footer) |
206 | + self.assertEqual('', err) |
207 | + # Yes this prevents using threads to run the test suite in parallel, |
208 | + # however we don't have a clean dependency injector for commands, |
209 | + # and even if we did - we'd still be testing that the glue is wired |
210 | + # up correctly. XXX: TODO: Solve this testing problem. |
211 | + original_selftest = tests.selftest |
212 | + tests.selftest = selftest |
213 | + try: |
214 | + outputs_nothing('selftest --list-only') |
215 | + outputs_nothing('selftest --list-only selftest') |
216 | + outputs_nothing(['selftest', '--list-only', '--exclude', 'selftest']) |
217 | + finally: |
218 | + tests.selftest = original_selftest |
219 | |
220 | |
221 | class TestSelftestWithIdList(TestCaseInTempDir): |
222 | |
223 | === modified file 'bzrlib/tests/test_selftest.py' |
224 | --- bzrlib/tests/test_selftest.py 2009-08-17 03:47:03 +0000 |
225 | +++ bzrlib/tests/test_selftest.py 2009-08-24 05:35:11 +0000 |
226 | @@ -50,6 +50,7 @@ |
227 | deprecated_method, |
228 | ) |
229 | from bzrlib.tests import ( |
230 | + SubUnitFeature, |
231 | test_lsprof, |
232 | test_sftp_transport, |
233 | TestUtil, |
234 | @@ -1770,6 +1771,99 @@ |
235 | test_suite_factory=factory) |
236 | self.assertEqual([True], factory_called) |
237 | |
238 | + def factory(self): |
239 | + """A test suite factory.""" |
240 | + class Test(tests.TestCase): |
241 | + def a(self): |
242 | + pass |
243 | + def b(self): |
244 | + pass |
245 | + def c(self): |
246 | + pass |
247 | + return TestUtil.TestSuite([Test("a"), Test("b"), Test("c")]) |
248 | + |
249 | + def run_selftest(self, **kwargs): |
250 | + """Run selftest returning its output.""" |
251 | + output = StringIO() |
252 | + old_transport = bzrlib.tests.default_transport |
253 | + old_root = tests.TestCaseWithMemoryTransport.TEST_ROOT |
254 | + tests.TestCaseWithMemoryTransport.TEST_ROOT = None |
255 | + try: |
256 | + self.assertEqual(True, tests.selftest(stream=output, **kwargs)) |
257 | + finally: |
258 | + bzrlib.tests.default_transport = old_transport |
259 | + tests.TestCaseWithMemoryTransport.TEST_ROOT = old_root |
260 | + output.seek(0) |
261 | + return output |
262 | + |
263 | + def test_list_only(self): |
264 | + output = self.run_selftest(test_suite_factory=self.factory, |
265 | + list_only=True) |
266 | + self.assertEqual(3, len(output.readlines())) |
267 | + |
268 | + def test_list_only_filtered(self): |
269 | + output = self.run_selftest(test_suite_factory=self.factory, |
270 | + list_only=True, pattern="Test.b") |
271 | + self.assertEndsWith(output.getvalue(), "Test.b\n") |
272 | + self.assertLength(1, output.readlines()) |
273 | + |
274 | + def test_list_only_excludes(self): |
275 | + output = self.run_selftest(test_suite_factory=self.factory, |
276 | + list_only=True, exclude_pattern="Test.b") |
277 | + self.assertNotContainsRe("Test.b", output.getvalue()) |
278 | + self.assertLength(2, output.readlines()) |
279 | + |
280 | + def test_random(self): |
281 | + # test randomising by listing a number of tests. |
282 | + output_123 = self.run_selftest(test_suite_factory=self.factory, |
283 | + list_only=True, random_seed="123") |
284 | + output_234 = self.run_selftest(test_suite_factory=self.factory, |
285 | + list_only=True, random_seed="234") |
286 | + self.assertNotEqual(output_123, output_234) |
287 | + # "Randominzing test order..\n\n |
288 | + self.assertLength(5, output_123.readlines()) |
289 | + self.assertLength(5, output_234.readlines()) |
290 | + |
291 | + def test_random_reuse_is_same_order(self): |
292 | + # test randomising by listing a number of tests. |
293 | + expected = self.run_selftest(test_suite_factory=self.factory, |
294 | + list_only=True, random_seed="123") |
295 | + repeated = self.run_selftest(test_suite_factory=self.factory, |
296 | + list_only=True, random_seed="123") |
297 | + self.assertEqual(expected.getvalue(), repeated.getvalue()) |
298 | + |
299 | + def test_runner_class(self): |
300 | + self.requireFeature(SubUnitFeature) |
301 | + from subunit import ProtocolTestCase |
302 | + stream = self.run_selftest(runner_class=tests.SubUnitBzrRunner, |
303 | + test_suite_factory=self.factory) |
304 | + test = ProtocolTestCase(stream) |
305 | + result = unittest.TestResult() |
306 | + test.run(result) |
307 | + self.assertEqual(3, result.testsRun) |
308 | + |
309 | + def check_transport_set(self, transport_server): |
310 | + captured_transport = [] |
311 | + def seen_transport(a_transport): |
312 | + captured_transport.append(a_transport) |
313 | + class Capture(tests.TestCase): |
314 | + def a(self): |
315 | + seen_transport(bzrlib.tests.default_transport) |
316 | + def factory(): |
317 | + return TestUtil.TestSuite([Capture("a")]) |
318 | + self.run_selftest(transport=transport_server, test_suite_factory=factory) |
319 | + self.assertEqual(transport_server, captured_transport[0]) |
320 | + |
321 | + def test_transport_sftp(self): |
322 | + try: |
323 | + import bzrlib.transport.sftp |
324 | + except ParamikoNotPresent: |
325 | + raise TestSkipped("Paramiko not present") |
326 | + self.check_transport_set(bzrlib.transport.sftp.SFTPAbsoluteServer) |
327 | + |
328 | + def test_transport_memory(self): |
329 | + self.check_transport_set(bzrlib.transport.memory.MemoryServer) |
330 | + |
331 | |
332 | class TestKnownFailure(tests.TestCase): |
333 |
This turns two 13 second tests into a few millisecond-scale tests
testing more precise layers.
-Rob
--