Merge lp:~sidnei/lazr.testing/yeti into lp:lazr.testing
- yeti
- Merge into trunk
Proposed by
Sidnei da Silva
Status: | Merged |
---|---|
Merged at revision: | 35 |
Proposed branch: | lp:~sidnei/lazr.testing/yeti |
Merge into: | lp:lazr.testing |
Diff against target: |
462 lines (+431/-3) 3 files modified
src/lazr/testing/tests/test_js.py (+10/-3) src/lazr/testing/tests/test_yeti.py (+272/-0) src/lazr/testing/yeti.py (+149/-0) |
To merge this branch: | bzr merge lp:~sidnei/lazr.testing/yeti |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Paul Hummer (community) | Approve | ||
Review via email: mp+44591@code.launchpad.net |
Commit message
Description of the change
- Adds a YetiLayer and YetiTestCase, which use a custom branch of yeti
with subunit output format to better integrate into unittest.
To post a comment you must log in.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'src/lazr/testing/tests/test_js.py' | |||
2 | --- src/lazr/testing/tests/test_js.py 2010-09-03 15:03:15 +0000 | |||
3 | +++ src/lazr/testing/tests/test_js.py 2010-12-23 16:11:33 +0000 | |||
4 | @@ -1,10 +1,12 @@ | |||
5 | 1 | import doctest | ||
6 | 2 | import operator | ||
7 | 1 | import os | 3 | import os |
8 | 2 | import re | 4 | import re |
9 | 3 | import unittest | ||
10 | 4 | import doctest | ||
11 | 5 | import socket | 5 | import socket |
12 | 6 | import operator | ||
13 | 7 | import sys | 6 | import sys |
14 | 7 | import unittest | ||
15 | 8 | import warnings | ||
16 | 9 | |||
17 | 8 | from cStringIO import StringIO | 10 | from cStringIO import StringIO |
18 | 9 | 11 | ||
19 | 10 | from os.path import dirname | 12 | from os.path import dirname |
20 | @@ -331,6 +333,11 @@ | |||
21 | 331 | def test_suite(): | 333 | def test_suite(): |
22 | 332 | suite = unittest.TestSuite() | 334 | suite = unittest.TestSuite() |
23 | 333 | 335 | ||
24 | 336 | if not "JSTESTDRIVER" in os.environ: | ||
25 | 337 | warnings.warn("Environment variable 'JSTESTDRIVER' not set. " | ||
26 | 338 | "Skipping 'JSTESTDRIVER' tests.") | ||
27 | 339 | return suite | ||
28 | 340 | |||
29 | 334 | if os.environ.get("JSTESTDRIVER_SELFTEST"): | 341 | if os.environ.get("JSTESTDRIVER_SELFTEST"): |
30 | 335 | suite.addTests(unittest.makeSuite(JsTestDriverSelfTest)) | 342 | suite.addTests(unittest.makeSuite(JsTestDriverSelfTest)) |
31 | 336 | else: | 343 | else: |
32 | 337 | 344 | ||
33 | === added file 'src/lazr/testing/tests/test_yeti.py' | |||
34 | --- src/lazr/testing/tests/test_yeti.py 1970-01-01 00:00:00 +0000 | |||
35 | +++ src/lazr/testing/tests/test_yeti.py 2010-12-23 16:11:33 +0000 | |||
36 | @@ -0,0 +1,272 @@ | |||
37 | 1 | import doctest | ||
38 | 2 | import operator | ||
39 | 3 | import os | ||
40 | 4 | import re | ||
41 | 5 | import socket | ||
42 | 6 | import sys | ||
43 | 7 | import unittest | ||
44 | 8 | import warnings | ||
45 | 9 | |||
46 | 10 | from cStringIO import StringIO | ||
47 | 11 | |||
48 | 12 | from os.path import dirname | ||
49 | 13 | |||
50 | 14 | from mocker import ANY, ARGS, KWARGS, MockerTestCase | ||
51 | 15 | |||
52 | 16 | from zope.testing import testrunner | ||
53 | 17 | |||
54 | 18 | from lazr.testing.yeti import YetiLayer | ||
55 | 19 | |||
56 | 20 | |||
57 | 21 | class YetiLayerErrorTests(MockerTestCase): | ||
58 | 22 | |||
59 | 23 | def setUp(self): | ||
60 | 24 | super(YetiLayerErrorTests, self).setUp() | ||
61 | 25 | env_keys = [ | ||
62 | 26 | "YETI", | ||
63 | 27 | "YETI_SERVER", | ||
64 | 28 | "YETI_PORT", | ||
65 | 29 | "YETI_CAPTURE_TIMEOUT", | ||
66 | 30 | "YETI_BROWSER"] | ||
67 | 31 | |||
68 | 32 | def cleanup_non_existing_key(some_key): | ||
69 | 33 | try: | ||
70 | 34 | del os.environ[some_key] | ||
71 | 35 | except KeyError: | ||
72 | 36 | pass | ||
73 | 37 | |||
74 | 38 | for key in env_keys: | ||
75 | 39 | if key in os.environ: | ||
76 | 40 | self.addCleanup( | ||
77 | 41 | operator.setitem, os.environ, key, os.environ[key]) | ||
78 | 42 | else: | ||
79 | 43 | self.addCleanup(cleanup_non_existing_key, key) | ||
80 | 44 | |||
81 | 45 | def test_binding(self): | ||
82 | 46 | """ | ||
83 | 47 | If a specific port is requested, and a server is already started in the | ||
84 | 48 | requested port, then the layer setup fails. | ||
85 | 49 | """ | ||
86 | 50 | s = socket.socket() | ||
87 | 51 | try: | ||
88 | 52 | s.bind((socket.gethostbyname(""), 4225)) | ||
89 | 53 | except socket.error: | ||
90 | 54 | s.close() | ||
91 | 55 | s = None | ||
92 | 56 | raise | ||
93 | 57 | |||
94 | 58 | if "YETI_SERVER" in os.environ: | ||
95 | 59 | del os.environ["YETI_SERVER"] | ||
96 | 60 | os.environ["YETI_PORT"] = "4225" | ||
97 | 61 | try: | ||
98 | 62 | try: | ||
99 | 63 | YetiLayer.setUp() | ||
100 | 64 | except ValueError, e: | ||
101 | 65 | msg = str(e) | ||
102 | 66 | self.assertIn( | ||
103 | 67 | "Failed to execute Yeti server on port 4225", msg) | ||
104 | 68 | self.assertIn( | ||
105 | 69 | "Address already in use", msg) | ||
106 | 70 | else: | ||
107 | 71 | self.fail("ValueError not raised") | ||
108 | 72 | finally: | ||
109 | 73 | YetiLayer.tearDown() | ||
110 | 74 | s.close() | ||
111 | 75 | |||
112 | 76 | def mock_popen(self): | ||
113 | 77 | """Replace subprocess.Popen and make it return a mock process. | ||
114 | 78 | |||
115 | 79 | The mock process is returned. | ||
116 | 80 | """ | ||
117 | 81 | mock_Popen = self.mocker.replace("subprocess.Popen") | ||
118 | 82 | self.mock_proc = self.mocker.mock() | ||
119 | 83 | mock_Popen(ARGS, KWARGS) | ||
120 | 84 | self.mocker.result(self.mock_proc) | ||
121 | 85 | return self.mock_proc | ||
122 | 86 | |||
123 | 87 | def mock_builtin_open(self): | ||
124 | 88 | """Replace built-in open and make it return a mock file. | ||
125 | 89 | |||
126 | 90 | The mock file is returned. | ||
127 | 91 | """ | ||
128 | 92 | mock_open = self.mocker.replace("__builtin__.open") | ||
129 | 93 | mock_open(ANY) | ||
130 | 94 | self.mock_file = self.mocker.mock() | ||
131 | 95 | self.mocker.result(self.mock_file) | ||
132 | 96 | return self.mock_file | ||
133 | 97 | |||
134 | 98 | def test_wait_for_server_startup(self): | ||
135 | 99 | """ | ||
136 | 100 | Even if we don't wait for the browser to be captured, we wait | ||
137 | 101 | for the server to start up. | ||
138 | 102 | """ | ||
139 | 103 | mock_proc = self.mock_popen() | ||
140 | 104 | mock_file = self.mock_builtin_open() | ||
141 | 105 | |||
142 | 106 | with self.mocker.order(): | ||
143 | 107 | mock_time = self.mocker.replace("time.time") | ||
144 | 108 | # The first time is to initialize the start time. | ||
145 | 109 | mock_time() | ||
146 | 110 | start_time = 0 | ||
147 | 111 | self.mocker.result(start_time) | ||
148 | 112 | |||
149 | 113 | # The second time is to check if the timeout is exceeded in | ||
150 | 114 | # the while loop. | ||
151 | 115 | mock_time() | ||
152 | 116 | self.mocker.result(start_time) | ||
153 | 117 | # Go one iteration of the while loop, reporting the server | ||
154 | 118 | # has started. | ||
155 | 119 | mock_proc.poll() | ||
156 | 120 | self.mocker.result(None) | ||
157 | 121 | mock_file.readline() | ||
158 | 122 | self.mocker.result("to run and report the results") | ||
159 | 123 | |||
160 | 124 | # The opened file is closed. | ||
161 | 125 | mock_file.close() | ||
162 | 126 | self.mocker.result(None) | ||
163 | 127 | |||
164 | 128 | # Last check to make sure the server is running ok. | ||
165 | 129 | mock_proc.poll() | ||
166 | 130 | self.mocker.result(None) | ||
167 | 131 | |||
168 | 132 | self.mocker.replay() | ||
169 | 133 | |||
170 | 134 | os.environ["YETI_BROWSER"] = "" | ||
171 | 135 | if "YETI_SERVER" in os.environ: | ||
172 | 136 | del os.environ["YETI_SERVER"] | ||
173 | 137 | os.environ["YETI_PORT"] = "4225" | ||
174 | 138 | |||
175 | 139 | YetiLayer.setUp() | ||
176 | 140 | self.assertEqual( | ||
177 | 141 | "http://localhost:4225", os.environ["YETI_SERVER"]) | ||
178 | 142 | |||
179 | 143 | def test_server_fail(self): | ||
180 | 144 | """ | ||
181 | 145 | If we a poll of the process returns a non-None value while we | ||
182 | 146 | are waiting, we report that server couldn't be started. | ||
183 | 147 | """ | ||
184 | 148 | mock_proc = self.mock_popen() | ||
185 | 149 | mock_file = self.mock_builtin_open() | ||
186 | 150 | |||
187 | 151 | with self.mocker.order(): | ||
188 | 152 | mock_time = self.mocker.replace("time.time") | ||
189 | 153 | # The first time is to initialize the start time. | ||
190 | 154 | mock_time() | ||
191 | 155 | start_time = 0 | ||
192 | 156 | self.mocker.result(start_time) | ||
193 | 157 | |||
194 | 158 | # The second time is to check if the timeout is exceeded in | ||
195 | 159 | # the while loop. | ||
196 | 160 | mock_time() | ||
197 | 161 | self.mocker.result(start_time) | ||
198 | 162 | # Go one iteration of the while loop, reporting the server | ||
199 | 163 | # is starting up. | ||
200 | 164 | mock_proc.poll() | ||
201 | 165 | self.mocker.result(None) | ||
202 | 166 | mock_file.readline() | ||
203 | 167 | self.mocker.result("not yeti?") | ||
204 | 168 | |||
205 | 169 | # Go another iteration of the while loop, reporting the | ||
206 | 170 | # server failed to start up. | ||
207 | 171 | mock_time() | ||
208 | 172 | self.mocker.result(start_time) | ||
209 | 173 | mock_proc.poll() | ||
210 | 174 | self.mocker.result(1) | ||
211 | 175 | |||
212 | 176 | # The opened file is closed. | ||
213 | 177 | mock_file.close() | ||
214 | 178 | self.mocker.result(None) | ||
215 | 179 | |||
216 | 180 | self.mocker.replay() | ||
217 | 181 | |||
218 | 182 | if "YETI_SERVER" in os.environ: | ||
219 | 183 | del os.environ["YETI_SERVER"] | ||
220 | 184 | os.environ["YETI_PORT"] = "4225" | ||
221 | 185 | |||
222 | 186 | try: | ||
223 | 187 | YetiLayer.setUp() | ||
224 | 188 | except ValueError, e: | ||
225 | 189 | msg = str(e) | ||
226 | 190 | self.assertIn( | ||
227 | 191 | "Failed to execute Yeti server on port 4225", msg) | ||
228 | 192 | else: | ||
229 | 193 | self.fail("ValueError not raised") | ||
230 | 194 | |||
231 | 195 | def test_server_timeout(self): | ||
232 | 196 | """ | ||
233 | 197 | If we don't see that the server is started before the timeout, a | ||
234 | 198 | ValueError is raised, even if the process is still running. | ||
235 | 199 | """ | ||
236 | 200 | timeout = 1 | ||
237 | 201 | os.environ["YETI_CAPTURE_TIMEOUT"] = "%s" % timeout | ||
238 | 202 | os.environ["YETI_BROWSER"] = "" | ||
239 | 203 | if "YETI_SERVER" in os.environ: | ||
240 | 204 | del os.environ["YETI_SERVER"] | ||
241 | 205 | os.environ["YETI_PORT"] = "4225" | ||
242 | 206 | |||
243 | 207 | mock_proc = self.mock_popen() | ||
244 | 208 | mock_file = self.mock_builtin_open() | ||
245 | 209 | |||
246 | 210 | with self.mocker.order(): | ||
247 | 211 | mock_time = self.mocker.replace("time.time") | ||
248 | 212 | # The first time is to initialize the start time. | ||
249 | 213 | mock_time() | ||
250 | 214 | start_time = 0 | ||
251 | 215 | self.mocker.result(start_time) | ||
252 | 216 | |||
253 | 217 | # The second time is to check if the timeout is exceeded in | ||
254 | 218 | # the while loop. | ||
255 | 219 | mock_time() | ||
256 | 220 | self.mocker.result(start_time) | ||
257 | 221 | # Go one iteration of the while loop, reporting the server | ||
258 | 222 | # is starting up. | ||
259 | 223 | mock_proc.poll() | ||
260 | 224 | self.mocker.result(None) | ||
261 | 225 | mock_file.readline() | ||
262 | 226 | self.mocker.result("not yeti?") | ||
263 | 227 | |||
264 | 228 | # Trigger a timeout. | ||
265 | 229 | mock_time() | ||
266 | 230 | self.mocker.result(start_time + timeout + 1) | ||
267 | 231 | |||
268 | 232 | # The opened file is closed. | ||
269 | 233 | mock_file.close() | ||
270 | 234 | self.mocker.result(None) | ||
271 | 235 | |||
272 | 236 | # Last check whether the server is still running. | ||
273 | 237 | mock_proc.poll() | ||
274 | 238 | self.mocker.result(None) | ||
275 | 239 | |||
276 | 240 | # Since the server is running, it gets terminated. | ||
277 | 241 | mock_proc.terminate() | ||
278 | 242 | self.mocker.result(None) | ||
279 | 243 | mock_proc.wait() | ||
280 | 244 | self.mocker.result(None) | ||
281 | 245 | |||
282 | 246 | self.mocker.replay() | ||
283 | 247 | |||
284 | 248 | try: | ||
285 | 249 | YetiLayer.setUp() | ||
286 | 250 | except ValueError, e: | ||
287 | 251 | msg = str(e) | ||
288 | 252 | self.assertIn( | ||
289 | 253 | "Failed to execute Yeti server in 1 seconds" | ||
290 | 254 | " on port 4225", msg) | ||
291 | 255 | else: | ||
292 | 256 | self.fail("ValueError not raised") | ||
293 | 257 | |||
294 | 258 | def tearDown(self): | ||
295 | 259 | super(YetiLayerErrorTests, self).tearDown() | ||
296 | 260 | self.mocker.restore() | ||
297 | 261 | |||
298 | 262 | |||
299 | 263 | def test_suite(): | ||
300 | 264 | suite = unittest.TestSuite() | ||
301 | 265 | |||
302 | 266 | if not "YETI" in os.environ: | ||
303 | 267 | warnings.warn("Environment variable 'YETI' not set. " | ||
304 | 268 | "Skipping 'YETI' tests.") | ||
305 | 269 | return suite | ||
306 | 270 | |||
307 | 271 | suite.addTests(unittest.makeSuite(YetiLayerErrorTests)) | ||
308 | 272 | return suite | ||
309 | 0 | 273 | ||
310 | === added file 'src/lazr/testing/yeti.py' | |||
311 | --- src/lazr/testing/yeti.py 1970-01-01 00:00:00 +0000 | |||
312 | +++ src/lazr/testing/yeti.py 2010-12-23 16:11:33 +0000 | |||
313 | @@ -0,0 +1,149 @@ | |||
314 | 1 | import os | ||
315 | 2 | import time | ||
316 | 3 | import fnmatch | ||
317 | 4 | import signal | ||
318 | 5 | import tempfile | ||
319 | 6 | import subprocess | ||
320 | 7 | |||
321 | 8 | from unittest import TestCase | ||
322 | 9 | from subunit import ProtocolTestCase | ||
323 | 10 | |||
324 | 11 | |||
325 | 12 | def startYeti(): | ||
326 | 13 | yeti = os.environ["YETI"] | ||
327 | 14 | port = os.environ.get("YETI_PORT", "4422") | ||
328 | 15 | |||
329 | 16 | capture_timeout = int(os.environ.get( | ||
330 | 17 | "YETI_CAPTURE_TIMEOUT", "30")) | ||
331 | 18 | |||
332 | 19 | cmd = yeti.split() + ["--port", port, "--server"] | ||
333 | 20 | |||
334 | 21 | browser = os.environ.get("YETI_BROWSER", "default") | ||
335 | 22 | |||
336 | 23 | if browser: | ||
337 | 24 | cmd.extend(["--browsers", browser]) | ||
338 | 25 | |||
339 | 26 | # Redirect stderr through a temporary file, so that it doesn't | ||
340 | 27 | # block and we don't get an IOError on readline(), apparently | ||
341 | 28 | # caused by an unhandled SIGINT (Google for it. :) | ||
342 | 29 | fd, name = tempfile.mkstemp() | ||
343 | 30 | stderr = open(name) | ||
344 | 31 | server_started = False | ||
345 | 32 | rc = None | ||
346 | 33 | |||
347 | 34 | try: | ||
348 | 35 | proc = subprocess.Popen(cmd, | ||
349 | 36 | shell=False, | ||
350 | 37 | stdin=subprocess.PIPE, | ||
351 | 38 | stdout=subprocess.PIPE, | ||
352 | 39 | stderr=fd, | ||
353 | 40 | close_fds=True) | ||
354 | 41 | |||
355 | 42 | # Give the server process a few seconds to start, and | ||
356 | 43 | # capture the browser if needed. | ||
357 | 44 | output = [] | ||
358 | 45 | start = time.time() | ||
359 | 46 | while time.time() - start < capture_timeout: | ||
360 | 47 | rc = proc.poll() | ||
361 | 48 | if rc is not None: | ||
362 | 49 | break | ||
363 | 50 | line = stderr.readline() | ||
364 | 51 | if not line: | ||
365 | 52 | continue | ||
366 | 53 | output.append(line) | ||
367 | 54 | if line.startswith("Running tests locally with:"): | ||
368 | 55 | server_started = True | ||
369 | 56 | break | ||
370 | 57 | if line.startswith("to run and report the results"): | ||
371 | 58 | server_started = True | ||
372 | 59 | break | ||
373 | 60 | finally: | ||
374 | 61 | stderr.close() | ||
375 | 62 | |||
376 | 63 | if rc is None: | ||
377 | 64 | rc = proc.poll() | ||
378 | 65 | if rc is not None: | ||
379 | 66 | raise ValueError( | ||
380 | 67 | "Failed to execute Yeti server on port %s:" | ||
381 | 68 | "\nError: (%s) %s" % | ||
382 | 69 | (port, rc, "\n".join(output))) | ||
383 | 70 | if not server_started: | ||
384 | 71 | terminateProcess(proc) | ||
385 | 72 | raise ValueError( | ||
386 | 73 | "Failed to execute Yeti server in %d seconds on port %s:" | ||
387 | 74 | "\nError: (%s) %s" % | ||
388 | 75 | (capture_timeout, port, rc, "\n".join(output))) | ||
389 | 76 | else: | ||
390 | 77 | os.environ["YETI_SERVER"] = ( | ||
391 | 78 | "http://localhost:%s" % port) | ||
392 | 79 | return proc | ||
393 | 80 | |||
394 | 81 | |||
395 | 82 | def terminateProcess(proc): | ||
396 | 83 | try: | ||
397 | 84 | proc.terminate() | ||
398 | 85 | except AttributeError: | ||
399 | 86 | os.kill(proc.pid, signal.SIGTERM) | ||
400 | 87 | proc.wait() | ||
401 | 88 | |||
402 | 89 | |||
403 | 90 | class YetiLayer(object): | ||
404 | 91 | """Manages startup/shutdown of a I{Yeti} server. | ||
405 | 92 | """ | ||
406 | 93 | |||
407 | 94 | @classmethod | ||
408 | 95 | def setUp(cls): | ||
409 | 96 | cls.proc = None | ||
410 | 97 | if os.environ.get("YETI_SERVER") is None: | ||
411 | 98 | cls.proc = startYeti() | ||
412 | 99 | |||
413 | 100 | @classmethod | ||
414 | 101 | def tearDown(cls): | ||
415 | 102 | if cls.proc is not None: | ||
416 | 103 | # If the process was created by us, then that means the | ||
417 | 104 | # environment variable has been set by ourselves too, so | ||
418 | 105 | # we must unset it. | ||
419 | 106 | del os.environ["YETI_SERVER"] | ||
420 | 107 | terminateProcess(cls.proc) | ||
421 | 108 | |||
422 | 109 | |||
423 | 110 | class YetiTestCase(TestCase): | ||
424 | 111 | """Controls a I{Yeti} client for a specific configuration. | ||
425 | 112 | |||
426 | 113 | Test output from I{Yeti} is captured and then parsed and | ||
427 | 114 | reported to unittest through clever subunit usage. | ||
428 | 115 | |||
429 | 116 | We require a L{tests_directory} class variable to be set by | ||
430 | 117 | subclasses, and that's the only configuration needed. | ||
431 | 118 | """ | ||
432 | 119 | layer = YetiLayer | ||
433 | 120 | |||
434 | 121 | def _runTest(self, result): | ||
435 | 122 | yeti = os.environ["YETI"] | ||
436 | 123 | port = os.environ.get("YETI_PORT", "4422") | ||
437 | 124 | cmd = yeti.split() + ["--formatter=subunit", | ||
438 | 125 | "--quiet", | ||
439 | 126 | "--solo=1", | ||
440 | 127 | "--port=%s" % port] | ||
441 | 128 | for base, dirs, files in os.walk(self.tests_directory): | ||
442 | 129 | for filename in fnmatch.filter(files, "test_*.html"): | ||
443 | 130 | cmd.append(os.path.join(base, filename)) | ||
444 | 131 | proc = subprocess.Popen(cmd, | ||
445 | 132 | stdin=subprocess.PIPE, | ||
446 | 133 | stdout=subprocess.PIPE, | ||
447 | 134 | stderr=subprocess.STDOUT) | ||
448 | 135 | suite = ProtocolTestCase(proc.stdout) | ||
449 | 136 | suite.run(result) | ||
450 | 137 | proc.wait() | ||
451 | 138 | |||
452 | 139 | def run(self, result=None): | ||
453 | 140 | if result is None: | ||
454 | 141 | result = self.defaultTestResult() | ||
455 | 142 | self.setUp() | ||
456 | 143 | try: | ||
457 | 144 | self._runTest(result) | ||
458 | 145 | finally: | ||
459 | 146 | self.tearDown() | ||
460 | 147 | |||
461 | 148 | def runTest(self): | ||
462 | 149 | self.run() |
sidnei, once again, I bow to your prowess.
Land this kthxbai! :)