Merge lp:~jml/launchpad/list-ec2-test-runs-721784 into lp:launchpad
- list-ec2-test-runs-721784
- Merge into devel
Status: | Merged | ||||
---|---|---|---|---|---|
Approved by: | Leonard Richardson | ||||
Approved revision: | no longer in the source branch. | ||||
Merged at revision: | 12453 | ||||
Proposed branch: | lp:~jml/launchpad/list-ec2-test-runs-721784 | ||||
Merge into: | lp:launchpad | ||||
Diff against target: |
526 lines (+238/-77) 3 files modified
lib/devscripts/ec2test/builtins.py (+111/-1) lib/devscripts/ec2test/remote.py (+45/-35) lib/devscripts/ec2test/tests/test_remote.py (+82/-41) |
||||
To merge this branch: | bzr merge lp:~jml/launchpad/list-ec2-test-runs-721784 | ||||
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Leonard Richardson (community) | Approve | ||
Review via email: mp+50537@code.launchpad.net |
Commit message
[no-qa] [r=leonardr][bug=721784] Add ec2 list command to list running ec2 instances and test status
Description of the change
This branch adds a new command 'ec2 list', which lists your running ec2 instances, tells you if they are currently passing, and how long they've been running for.
The implementation approach is to have the remote instance publish a JSON file, and have 'ec2 list' scan all instances for the JSON file. Still have some details to work out re exact output.
Jonathan Lange (jml) wrote : | # |
Leonard Richardson (leonardr) wrote : | # |
This is a good start. I'd like to see some tests of cmd_list, especially format_summary and format_instance, but that can wait for a follow-up.
I don't think you need to call .replace(
I think using 'success: True' to describe a run that's been successful *so far* might give people the wrong impression? Is there an easy way to distinguish between the case when you list a run that's shutting down after a complete success and a run that just hasn't failed yet?
Jonathan Lange (jml) wrote : | # |
Thanks for the review.
You do have to replace the tzinfo from utcnow. It returns a naive timestamp.
I can say 'failed-yet' in the metadata, still format the output as [OK], and make this clear in the documentation for the command.
Preview Diff
1 | === modified file 'lib/devscripts/ec2test/builtins.py' |
2 | --- lib/devscripts/ec2test/builtins.py 2010-11-29 21:32:06 +0000 |
3 | +++ lib/devscripts/ec2test/builtins.py 2011-02-23 11:43:52 +0000 |
4 | @@ -6,6 +6,7 @@ |
5 | __metaclass__ = type |
6 | __all__ = [] |
7 | |
8 | +from datetime import datetime |
9 | import os |
10 | import pdb |
11 | import socket |
12 | @@ -13,12 +14,20 @@ |
13 | |
14 | from bzrlib.bzrdir import BzrDir |
15 | from bzrlib.commands import Command |
16 | -from bzrlib.errors import BzrCommandError |
17 | +from bzrlib.errors import ( |
18 | + BzrCommandError, |
19 | + ConnectionError, |
20 | + NoSuchFile, |
21 | + ) |
22 | from bzrlib.help import help_commands |
23 | from bzrlib.option import ( |
24 | ListOption, |
25 | Option, |
26 | ) |
27 | +from bzrlib.transport import get_transport |
28 | +from pytz import UTC |
29 | +import simplejson |
30 | + |
31 | from devscripts import get_launchpad_root |
32 | from devscripts.ec2test.account import VALID_AMI_OWNERS |
33 | from devscripts.ec2test.credentials import EC2Credentials |
34 | @@ -652,6 +661,107 @@ |
35 | VALID_AMI_OWNERS.get(image.ownerId, "unknown"))) |
36 | |
37 | |
38 | +class cmd_list(EC2Command): |
39 | + """List all your current EC2 test runs. |
40 | + |
41 | + If an instance is publishing an 'info.json' file with 'description' and |
42 | + 'failed-yet' fields, this command will list that instance, whether it has |
43 | + failed the test run and how long it has been up for. |
44 | + |
45 | + [FAILED] means that the has been a failing test. [OK] means that the test |
46 | + run has had no failures yet, it's not a guarantee of a successful run. |
47 | + """ |
48 | + |
49 | + aliases = ["ls"] |
50 | + |
51 | + takes_options = [ |
52 | + Option('show-urls', |
53 | + help="Include more information about each instance"), |
54 | + Option('all', short_name='a', |
55 | + help="Show all instances, not just ones with ec2test data."), |
56 | + ] |
57 | + |
58 | + def iter_instances(self, account): |
59 | + """Iterate through all instances in 'account'.""" |
60 | + for reservation in account.conn.get_all_instances(): |
61 | + for instance in reservation.instances: |
62 | + yield instance |
63 | + |
64 | + def get_uptime(self, instance): |
65 | + """How long has 'instance' been running?""" |
66 | + expected_format = '%Y-%m-%dT%H:%M:%S.000Z' |
67 | + launch_time = datetime.strptime(instance.launch_time, expected_format) |
68 | + return ( |
69 | + datetime.utcnow().replace(tzinfo=UTC) |
70 | + - launch_time.replace(tzinfo=UTC)) |
71 | + |
72 | + def get_http_url(self, instance): |
73 | + hostname = instance.public_dns_name |
74 | + if not hostname: |
75 | + return |
76 | + return 'http://%s/' % (hostname,) |
77 | + |
78 | + def get_ec2test_info(self, instance): |
79 | + """Load the ec2test-specific information published by 'instance'.""" |
80 | + url = self.get_http_url(instance) |
81 | + if url is None: |
82 | + return |
83 | + try: |
84 | + json = get_transport(url).get_bytes('info.json') |
85 | + except (ConnectionError, NoSuchFile): |
86 | + # Probably not an ec2test instance, or not ready yet. |
87 | + return None |
88 | + return simplejson.loads(json) |
89 | + |
90 | + def format_instance(self, instance, data, verbose): |
91 | + """Format 'instance' for display. |
92 | + |
93 | + :param instance: The EC2 instance to display. |
94 | + :param data: Launchpad-specific data. |
95 | + :param verbose: Whether we want verbose output. |
96 | + """ |
97 | + uptime = self.get_uptime(instance) |
98 | + if data is None: |
99 | + description = instance.id |
100 | + current_status = 'unknown ' |
101 | + else: |
102 | + description = data['description'] |
103 | + if data['failed-yet']: |
104 | + current_status = '[FAILED]' |
105 | + else: |
106 | + current_status = '[OK] ' |
107 | + output = '%s %s (up for %s)' % (description, current_status, uptime) |
108 | + if verbose: |
109 | + url = self.get_http_url(instance) |
110 | + if url is None: |
111 | + url = "No web service" |
112 | + output += '\n %s' % (url,) |
113 | + return output |
114 | + |
115 | + def format_summary(self, by_state): |
116 | + return ', '.join( |
117 | + ': '.join((state, str(num))) |
118 | + for (state, num) in sorted(list(by_state.items()))) |
119 | + |
120 | + def run(self, show_urls=False, all=False): |
121 | + credentials = EC2Credentials.load_from_file() |
122 | + session_name = EC2SessionName.make(EC2TestRunner.name) |
123 | + account = credentials.connect(session_name) |
124 | + instances = list(self.iter_instances(account)) |
125 | + if len(instances) == 0: |
126 | + print "No instances running." |
127 | + return |
128 | + |
129 | + by_state = {} |
130 | + for instance in instances: |
131 | + by_state[instance.state] = by_state.get(instance.state, 0) + 1 |
132 | + data = self.get_ec2test_info(instance) |
133 | + if data is None and not all: |
134 | + continue |
135 | + print self.format_instance(instance, data, show_urls) |
136 | + print 'Summary: %s' % (self.format_summary(by_state),) |
137 | + |
138 | + |
139 | class cmd_help(EC2Command): |
140 | """Show general help or help for a command.""" |
141 | |
142 | |
143 | === modified file 'lib/devscripts/ec2test/remote.py' |
144 | --- lib/devscripts/ec2test/remote.py 2010-10-26 15:47:24 +0000 |
145 | +++ lib/devscripts/ec2test/remote.py 2011-02-23 11:43:52 +0000 |
146 | @@ -10,7 +10,7 @@ |
147 | "test merging foo-bar-bug-12345 into db-devel"). |
148 | |
149 | * `LaunchpadTester` knows how to actually run the tests and gather the |
150 | - results. It uses `SummaryResult` and `FlagFallStream` to do so. |
151 | + results. It uses `SummaryResult` to do so. |
152 | |
153 | * `WebTestLogger` knows how to display the results to the user, and is given |
154 | the responsibility of handling the results that `LaunchpadTester` gathers. |
155 | @@ -34,9 +34,10 @@ |
156 | import time |
157 | import traceback |
158 | import unittest |
159 | - |
160 | from xml.sax.saxutils import escape |
161 | |
162 | +import simplejson |
163 | + |
164 | import bzrlib.branch |
165 | import bzrlib.config |
166 | import bzrlib.errors |
167 | @@ -47,6 +48,8 @@ |
168 | |
169 | import subunit |
170 | |
171 | +from testtools import MultiTestResult |
172 | + |
173 | |
174 | class NonZeroExitCode(Exception): |
175 | """Raised when the child process exits with a non-zero exit code.""" |
176 | @@ -94,34 +97,19 @@ |
177 | self.stream.flush() |
178 | |
179 | |
180 | -class FlagFallStream: |
181 | - """Wrapper around a stream that only starts forwarding after a flagfall. |
182 | - """ |
183 | - |
184 | - def __init__(self, stream, flag): |
185 | - """Construct a `FlagFallStream` that wraps 'stream'. |
186 | - |
187 | - :param stream: A stream, a file-like object. |
188 | - :param flag: A string that needs to be written to this stream before |
189 | - we start forwarding the output. |
190 | - """ |
191 | - self._stream = stream |
192 | - self._flag = flag |
193 | - self._flag_fallen = False |
194 | - |
195 | - def write(self, bytes): |
196 | - if self._flag_fallen: |
197 | - self._stream.write(bytes) |
198 | - else: |
199 | - index = bytes.find(self._flag) |
200 | - if index == -1: |
201 | - return |
202 | - else: |
203 | - self._stream.write(bytes[index:]) |
204 | - self._flag_fallen = True |
205 | - |
206 | - def flush(self): |
207 | - self._stream.flush() |
208 | +class FailureUpdateResult(unittest.TestResult): |
209 | + |
210 | + def __init__(self, logger): |
211 | + super(FailureUpdateResult, self).__init__() |
212 | + self._logger = logger |
213 | + |
214 | + def addError(self, *args, **kwargs): |
215 | + super(FailureUpdateResult, self).addError(*args, **kwargs) |
216 | + self._logger.got_failure() |
217 | + |
218 | + def addFailure(self, *args, **kwargs): |
219 | + super(FailureUpdateResult, self).addFailure(*args, **kwargs) |
220 | + self._logger.got_failure() |
221 | |
222 | |
223 | class EC2Runner: |
224 | @@ -293,7 +281,9 @@ |
225 | def _gather_test_output(self, input_stream, logger): |
226 | """Write the testrunner output to the logs.""" |
227 | summary_stream = logger.get_summary_stream() |
228 | - result = SummaryResult(summary_stream) |
229 | + result = MultiTestResult( |
230 | + SummaryResult(summary_stream), |
231 | + FailureUpdateResult(logger)) |
232 | subunit_server = subunit.TestProtocolServer(result, summary_stream) |
233 | for line in input_stream: |
234 | subunit_server.lineReceived(line) |
235 | @@ -302,6 +292,8 @@ |
236 | return result |
237 | |
238 | |
239 | +# XXX: Publish a JSON file that includes the relevant details from this |
240 | +# request. |
241 | class Request: |
242 | """A request to have a branch tested and maybe landed.""" |
243 | |
244 | @@ -433,9 +425,6 @@ |
245 | we're just running tests for a trunk branch without merging return |
246 | '$TRUNK_NICK'. |
247 | """ |
248 | - # XXX: JonathanLange 2010-08-17: Not actually used yet. I think it |
249 | - # would be a great thing to have in the subject of the emails we |
250 | - # receive. |
251 | source = self.get_source_details() |
252 | if not source: |
253 | return '%s r%s' % (self.get_nick(), self.get_revno()) |
254 | @@ -532,7 +521,11 @@ |
255 | |
256 | |
257 | class WebTestLogger: |
258 | - """Logs test output to disk and a simple web page.""" |
259 | + """Logs test output to disk and a simple web page. |
260 | + |
261 | + :ivar successful: Whether the logger has received only successful input up |
262 | + until now. |
263 | + """ |
264 | |
265 | def __init__(self, full_log_filename, summary_filename, index_filename, |
266 | request, echo_to_stdout): |
267 | @@ -556,11 +549,14 @@ |
268 | self._full_log_filename = full_log_filename |
269 | self._summary_filename = summary_filename |
270 | self._index_filename = index_filename |
271 | + self._info_json = os.path.join( |
272 | + os.path.dirname(index_filename), 'info.json') |
273 | self._request = request |
274 | self._echo_to_stdout = echo_to_stdout |
275 | # Actually set by prepare(), but setting to a dummy value to make |
276 | # testing easier. |
277 | self._start_time = datetime.datetime.utcnow() |
278 | + self.successful = True |
279 | |
280 | @classmethod |
281 | def make_in_directory(cls, www_dir, request, echo_to_stdout): |
282 | @@ -628,6 +624,11 @@ |
283 | if e.errno == errno.ENOENT: |
284 | return '' |
285 | |
286 | + def got_failure(self): |
287 | + """Called when we receive word that a test has failed.""" |
288 | + self.successful = False |
289 | + self._dump_json() |
290 | + |
291 | def got_result(self, result): |
292 | """The tests are done and the results are known.""" |
293 | self._end_time = datetime.datetime.utcnow() |
294 | @@ -666,12 +667,21 @@ |
295 | """Write to the summary and full log file with a newline.""" |
296 | self._write(msg + '\n') |
297 | |
298 | + def _dump_json(self): |
299 | + fd = open(self._info_json, 'w') |
300 | + simplejson.dump( |
301 | + {'description': self._request.get_merge_description(), |
302 | + 'failed-yet': not self.successful, |
303 | + }, fd) |
304 | + fd.close() |
305 | + |
306 | def prepare(self): |
307 | """Prepares the log files on disk. |
308 | |
309 | Writes three log files: the raw output log, the filtered "summary" |
310 | log file, and a HTML index page summarizing the test run paramters. |
311 | """ |
312 | + self._dump_json() |
313 | # XXX: JonathanLange 2010-07-18: Mostly untested. |
314 | log = self.write_line |
315 | |
316 | |
317 | === modified file 'lib/devscripts/ec2test/tests/test_remote.py' |
318 | --- lib/devscripts/ec2test/tests/test_remote.py 2010-10-06 11:46:51 +0000 |
319 | +++ lib/devscripts/ec2test/tests/test_remote.py 2011-02-23 11:43:52 +0000 |
320 | @@ -20,6 +20,8 @@ |
321 | import traceback |
322 | import unittest |
323 | |
324 | +import simplejson |
325 | + |
326 | from bzrlib.config import GlobalConfig |
327 | from bzrlib.tests import TestCaseWithTransport |
328 | |
329 | @@ -30,7 +32,7 @@ |
330 | |
331 | from devscripts.ec2test.remote import ( |
332 | EC2Runner, |
333 | - FlagFallStream, |
334 | + FailureUpdateResult, |
335 | gunzip_data, |
336 | gzip_data, |
337 | LaunchpadTester, |
338 | @@ -122,39 +124,6 @@ |
339 | 'full.log', 'summary.log', 'index.html', request, echo_to_stdout) |
340 | |
341 | |
342 | -class TestFlagFallStream(TestCase): |
343 | - """Tests for `FlagFallStream`.""" |
344 | - |
345 | - def test_doesnt_write_before_flag(self): |
346 | - # A FlagFallStream does not forward any writes before it sees the |
347 | - # 'flag'. |
348 | - stream = StringIO() |
349 | - flag = self.getUniqueString('flag') |
350 | - flagfall = FlagFallStream(stream, flag) |
351 | - flagfall.write('foo') |
352 | - flagfall.flush() |
353 | - self.assertEqual('', stream.getvalue()) |
354 | - |
355 | - def test_writes_after_flag(self): |
356 | - # After a FlagFallStream sees the flag, it forwards all writes. |
357 | - stream = StringIO() |
358 | - flag = self.getUniqueString('flag') |
359 | - flagfall = FlagFallStream(stream, flag) |
360 | - flagfall.write('foo') |
361 | - flagfall.write(flag) |
362 | - flagfall.write('bar') |
363 | - self.assertEqual('%sbar' % (flag,), stream.getvalue()) |
364 | - |
365 | - def test_mixed_write(self): |
366 | - # If a single call to write has pre-flagfall and post-flagfall data in |
367 | - # it, then only the post-flagfall data is forwarded to the stream. |
368 | - stream = StringIO() |
369 | - flag = self.getUniqueString('flag') |
370 | - flagfall = FlagFallStream(stream, flag) |
371 | - flagfall.write('foo%sbar' % (flag,)) |
372 | - self.assertEqual('%sbar' % (flag,), stream.getvalue()) |
373 | - |
374 | - |
375 | class TestSummaryResult(TestCase): |
376 | """Tests for `SummaryResult`.""" |
377 | |
378 | @@ -207,6 +176,29 @@ |
379 | self.assertEqual(1, len(flush_calls)) |
380 | |
381 | |
382 | +class TestFailureUpdateResult(TestCaseWithTransport, RequestHelpers): |
383 | + |
384 | + def makeException(self, factory=None, *args, **kwargs): |
385 | + if factory is None: |
386 | + factory = RuntimeError |
387 | + try: |
388 | + raise factory(*args, **kwargs) |
389 | + except: |
390 | + return sys.exc_info() |
391 | + |
392 | + def test_addError_is_unsuccessful(self): |
393 | + logger = self.make_logger() |
394 | + result = FailureUpdateResult(logger) |
395 | + result.addError(self, self.makeException()) |
396 | + self.assertEqual(False, logger.successful) |
397 | + |
398 | + def test_addFailure_is_unsuccessful(self): |
399 | + logger = self.make_logger() |
400 | + result = FailureUpdateResult(logger) |
401 | + result.addFailure(self, self.makeException(AssertionError)) |
402 | + self.assertEqual(False, logger.successful) |
403 | + |
404 | + |
405 | class FakePopen: |
406 | """Fake Popen object so we don't have to spawn processes in tests.""" |
407 | |
408 | @@ -267,6 +259,16 @@ |
409 | # Message being sent implies got_result thought it got a success. |
410 | self.assertEqual([message], log) |
411 | |
412 | + def test_failing_test(self): |
413 | + # If LaunchpadTester gets a failing test, then it records that on the |
414 | + # logger. |
415 | + logger = self.make_logger() |
416 | + tester = self.make_tester(logger=logger) |
417 | + output = "test: foo\nerror: foo\n" |
418 | + tester._spawn_test_process = lambda: FakePopen(output, 0) |
419 | + tester.test() |
420 | + self.assertEqual(False, logger.successful) |
421 | + |
422 | def test_error_in_testrunner(self): |
423 | # Any exception is raised within LaunchpadTester.test() is an error in |
424 | # the testrunner. When we detect these, we do three things: |
425 | @@ -526,8 +528,8 @@ |
426 | [body, attachment] = email.get_payload() |
427 | self.assertIsInstance(body, MIMEText) |
428 | self.assertEqual('inline', body['Content-Disposition']) |
429 | - self.assertEqual('text/plain; charset="utf-8"', body['Content-Type']) |
430 | - self.assertEqual("foo", body.get_payload()) |
431 | + self.assertEqual('text/plain; charset="utf8"', body['Content-Type']) |
432 | + self.assertEqual("foo", body.get_payload(decode=True)) |
433 | |
434 | def test_report_email_attachment(self): |
435 | req = self.make_request(emails=['foo@example.com']) |
436 | @@ -540,7 +542,7 @@ |
437 | req.get_nick(), req.get_revno()), |
438 | attachment['Content-Disposition']) |
439 | self.assertEqual( |
440 | - "gobbledygook", attachment.get_payload().decode('base64')) |
441 | + "gobbledygook", attachment.get_payload(decode=True)) |
442 | |
443 | def test_send_report_email_sends_email(self): |
444 | log = [] |
445 | @@ -707,6 +709,43 @@ |
446 | logger = self.make_logger() |
447 | self.assertEqual('', logger.get_summary_contents()) |
448 | |
449 | + def test_initial_json(self): |
450 | + self.build_tree(['www/']) |
451 | + request = self.make_request() |
452 | + logger = WebTestLogger.make_in_directory('www', request, False) |
453 | + logger.prepare() |
454 | + self.assertEqual( |
455 | + {'description': request.get_merge_description(), |
456 | + 'successful': True, |
457 | + }, |
458 | + simplejson.loads(open('www/info.json').read())) |
459 | + |
460 | + def test_initial_success(self): |
461 | + # The Logger initially thinks it is successful because there have been |
462 | + # no failures yet. |
463 | + logger = self.make_logger() |
464 | + self.assertEqual(True, logger.successful) |
465 | + |
466 | + def test_got_failure_changes_success(self): |
467 | + # Logger.got_failure() tells the logger it is no longer successful. |
468 | + logger = self.make_logger() |
469 | + logger.got_failure() |
470 | + self.assertEqual(False, logger.successful) |
471 | + |
472 | + def test_got_failure_updates_json(self): |
473 | + # Logger.got_failure() updates JSON so that interested parties can |
474 | + # determine that it is unsuccessful. |
475 | + self.build_tree(['www/']) |
476 | + request = self.make_request() |
477 | + logger = WebTestLogger.make_in_directory('www', request, False) |
478 | + logger.prepare() |
479 | + logger.got_failure() |
480 | + self.assertEqual( |
481 | + {'description': request.get_merge_description(), |
482 | + 'successful': False, |
483 | + }, |
484 | + simplejson.loads(open('www/info.json').read())) |
485 | + |
486 | def test_got_line_no_echo(self): |
487 | # got_line forwards the line to the full log, but does not forward to |
488 | # stdout if echo_to_stdout is False. |
489 | @@ -769,7 +808,7 @@ |
490 | [body, attachment] = email.get_payload() |
491 | self.assertIsInstance(body, MIMEText) |
492 | self.assertEqual('inline', body['Content-Disposition']) |
493 | - self.assertEqual('text/plain; charset="utf-8"', body['Content-Type']) |
494 | + self.assertEqual('text/plain; charset="utf8"', body['Content-Type']) |
495 | self.assertEqual( |
496 | logger.get_summary_contents(), body.get_payload(decode=True)) |
497 | self.assertIsInstance(attachment, MIMEApplication) |
498 | @@ -838,7 +877,9 @@ |
499 | self.assertNotEqual([], email_log) |
500 | [tester_msg] = email_log |
501 | self.assertEqual('foo@example.com', tester_msg['To']) |
502 | - self.assertIn('ZeroDivisionError', str(tester_msg)) |
503 | + self.assertIn( |
504 | + 'ZeroDivisionError', |
505 | + tester_msg.get_payload()[0].get_payload(decode=True)) |
506 | |
507 | |
508 | class TestDaemonizationInteraction(TestCaseWithTransport, RequestHelpers): |
509 | @@ -888,7 +929,7 @@ |
510 | """Tests for how we handle the result at the end of the test suite.""" |
511 | |
512 | def get_body_text(self, email): |
513 | - return email.get_payload()[0].get_payload() |
514 | + return email.get_payload()[0].get_payload(decode=True) |
515 | |
516 | def make_empty_result(self): |
517 | return TestResult() |
518 | @@ -989,7 +1030,7 @@ |
519 | self.assertEqual( |
520 | request.format_result( |
521 | result, logger._start_time, logger._end_time), |
522 | - self.get_body_text(user_message).decode('quoted-printable')) |
523 | + self.get_body_text(user_message)) |
524 | |
525 | def test_gzip_of_full_log_attached(self): |
526 | # The full log is attached to the email. |
Got something worth looking out now. It's not perfect, but I personally think it's good enough to land.