1
=== modified file 'autopilot/__init__.py'
2
--- autopilot/__init__.py	2014-05-20 14:43:24 +0000
3
+++ autopilot/__init__.py	2014-07-22 03:40:04 +0000
4
@@ -1,7 +1,7 @@
5
1
# -*- Mode: Python; coding: utf-8; indent-tabs-mode: nil; tab-width: 4 -*-
1
# -*- Mode: Python; coding: utf-8; indent-tabs-mode: nil; tab-width: 4 -*-
6
2
#
2
#
7
3
# Autopilot Functional Test Tool
3
# Autopilot Functional Test Tool
9
4
# Copyright (C) 2012-2013 Canonical
4
# Copyright (C) 2012-2014 Canonical
10
5
#
5
#
11
6
# This program is free software: you can redistribute it and/or modify
6
# This program is free software: you can redistribute it and/or modify
12
7
# it under the terms of the GNU General Public License as published by
7
# it under the terms of the GNU General Public License as published by
13
8
8
14
=== modified file 'autopilot/run.py'
15
--- autopilot/run.py	2014-07-02 21:07:33 +0000
16
+++ autopilot/run.py	2014-07-22 03:40:04 +0000
17
@@ -31,15 +31,14 @@
18
31
import os.path
31
import os.path
19
32
from platform import node
32
from platform import node
20
33
from random import shuffle
33
from random import shuffle
21
34
import six
22
35
import subprocess
34
import subprocess
23
36
import sys
35
import sys
24
37
from unittest import TestLoader, TestSuite
36
from unittest import TestLoader, TestSuite
25
38
37
26
38
import six
27
39
from testtools import iterate_tests
39
from testtools import iterate_tests
28
40
40
29
41
from autopilot import get_version_string, have_vis
41
from autopilot import get_version_string, have_vis
30
42
import autopilot.globals
31
43
from autopilot import _config as test_config
42
from autopilot import _config as test_config
32
44
from autopilot._debug import (
43
from autopilot._debug import (
33
45
    get_all_debug_profiles,
44
    get_all_debug_profiles,
34
@@ -52,6 +51,7 @@
35
52
    get_application_launcher_wrapper,
51
    get_application_launcher_wrapper,
36
53
    launch_process,
52
    launch_process,
37
54
)
53
)
38
54
import autopilot.globals
39
55
55
40
56
56
41
57
def _get_parser():
57
def _get_parser():
42
@@ -215,15 +215,18 @@
43
215
        setattr(namespace, self.dest, values)
215
        setattr(namespace, self.dest, values)
44
216
216
45
217
217
48
218
def setup_logging(verbose):
218
def setup_logging(verbosity, path=None, format=None):
49
219
    """Configure the root logger and verbose logging to stderr."""
219
    """Configure the root logger"""
50
220
    root_logger = get_root_logger()
220
    root_logger = get_root_logger()
51
221
    root_logger.setLevel(logging.DEBUG)
221
    root_logger.setLevel(logging.DEBUG)
53
222
    if verbose == 0:
222
    if format == 'text':
54
223
        stream = get_output_stream(format, path)
55
224
        set_stream_handler(root_logger, stream)
56
225
    else:
57
223
        set_null_log_handler(root_logger)
226
        set_null_log_handler(root_logger)
61
224
    if verbose >= 1:
227
    if verbosity >= 1:
62
225
        set_stderr_stream_handler(root_logger)
228
        autopilot.globals.set_log_verbose(True)
63
226
    if verbose >= 2:
229
    if verbosity >= 2:
64
227
        enable_debug_log_messages()
230
        enable_debug_log_messages()
65
228
    # log autopilot version
231
    # log autopilot version
66
229
    root_logger.info(get_version_string())
232
    root_logger.info(get_version_string())
67
@@ -237,11 +240,10 @@
68
237
    root_logger.addHandler(logging.NullHandler())
240
    root_logger.addHandler(logging.NullHandler())
69
238
241
70
239
242
76
240
def set_stderr_stream_handler(root_logger):
243
def set_stream_handler(root_logger, stream):
77
241
    formatter = LogFormatter()
244
    handler = logging.StreamHandler(stream=stream)
78
242
    stderr_handler = logging.StreamHandler(stream=sys.stderr)
245
    handler.setFormatter(LogFormatter())
79
243
    stderr_handler.setFormatter(formatter)
246
    root_logger.addHandler(handler)
75
244
    root_logger.addHandler(stderr_handler)
80
245
247
81
246
248
82
247
def enable_debug_log_messages():
249
def enable_debug_log_messages():
83
@@ -256,7 +258,7 @@
84
256
    )
258
    )
85
257
259
86
258
260
88
259
def get_output_stream(format, path):
261
def get_output_stream(format, path=None):
89
260
    """Get an output stream pointing to 'path' that's appropriate for format
262
    """Get an output stream pointing to 'path' that's appropriate for format
90
261
    'format'.
263
    'format'.
91
262
264
92
@@ -270,7 +272,7 @@
93
270
        log_file = _get_log_file_path(path)
272
        log_file = _get_log_file_path(path)
94
271
        if format == 'xml':
273
        if format == 'xml':
95
272
            return _get_text_mode_file_stream(log_file)
274
            return _get_text_mode_file_stream(log_file)
97
273
        elif format == 'text':
275
        elif format in ('status', 'text'):
98
274
            return _get_binary_mode_file_stream(log_file)
276
            return _get_binary_mode_file_stream(log_file)
99
275
        else:
277
        else:
100
276
            return _get_raw_binary_mode_file_stream(log_file)
278
            return _get_raw_binary_mode_file_stream(log_file)
101
@@ -639,7 +641,10 @@
102
639
        self.args = defined_args or _parse_arguments()
641
        self.args = defined_args or _parse_arguments()
103
640
642
104
641
    def run(self):
643
    def run(self):
106
642
        setup_logging(getattr(self.args, 'verbose', False))
644
        verbosity = getattr(self.args, 'verbose', 0)
107
645
        format = getattr(self.args, 'format', 'status')
108
646
        path = getattr(self.args, 'output', '')
109
647
        setup_logging(verbosity, path, format)
110
643
648
111
644
        action = None
649
        action = None
112
645
        if self.args.mode == 'list':
650
        if self.args.mode == 'list':
113
@@ -712,9 +717,6 @@
114
712
            print("Error: %s" % str(e))
717
            print("Error: %s" % str(e))
115
713
            exit(1)
718
            exit(1)
116
714
719
117
715
        if self.args.verbose:
118
716
            autopilot.globals.set_log_verbose(True)
119
717
120
718
        result = construct_test_result(self.args)
720
        result = construct_test_result(self.args)
121
719
        result.startTestRun()
721
        result.startTestRun()
122
720
        try:
722
        try:
123
721
723
124
=== modified file 'autopilot/testresult.py'
125
--- autopilot/testresult.py	2014-07-14 23:30:58 +0000
126
+++ autopilot/testresult.py	2014-07-22 03:40:04 +0000
127
@@ -1,7 +1,7 @@
128
1
# -*- Mode: Python; coding: utf-8; indent-tabs-mode: nil; tab-width: 4 -*-
1
# -*- Mode: Python; coding: utf-8; indent-tabs-mode: nil; tab-width: 4 -*-
129
2
#
2
#
130
3
# Autopilot Functional Test Tool
3
# Autopilot Functional Test Tool
132
4
# Copyright (C) 2012-2013 Canonical
4
# Copyright (C) 2012,2013,2014 Canonical
133
5
#
5
#
134
6
# This program is free software: you can redistribute it and/or modify
6
# This program is free software: you can redistribute it and/or modify
135
7
# it under the terms of the GNU General Public License as published by
7
# it under the terms of the GNU General Public License as published by
136
@@ -32,18 +32,81 @@
137
32
    try_import,
32
    try_import,
138
33
)
33
)
139
34
34
140
35
from autopilot.globals import get_log_verbose
141
36
from autopilot.utilities import _raise_on_unknown_kwargs
35
from autopilot.utilities import _raise_on_unknown_kwargs
142
37
36
143
38
37
144
38
class StatusFormatTestResult(TextTestResult):
145
39
    """A TextTestResult that prints status messages to a stream."""
146
40
147
41
    def __init__(self, stream, failfast):
148
42
        super(StatusFormatTestResult, self).__init__(stream, failfast)
149
43
        self.stream = stream
150
44
151
45
    def startTest(self, test):
152
46
        self.stream.write(u'%s' % test.id())
153
47
        self.stream.write(u' ... ')
154
48
        self.stream.flush()
155
49
        super(StatusFormatTestResult, self).startTest(test)
156
50
157
51
    def stopTest(self, test):
158
52
        self.stream.write(u'\n')
159
53
        self.stream.flush()
160
54
        super(StatusFormatTestResult, self).stopTest(test)
161
55
162
56
    def stopTestRun(self):
163
57
        self.stream.write(u'-' * 80)
164
58
        self.stream.write(u'\n')
165
59
        self.stream.flush()
166
60
        super(StatusFormatTestResult, self).stopTestRun()
167
61
168
62
    def addExpectedFailure(self, test, err=None, details=None):
169
63
        self.stream.write(u'EXPECTED FAIL')
170
64
        super(StatusFormatTestResult, self).addExpectedFailure(
171
65
            test, err, details
172
66
        )
173
67
174
68
    def addError(self, test, err=None, details=None):
175
69
        self.stream.write(u'ERROR')
176
70
        super(StatusFormatTestResult, self).addError(
177
71
            test, err, details
178
72
        )
179
73
180
74
    def addFailure(self, test, err=None, details=None):
181
75
        self.stream.write(u'FAIL')
182
76
        super(StatusFormatTestResult, self).addFailure(
183
77
            test, err, details
184
78
        )
185
79
186
80
    def addSkip(self, test, reason, details=None):
187
81
        if not reason:
188
82
            reason_displayed = ''
189
83
        else:
190
84
            reason_displayed = ' (%s)' % reason
191
85
        self.stream.write(u'SKIP%s' % reason_displayed)
192
86
        super(StatusFormatTestResult, self).addSkip(
193
87
            test, reason, details
194
88
        )
195
89
196
90
    def addSuccess(self, test, details=None):
197
91
        self.stream.write(u'OK')
198
92
        super(StatusFormatTestResult, self).addSuccess(
199
93
            test, details
200
94
        )
201
95
202
96
    def addUnexpectedSuccess(self, test, details=None):
203
97
        self.stream.write(u'UNEXPECTED SUCCESS')
204
98
        super(StatusFormatTestResult, self).addUnexpectedSuccess(
205
99
            test, details
206
100
        )
207
101
208
102
209
39
class LoggedTestResultDecorator(TestResultDecorator):
103
class LoggedTestResultDecorator(TestResultDecorator):
210
40
104
211
41
    """A decorator that logs messages to python's logging system."""
105
    """A decorator that logs messages to python's logging system."""
212
42
106
213
43
    def _log(self, level, message):
107
    def _log(self, level, message):
217
44
        """Perform the actual message logging."""
108
        """Perform the actual message logging"""
218
45
        if get_log_verbose():
109
        logging.getLogger().log(level, message)
216
46
            logging.getLogger().log(level, message)
219
47
110
220
48
    def _log_details(self, level, details):
111
    def _log_details(self, level, details):
221
49
        """Log the relavent test details."""
112
        """Log the relavent test details."""
222
@@ -87,9 +150,8 @@
223
87
150
224
88
    """
151
    """
225
89
    supported_formats = {}
152
    supported_formats = {}
227
90
153
    supported_formats['status'] = _construct_status
228
91
    supported_formats['text'] = _construct_text
154
    supported_formats['text'] = _construct_text
229
92
230
93
    if try_import('junitxml'):
155
    if try_import('junitxml'):
231
94
        supported_formats['xml'] = _construct_xml
156
        supported_formats['xml'] = _construct_xml
232
95
    if try_import('subunit'):
157
    if try_import('subunit'):
233
@@ -98,7 +160,7 @@
234
98
160
235
99
161
236
100
def get_default_format():
162
def get_default_format():
238
101
    return 'text'
163
    return 'status'
239
102
164
240
103
165
241
104
def _construct_xml(**kwargs):
166
def _construct_xml(**kwargs):
242
@@ -106,16 +168,21 @@
243
106
    stream = kwargs.pop('stream')
168
    stream = kwargs.pop('stream')
244
107
    failfast = kwargs.pop('failfast')
169
    failfast = kwargs.pop('failfast')
245
108
    _raise_on_unknown_kwargs(kwargs)
170
    _raise_on_unknown_kwargs(kwargs)
253
109
    result_object = LoggedTestResultDecorator(
171
    result = ExtendedToOriginalDecorator(JUnitXmlResult(stream))
254
110
        ExtendedToOriginalDecorator(
172
    result.failfast = failfast
255
111
            JUnitXmlResult(stream)
173
    return result
256
112
        )
174
257
113
    )
175
258
114
    result_object.failfast = failfast
176
def _construct_status(**kwargs):
259
115
    return result_object
177
    """Status (terse) text output."""
260
178
    stream = kwargs.pop('stream')
261
179
    failfast = kwargs.pop('failfast')
262
180
    _raise_on_unknown_kwargs(kwargs)
263
181
    return StatusFormatTestResult(stream, failfast)
264
116
182
265
117
183
266
118
def _construct_text(**kwargs):
184
def _construct_text(**kwargs):
267
185
    """Verbose text output."""
268
119
    stream = kwargs.pop('stream')
186
    stream = kwargs.pop('stream')
269
120
    failfast = kwargs.pop('failfast')
187
    failfast = kwargs.pop('failfast')
270
121
    _raise_on_unknown_kwargs(kwargs)
188
    _raise_on_unknown_kwargs(kwargs)
271
@@ -127,10 +194,6 @@
272
127
    stream = kwargs.pop('stream')
194
    stream = kwargs.pop('stream')
273
128
    failfast = kwargs.pop('failfast')
195
    failfast = kwargs.pop('failfast')
274
129
    _raise_on_unknown_kwargs(kwargs)
196
    _raise_on_unknown_kwargs(kwargs)
282
130
    result_object = LoggedTestResultDecorator(
197
    result = ExtendedToStreamDecorator(StreamResultToBytes(stream))
283
131
        ExtendedToStreamDecorator(
198
    result.failfast = failfast
284
132
            StreamResultToBytes(stream)
199
    return result
278
133
        )
279
134
    )
280
135
    result_object.failfast = failfast
281
136
    return result_object
285
137
200
286
=== modified file 'autopilot/tests/functional/test_autopilot_functional.py'
287
--- autopilot/tests/functional/test_autopilot_functional.py	2014-07-14 04:07:05 +0000
288
+++ autopilot/tests/functional/test_autopilot_functional.py	2014-07-22 03:40:04 +0000
289
@@ -25,9 +25,11 @@
290
25
import os.path
25
import os.path
291
26
import re
26
import re
292
27
from tempfile import mktemp
27
from tempfile import mktemp
293
28
from textwrap import dedent
294
29
295
30
from testscenarios import WithScenarios
296
28
from testtools import skipIf
31
from testtools import skipIf
297
29
from testtools.matchers import Contains, Equals, MatchesRegex, Not
32
from testtools.matchers import Contains, Equals, MatchesRegex, Not
298
30
from textwrap import dedent
299
31
33
300
32
from autopilot import platform
34
from autopilot import platform
301
33
from autopilot.tests.functional import AutopilotRunTestBase, remove_if_exists
35
from autopilot.tests.functional import AutopilotRunTestBase, remove_if_exists
302
@@ -770,15 +772,10 @@
303
770
772
304
771
class AutopilotVerboseFunctionalTests(AutopilotFunctionalTestsBase):
773
class AutopilotVerboseFunctionalTests(AutopilotFunctionalTestsBase):
305
772
774
312
773
    """Scenarioed functional tests for autopilot's verbose logging."""
775
    """Functional tests for autopilot's verbose logging."""
307
774
308
775
    scenarios = [
309
776
        ('text_format', dict(output_format='text')),
310
777
        ('xml_format', dict(output_format='xml'))
311
778
    ]
313
779
776
314
780
    def test_verbose_flag_works(self):
777
    def test_verbose_flag_works(self):
316
781
        """Verbose flag must log to stderr."""
778
        """Verbose flag must log to stdout."""
317
782
        self.create_test_file(
779
        self.create_test_file(
318
783
            "test_simple.py", dedent("""\
780
            "test_simple.py", dedent("""\
319
784
781
320
@@ -792,13 +789,13 @@
321
792
            """)
789
            """)
322
793
        )
790
        )
323
794
791
327
795
        code, output, error = self.run_autopilot(["run",
792
        code, output, error = self.run_autopilot(
328
796
                                                  "-f", self.output_format,
793
            ["run", "-f", "text", "-v", "tests"]
329
797
                                                  "-v", "tests"])
794
        )
330
798
795
331
799
        self.assertThat(code, Equals(0))
796
        self.assertThat(code, Equals(0))
332
800
        self.assertThat(
797
        self.assertThat(
334
801
            error, Contains(
798
            output, Contains(
335
802
                "Starting test tests.test_simple.SimpleTest.test_simple"))
799
                "Starting test tests.test_simple.SimpleTest.test_simple"))
336
803
800
337
804
    def test_verbose_flag_shows_timestamps(self):
801
    def test_verbose_flag_shows_timestamps(self):
338
@@ -816,11 +813,11 @@
339
816
            """)
813
            """)
340
817
        )
814
        )
341
818
815
345
819
        code, output, error = self.run_autopilot(["run",
816
        code, output, error = self.run_autopilot(
346
820
                                                  "-f", self.output_format,
817
            ["run", "-f", "text", "-v", "tests"]
347
821
                                                  "-v", "tests"])
818
        )
348
822
819
350
823
        self.assertThat(error, MatchesRegex("^\d\d:\d\d:\d\d\.\d\d\d"))
820
        self.assertThat(output, MatchesRegex("^\d\d:\d\d:\d\d\.\d\d\d"))
351
824
821
352
825
    def test_verbose_flag_shows_success(self):
822
    def test_verbose_flag_shows_success(self):
353
826
        """Verbose log must indicate successful tests (text format)."""
823
        """Verbose log must indicate successful tests (text format)."""
354
@@ -837,12 +834,12 @@
355
837
            """)
834
            """)
356
838
        )
835
        )
357
839
836
361
840
        code, output, error = self.run_autopilot(["run",
837
        code, output, error = self.run_autopilot(
362
841
                                                  "-f", self.output_format,
838
            ["run", "-f", "text", "-v", "tests"]
363
842
                                                  "-v", "tests"])
839
        )
364
843
840
365
844
        self.assertThat(
841
        self.assertThat(
367
845
            error, Contains("OK: tests.test_simple.SimpleTest.test_simple"))
842
            output, Contains("OK: tests.test_simple.SimpleTest.test_simple"))
368
846
843
369
847
    def test_verbose_flag_shows_error(self):
844
    def test_verbose_flag_shows_error(self):
370
848
        """Verbose log must indicate test error with a traceback."""
845
        """Verbose log must indicate test error with a traceback."""
371
@@ -859,15 +856,17 @@
372
859
            """)
856
            """)
373
860
        )
857
        )
374
861
858
378
862
        code, output, error = self.run_autopilot(["run",
859
        code, output, error = self.run_autopilot(
379
863
                                                  "-f", self.output_format,
860
            ["run", "-f", "text", "-v", "tests"]
380
864
                                                  "-v", "tests"])
861
        )
381
865
862
382
866
        self.assertThat(
863
        self.assertThat(
385
867
            error, Contains("ERROR: tests.test_simple.SimpleTest.test_simple"))
864
            output,
386
868
        self.assertThat(error, Contains("traceback:"))
865
            Contains("ERROR: tests.test_simple.SimpleTest.test_simple")
387
866
        )
388
867
        self.assertThat(output, Contains("traceback:"))
389
869
        self.assertThat(
868
        self.assertThat(
391
870
            error,
869
            output,
392
871
            Contains("RuntimeError: Intentionally fail test.")
870
            Contains("RuntimeError: Intentionally fail test.")
393
872
        )
871
        )
394
873
872
395
@@ -887,13 +886,13 @@
396
887
            """)
886
            """)
397
888
        )
887
        )
398
889
888
402
890
        code, output, error = self.run_autopilot(["run",
889
        code, output, error = self.run_autopilot(
403
891
                                                  "-f", self.output_format,
890
            ["run", "-f", "text", "-v", "tests"]
404
892
                                                  "-v", "tests"])
891
        )
405
893
892
409
894
        self.assertIn("FAIL: tests.test_simple.SimpleTest.test_simple", error)
893
        self.assertIn("FAIL: tests.test_simple.SimpleTest.test_simple", output)
410
895
        self.assertIn("traceback:", error)
894
        self.assertIn("traceback:", output)
411
896
        self.assertIn("AssertionError: False is not true", error)
895
        self.assertIn("AssertionError: False is not true", output)
412
897
896
413
898
    def test_verbose_flag_captures_nested_autopilottestcase_classes(self):
897
    def test_verbose_flag_captures_nested_autopilottestcase_classes(self):
414
899
        """Verbose log must contain the log details of both the nested and
898
        """Verbose log must contain the log details of both the nested and
415
@@ -917,20 +916,20 @@
416
917
            """)
916
            """)
417
918
        )
917
        )
418
919
918
422
920
        code, output, error = self.run_autopilot(["run",
919
        code, output, error = self.run_autopilot(
423
921
                                                  "-f", self.output_format,
920
            ["run", "-f", "text", "-v", "tests"])
421
922
                                                  "-v", "tests"])
424
923
921
425
924
        self.assertThat(code, Equals(0))
922
        self.assertThat(code, Equals(0))
426
923
427
925
        self.assertThat(
924
        self.assertThat(
429
926
            error,
925
            output,
430
927
            Contains(
926
            Contains(
431
928
                "Starting test tests.test_simple.OuterTestCase."
927
                "Starting test tests.test_simple.OuterTestCase."
432
929
                "test_nested_classes"
928
                "test_nested_classes"
433
930
            )
929
            )
434
931
        )
930
        )
435
932
        self.assertThat(
931
        self.assertThat(
437
933
            error,
932
            output,
438
934
            Contains(
933
            Contains(
439
935
                "Starting test tests.test_simple.InnerTestCase."
934
                "Starting test tests.test_simple.InnerTestCase."
440
936
                "test_produce_log_output"
935
                "test_produce_log_output"
441
@@ -953,11 +952,10 @@
442
953
            """)
952
            """)
443
954
        )
953
        )
444
955
954
448
956
        code, output, error = self.run_autopilot(["run",
955
        code, output, error = self.run_autopilot(
449
957
                                                  "-f", self.output_format,
956
            ["run", "-f", "text", "-vv", "tests"])
447
958
                                                  "-vv", "tests"])
450
959
957
452
960
        self.assertThat(error, Contains("Hello World"))
958
        self.assertThat(output, Contains("Hello World"))
453
961
959
454
962
    def test_debug_output_not_shown_by_default(self):
960
    def test_debug_output_not_shown_by_default(self):
455
963
        """Verbose log must not show debug messages unless we specify '-vv'."""
961
        """Verbose log must not show debug messages unless we specify '-vv'."""
456
@@ -975,11 +973,11 @@
457
975
            """)
973
            """)
458
976
        )
974
        )
459
977
975
463
978
        code, output, error = self.run_autopilot(["run",
976
        code, output, error = self.run_autopilot(
464
979
                                                  "-f", self.output_format,
977
            ["run", "-f", "text", "-v", "tests"]
465
980
                                                  "-v", "tests"])
978
        )
466
981
979
468
982
        self.assertThat(error, Not(Contains("Hello World")))
980
        self.assertThat(output, Not(Contains("Hello World")))
469
983
981
470
984
    def test_verbose_flag_shows_autopilot_version(self):
982
    def test_verbose_flag_shows_autopilot_version(self):
471
985
        from autopilot import get_version_string
983
        from autopilot import get_version_string
472
@@ -997,13 +995,23 @@
473
997
            """)
995
            """)
474
998
        )
996
        )
475
999
997
479
1000
        code, output, error = self.run_autopilot(["run",
998
        code, output, error = self.run_autopilot(
480
1001
                                                  "-f", self.output_format,
999
            ["run", "-f", "text", "-v", "tests"]
481
1002
                                                  "-v", "tests"])
1000
        )
482
1003
        self.assertThat(
1001
        self.assertThat(
486
1004
            error, Contains(get_version_string()))
1002
            output, Contains(get_version_string()))
487
1005
1003
488
1006
    def test_failfast(self):
1004
489
1005
class AutopilotFailFastTests(AutopilotRunTestBase, WithScenarios):
490
1006
491
1007
    scenarios = [
492
1008
        ('status', dict(format='status')),
493
1009
        ('text', dict(format='text')),
494
1010
        ('xml', dict(format='xml')),
495
1011
        ('subunit', dict(format='subunit')),
496
1012
    ]
497
1013
498
1014
    def test_failfast_text_mode(self):
499
1007
        """Run stops after first error encountered."""
1015
        """Run stops after first error encountered."""
500
1008
        self.create_test_file(
1016
        self.create_test_file(
501
1009
            'test_failfast.py', dedent("""\
1017
            'test_failfast.py', dedent("""\
502
@@ -1020,9 +1028,34 @@
503
1020
                    raise Exception
1028
                    raise Exception
504
1021
            """)
1029
            """)
505
1022
        )
1030
        )
509
1023
        code, output, error = self.run_autopilot(["run",
1031
        code, output, error = self.run_autopilot(
510
1024
                                                  "--failfast",
1032
            ["run", "--failfast", "-f", self.format, "tests"]
511
1025
                                                  "tests"])
1033
        )
512
1026
        self.assertThat(code, Equals(1))
1034
        self.assertThat(code, Equals(1))
513
1027
        self.assertIn("Ran 1 test", output)
1035
        self.assertIn("Ran 1 test", output)
514
1028
        self.assertIn("FAILED (failures=1)", output)
1036
        self.assertIn("FAILED (failures=1)", output)
515
1037
516
1038
517
1039
class AutopilotTestDetailsTests(AutopilotRunTestBase):
518
1040
519
1041
    def test_logging_added_as_test_detail(self):
520
1042
        msg = "This is some information"
521
1043
        self.create_test_file(
522
1044
            'test_details.py', dedent("""\
523
1045
            import logging
524
1046
            from autopilot.testcase import AutopilotTestCase
525
1047
526
1048
527
1049
            logger = logging.getLogger(__name__)
528
1050
529
1051
            class SimpleTest(AutopilotTestCase):
530
1052
531
1053
                def test_one(self):
532
1054
                    logger.info('%s')
533
1055
534
1056
            """ % msg)
535
1057
        )
536
1058
        code, output, error = self.run_autopilot(
537
1059
            ["run", "-f", "text", "tests"]
538
1060
        )
539
1061
        self.assertIn(msg, output)
540
1029
1062
541
=== modified file 'autopilot/tests/unit/test_command_line_args.py'
542
--- autopilot/tests/unit/test_command_line_args.py	2014-05-23 11:28:21 +0000
543
+++ autopilot/tests/unit/test_command_line_args.py	2014-07-22 03:40:04 +0000
544
@@ -76,7 +76,7 @@
545
76
76
546
77
    def test_launch_command_has_correct_default_verbosity(self):
77
    def test_launch_command_has_correct_default_verbosity(self):
547
78
        args = parse_args("launch app")
78
        args = parse_args("launch app")
549
79
        self.assertThat(args.verbose, Equals(False))
79
        self.assertThat(args.verbose, Equals(0))
550
80
80
551
81
    def test_launch_command_can_specify_verbosity(self):
81
    def test_launch_command_can_specify_verbosity(self):
552
82
        args = parse_args("launch -v app")
82
        args = parse_args("launch -v app")
553
@@ -211,7 +211,15 @@
554
211
211
555
212
    def test_run_command_default_format(self):
212
    def test_run_command_default_format(self):
556
213
        args = parse_args('run foo')
213
        args = parse_args('run foo')
558
214
        self.assertThat(args.format, Equals("text"))
214
        self.assertThat(args.format, Equals("status"))
559
215
560
216
    def test_run_command_status_format_short_version(self):
561
217
        args = parse_args('run -f status foo')
562
218
        self.assertThat(args.format, Equals("status"))
563
219
564
220
    def test_run_command_status_format_long_version(self):
565
221
        args = parse_args('run --format status foo')
566
222
        self.assertThat(args.format, Equals("status"))
567
215
223
568
216
    def test_run_command_text_format_short_version(self):
224
    def test_run_command_text_format_short_version(self):
569
217
        args = parse_args('run -f text foo')
225
        args = parse_args('run -f text foo')
570
218
226
571
=== modified file 'autopilot/tests/unit/test_run.py'
572
--- autopilot/tests/unit/test_run.py	2014-05-23 11:28:21 +0000
573
+++ autopilot/tests/unit/test_run.py	2014-07-22 03:40:04 +0000
574
@@ -18,7 +18,7 @@
575
18
#
18
#
576
19
19
577
20
from argparse import Namespace
20
from argparse import Namespace
579
21
from unittest.mock import Mock, patch
21
from io import StringIO
580
22
import logging
22
import logging
581
23
import os.path
23
import os.path
582
24
from shutil import rmtree
24
from shutil import rmtree
583
@@ -37,12 +37,17 @@
584
37
    Raises,
37
    Raises,
585
38
    StartsWith,
38
    StartsWith,
586
39
)
39
)
587
40
from unittest.mock import Mock, patch
588
40
41
589
41
if six.PY3:
42
if six.PY3:
590
42
    from contextlib import ExitStack
43
    from contextlib import ExitStack
591
43
else:
44
else:
592
44
    from contextlib2 import ExitStack
45
    from contextlib2 import ExitStack
593
45
46
594
47
from autopilot.testresult import (
595
48
    LoggedTestResultDecorator,
596
49
    StatusFormatTestResult,
597
50
)
598
46
from autopilot import have_vis, run
51
from autopilot import have_vis, run
599
47
52
600
48
53
601
@@ -595,27 +600,10 @@
602
595
                logging.DEBUG
600
                logging.DEBUG
603
596
            )
601
            )
604
597
602
626
598
    def test_set_null_log_handler(self):
603
    def test_stream_handler_sets_custom_formatter(self):
627
599
        mock_root_logger = Mock()
604
        mock_root_logger = Mock()
628
600
        run.set_null_log_handler(mock_root_logger)
605
        stream = StringIO()
629
601
606
        run.set_stream_handler(mock_root_logger, stream)
609
602
        self.assertThat(
610
603
            mock_root_logger.addHandler.call_args[0][0],
611
604
            IsInstance(logging.NullHandler)
612
605
        )
613
606
614
607
    @patch.object(run, 'get_root_logger')
615
608
    def test_verbse_level_zero_sets_null_handler(self, fake_get_logger):
616
609
        with patch.object(run, 'set_null_log_handler') as fake_set_null:
617
610
            run.setup_logging(0)
618
611
619
612
            fake_set_null.assert_called_once_with(
620
613
                fake_get_logger.return_value
621
614
            )
622
615
623
616
    def test_stderr_handler_sets_stream_handler_with_custom_formatter(self):
624
617
        mock_root_logger = Mock()
625
618
        run.set_stderr_stream_handler(mock_root_logger)
630
619
607
631
620
        self.assertThat(mock_root_logger.addHandler.call_count, Equals(1))
608
        self.assertThat(mock_root_logger.addHandler.call_count, Equals(1))
632
621
        created_handler = mock_root_logger.addHandler.call_args[0][0]
609
        created_handler = mock_root_logger.addHandler.call_args[0][0]
633
@@ -630,12 +618,16 @@
634
630
        )
618
        )
635
631
619
636
632
    @patch.object(run, 'get_root_logger')
620
    @patch.object(run, 'get_root_logger')
640
633
    def test_verbose_level_one_sets_stream_handler(self, fake_get_logger):
621
    def test_text_format_sets_stream_handler(self, fake_get_logger):
641
634
        with patch.object(run, 'set_stderr_stream_handler') as stderr_handler:
622
        verbosity = 0
642
635
            run.setup_logging(1)
623
        path = None
643
624
        # handler is set when running in 'text' format mode
644
625
        with patch.object(run, 'set_stream_handler') as handler:
645
626
            run.setup_logging(verbosity, path, format='text')
646
636
627
649
637
            stderr_handler.assert_called_once_with(
628
            handler.assert_called_once_with(
650
638
                fake_get_logger.return_value
629
                fake_get_logger.return_value,
651
630
                run.get_output_stream('text')
652
639
            )
631
            )
653
640
632
654
641
    def test_enable_debug_log_messages_sets_debugFilter_attr(self):
633
    def test_enable_debug_log_messages_sets_debugFilter_attr(self):
655
@@ -650,7 +642,7 @@
656
650
    @patch.object(run, 'get_root_logger')
642
    @patch.object(run, 'get_root_logger')
657
651
    def test_verbose_level_two_enables_debug_messages(self, fake_get_logger):
643
    def test_verbose_level_two_enables_debug_messages(self, fake_get_logger):
658
652
        with patch.object(run, 'enable_debug_log_messages') as enable_debug:
644
        with patch.object(run, 'enable_debug_log_messages') as enable_debug:
660
653
            run.setup_logging(2)
645
            run.setup_logging(2, None)
661
654
646
662
655
            enable_debug.assert_called_once_with()
647
            enable_debug.assert_called_once_with()
663
656
648
664
@@ -791,6 +783,13 @@
665
791
            run.get_output_stream(format, output)
783
            run.get_output_stream(format, output)
666
792
            pgts.assert_called_once_with(output)
784
            pgts.assert_called_once_with(output)
667
793
785
668
786
    def test_status_format_opens_binary_mode_stream(self):
669
787
        output = tempfile.mktemp()
670
788
        format = 'status'
671
789
        with patch.object(run, '_get_binary_mode_file_stream') as pgbs:
672
790
            run.get_output_stream(format, output)
673
791
            pgbs.assert_called_once_with(output)
674
792
675
794
    def test_txt_format_opens_binary_mode_stream(self):
793
    def test_txt_format_opens_binary_mode_stream(self):
676
795
        output = tempfile.mktemp()
794
        output = tempfile.mktemp()
677
796
        format = 'text'
795
        format = 'text'
678
@@ -815,6 +814,14 @@
679
815
        expected = "Using default log filename: %s\n" % path
814
        expected = "Using default log filename: %s\n" % path
680
816
        self.assertThat(expected, Equals(output))
815
        self.assertThat(expected, Equals(output))
681
817
816
682
817
    def test_text_result_uses_logged_decorator(self):
683
818
        args = Namespace()
684
819
        args.failfast = False
685
820
        args.output = None
686
821
        args.format = "text"
687
822
        result = run.construct_test_result(args)
688
823
        self.assertIsInstance(result, LoggedTestResultDecorator)
689
824
690
818
825
691
819
class TestProgramTests(TestCase):
826
class TestProgramTests(TestCase):
692
820
827
693
@@ -841,12 +848,12 @@
694
841
            self.assertThat(program.args, Equals(fake_args))
848
            self.assertThat(program.args, Equals(fake_args))
695
842
849
696
843
    def test_run_calls_setup_logging_with_verbose_arg(self):
850
    def test_run_calls_setup_logging_with_verbose_arg(self):
698
844
        fake_args = Namespace(verbose=1, mode='')
851
        fake_args = Namespace(verbose=1, mode='text')
699
845
        program = run.TestProgram(fake_args)
852
        program = run.TestProgram(fake_args)
700
846
        with patch.object(run, 'setup_logging') as patched_setup_logging:
853
        with patch.object(run, 'setup_logging') as patched_setup_logging:
701
847
            program.run()
854
            program.run()
702
848
855
704
849
            patched_setup_logging.assert_called_once_with(True)
856
            patched_setup_logging.assert_called_once_with(1, '', 'status')
705
850
857
706
851
    def test_list_command_calls_list_tests_method(self):
858
    def test_list_command_calls_list_tests_method(self):
707
852
        fake_args = Namespace(mode='list')
859
        fake_args = Namespace(mode='list')
708
@@ -966,3 +973,93 @@
709
966
    )
973
    )
710
967
    defaults.update(kwargs)
974
    defaults.update(kwargs)
711
968
    return Namespace(**defaults)
975
    return Namespace(**defaults)
712
976
713
977
714
978
def get_case(kind):
715
979
    # Define the class in a function so test loading doesn't
716
980
    #   try to load it as a regular test class.
717
981
    class Test(TestCase):
718
982
719
983
        def test_pass(self):
720
984
            pass
721
985
722
986
        def test_fail(self):
723
987
            raise self.failureException
724
988
725
989
        def test_error(self):
726
990
            raise SyntaxError
727
991
728
992
        def test_skip(self):
729
993
            self.skipTest('')
730
994
731
995
        def test_skip_reason(self):
732
996
            self.skipTest('Because')
733
997
734
998
        def test_expected_failure(self):
735
999
            # We expect the test to fail and it does
736
1000
            self.expectFailure("1 should be 0", self.assertEqual, 1, 0)
737
1001
738
1002
        def test_unexpected_success(self):
739
1003
            # We expect the test to fail but it doesn't
740
1004
            self.expectFailure("1 is not 1", self.assertEqual, 1, 1)
741
1005
742
1006
    test_method = 'test_%s' % kind
743
1007
    return Test(test_method)
744
1008
745
1009
746
1010
def expand_template_for_test(template, test):
747
1011
    """Expand common references in template.
748
1012
749
1013
    Tests that check runs output can be simplified if they use templates
750
1014
    instead of literal expected strings.
751
1015
752
1016
    :param template: A string where common strings have been replaced by a
753
1017
        keyword.
754
1018
755
1019
    :param test: The test case under scrutiny.
756
1020
    """
757
1021
    kwargs = dict(id=test.id())
758
1022
    return template.format(**kwargs)
759
1023
760
1024
761
1025
class StatusFormatTestResultTests(TestCase):
762
1026
763
1027
    def setUp(self):
764
1028
        super(StatusFormatTestResultTests, self).setUp()
765
1029
766
1030
    def assertOutput(self, template, kind):
767
1031
        test = get_case(kind)
768
1032
        result = StatusFormatTestResult(StringIO(), False)
769
1033
        test.run(result)
770
1034
771
1035
        expected = expand_template_for_test(template, test)
772
1036
        self.assertEquals(expected, result.stream.getvalue().strip())
773
1037
774
1038
    def test_pass(self):
775
1039
        self.assertOutput('{id} ... OK', 'pass')
776
1040
777
1041
    def test_fail(self):
778
1042
        self.assertOutput('{id} ... FAIL', 'fail')
779
1043
780
1044
    def test_error(self):
781
1045
        self.assertOutput('{id} ... ERROR', 'error')
782
1046
783
1047
    def test_skip(self):
784
1048
        self.assertOutput('{id} ... SKIP', 'skip')
785
1049
786
1050
    def test_skip_reason(self):
787
1051
        self.assertOutput('{id} ... SKIP (Because)', 'skip_reason')
788
1052
789
1053
    def test_expected_failure(self):
790
1054
        self.assertOutput('{id} ... EXPECTED FAIL', 'expected_failure')
791
1055
792
1056
    def test_unexpected_success(self):
793
1057
        self.assertOutput('{id} ... UNEXPECTED SUCCESS', 'unexpected_success')
794
1058
795
1059
    def test_result_uses_status_class(self):
796
1060
        args = Namespace()
797
1061
        args.failfast = False
798
1062
        args.output = None
799
1063
        args.format = 'status'
800
1064
        result = run.construct_test_result(args)
801
1065
        self.assertIsInstance(result, StatusFormatTestResult)
802
969
1066
803
=== modified file 'autopilot/tests/unit/test_testresults.py'
804
--- autopilot/tests/unit/test_testresults.py	2014-07-01 00:36:26 +0000
805
+++ autopilot/tests/unit/test_testresults.py	2014-07-22 03:40:04 +0000
806
@@ -27,8 +27,7 @@
807
27
from testscenarios import WithScenarios
27
from testscenarios import WithScenarios
808
28
import unittest
28
import unittest
809
29
29
812
30
from autopilot import testresult
30
from autopilot import run, testresult
811
31
from autopilot import run
813
32
31
814
33
32
815
34
class LoggedTestResultDecoratorTests(TestCase):
33
class LoggedTestResultDecoratorTests(TestCase):
816
@@ -110,6 +109,9 @@
817
110
109
818
111
class OutputFormatFactoryTests(TestCase):
110
class OutputFormatFactoryTests(TestCase):
819
112
111
820
112
    def test_has_status_format(self):
821
113
        self.assertTrue('status' in testresult.get_output_formats())
822
114
823
113
    def test_has_text_format(self):
115
    def test_has_text_format(self):
824
114
        self.assertTrue('text' in testresult.get_output_formats())
116
        self.assertTrue('text' in testresult.get_output_formats())
825
115
117
826
@@ -278,6 +280,22 @@
827
278
        self.assertFalse(test_result.wasSuccessful())
280
        self.assertFalse(test_result.wasSuccessful())
828
279
        self.assertEqual(1, test_result.testsRun)
281
        self.assertEqual(1, test_result.testsRun)
829
280
282
830
283
    def test_traceback_added_as_detail_on_fail(self):
831
284
        msg = "failure info"
832
285
833
286
        class FailingTests(TestCase):
834
287
835
288
            def test_fails(self):
836
289
                self.fail(msg)
837
290
838
291
        test = FailingTests('test_fails')
839
292
        test_result, output_path = self.run_test_with_result(
840
293
            test
841
294
        )
842
295
        self.assertFalse(test_result.wasSuccessful())
843
296
        tb_detail = test.getDetails()['traceback'].as_text()
844
297
        self.assertThat(tb_detail, Contains(msg))
845
298
846
281
299
847
282
def remove_if_exists(path):
300
def remove_if_exists(path):
848
283
    if os.path.exists(path):
301
    if os.path.exists(path):
Status:	Needs review
Proposed branch:	lp:~veebers/autopilot/texttest-run-uprefresh
Merge into:	lp:autopilot
Diff against target:	848 lines (+351/-130) 7 files modified autopilot/__init__.py (+1/-1) autopilot/run.py (+21/-19) autopilot/testresult.py (+85/-22) autopilot/tests/functional/test_autopilot_functional.py (+87/-54) autopilot/tests/unit/test_command_line_args.py (+10/-2) autopilot/tests/unit/test_run.py (+127/-30) autopilot/tests/unit/test_testresults.py (+20/-2)
To merge this branch:	bzr merge lp:~veebers/autopilot/texttest-run-uprefresh
Related bugs:	Link a bug report
Reviewer	Review Type	Date Requested	Status
platform-qa-bot	continuous-integration		Approve on 2016-01-19
PS Jenkins bot	continuous-integration	2014-07-02	Needs Fixing on 2014-07-22
prod-platform-qa	continuous-integration	2017-02-22	Pending
Christopher Lee			Pending
Thomi Richards		2014-07-02	Pending
Corey Goldberg		2014-07-02	Pending
Review via email: mp+225247@code.launchpad.net