Merge lp:~jml/testtools/better-doctest-output-checker into lp:~testtools-committers/testtools/trunk

Proposed by Jonathan Lange on 2011-09-09
Status: Work in progress
Proposed branch: lp:~jml/testtools/better-doctest-output-checker
Merge into: lp:~testtools-committers/testtools/trunk
Diff against target: 356 lines (+216/-40) 5 files modified
To merge this branch: bzr merge lp:~jml/testtools/better-doctest-output-checker
Reviewer Review Type Date Requested Status
testtools committers 2011-09-09 Pending
Review via email: mp+74842@code.launchpad.net

Description of the Change

An attempt to make a better OutputChecker for doctests.

To post a comment you must log in.

Unmerged revisions

239. By Jonathan Lange on 2011-09-09

Documentation.

238. By Jonathan Lange on 2011-09-09

Use it for DocTestMatches

237. By Jonathan Lange on 2011-09-09

NEWS update.

236. By Jonathan Lange on 2011-09-09

Whitespace.

235. By Jonathan Lange on 2011-09-09

Get the abstraction right.:

234. By Jonathan Lange on 2011-09-09

Merge IntelligentOutputChecker with _NonManglingOutputChecker.

233. By Jonathan Lange on 2011-09-09

Collapse multiple blank lines.

232. By Jonathan Lange on 2011-09-09

Move ellipsize into the smart output checker

231. By Jonathan Lange on 2011-09-09

Refactor so we don't have to copy and paste.

230. By Jonathan Lange on 2011-09-09

Limited line-based ellipsis normalization.

Preview Diff

1=== modified file 'NEWS'
2--- NEWS 2011-08-14 12:13:52 +0000
3+++ NEWS 2011-09-09 17:39:51 +0000
4@@ -14,6 +14,9 @@
5 now deprecated. Please stop using it.
6 (Jonathan Lange, #813460)
7
8+* ``DocTestMatches`` now uses ``SmartOutputChecker``, leading to slightly
9+ different error output. (Jonathan Lange)
10+
11 * ``gather_details`` takes two dicts, rather than two detailed objects.
12 (Jonathan Lange, #801027)
13
14@@ -76,6 +79,10 @@
15 * New convenience assertions, ``assertIsNone`` and ``assertIsNotNone``.
16 (Christian Kampka)
17
18+* New doctest helper, ``SmartOutputChecker`` added. It normalizes whitespace
19+ and ellipses in diff output if the checker has those options set.
20+ (Jonathan Lange)
21+
22 * New matchers:
23
24 * ``AllMatch`` matches many values against a single matcher.
25
26=== modified file 'doc/for-test-authors.rst'
27--- doc/for-test-authors.rst 2011-08-15 16:14:42 +0000
28+++ doc/for-test-authors.rst 2011-09-09 17:39:51 +0000
29@@ -1179,6 +1179,16 @@
30 particular attribute.
31
32
33+Better doctest output
34+---------------------
35+
36+If you have doctests, then you might want to use ``SmartOutputChecker``
37+instead of the default. It has correct unicode behaviour across all of the
38+Pythons supported by testtools, and also will reformat the test output
39+slightly if NORMALIZE_WHITESPACE or ELLIPSIS options have been specified, in
40+order to reduce the amount of noise in the error messages.
41+
42+
43 .. _testrepository: https://launchpad.net/testrepository
44 .. _Trial: http://twistedmatrix.com/documents/current/core/howto/testing.html
45 .. _nose: http://somethingaboutorange.com/mrl/projects/nose/
46
47=== modified file 'testtools/helpers.py'
48--- testtools/helpers.py 2011-07-20 20:34:29 +0000
49+++ testtools/helpers.py 2011-09-09 17:39:51 +0000
50@@ -2,12 +2,17 @@
51
52 __all__ = [
53 'safe_hasattr',
54+ 'SmartOutputChecker',
55 'try_import',
56 'try_imports',
57 ]
58
59+import doctest
60+import re
61 import sys
62
63+from testtools.monkey import MonkeyPatcher
64+
65
66 def try_import(name, alternative=None, error_callback=None):
67 """Attempt to import ``name``. If it fails, return ``alternative``.
68@@ -85,3 +90,116 @@
69 properties.
70 """
71 return getattr(obj, attr, _marker) is not _marker
72+
73+
74+class _NonManglingOutputChecker(doctest.OutputChecker):
75+ """Doctest checker that works with unicode rather than mangling strings
76+
77+ This is needed because current Python versions have tried to fix string
78+ encoding related problems, but regressed the default behaviour with
79+ unicode inputs in the process.
80+
81+ In Python 2.6 and 2.7 `OutputChecker.output_difference` is was changed to
82+ return a bytestring encoded as per `sys.stdout.encoding`, or utf-8 if that
83+ can't be determined. Worse, that encoding process happens in the innocent
84+ looking `_indent` global function. Because the `DocTestMismatch.describe`
85+ result may well not be destined for printing to stdout, this is no good
86+ for us. To get a unicode return as before, the method is monkey patched if
87+ `doctest._encoding` exists.
88+
89+ Python 3 has a different problem. For some reason both inputs are encoded
90+ to ascii with 'backslashreplace', making an escaped string matches its
91+ unescaped form. Overriding the offending `OutputChecker._toAscii` method
92+ is sufficient to revert this.
93+ """
94+
95+ def _toAscii(self, s):
96+ """Return `s` unchanged rather than mangling it to ascii"""
97+ return s
98+
99+ def output_difference(self, example, got, optionflags):
100+ """Describe the difference between 'example' and 'got'.
101+
102+ On versions of Python that have a broken doctest._indent function,
103+ replace the behaviour of that function.
104+ """
105+ if getattr(doctest, "_encoding", None) is None:
106+ return doctest.OutputChecker.output_difference(
107+ self, example, got, optionflags)
108+ else:
109+ def _indent(s, indent=4, _pattern=re.compile("^(?!$)", re.MULTILINE)):
110+ """Prepend non-empty lines in `s` with `indent` number of spaces"""
111+ return _pattern.sub(indent*" ", s)
112+ # Only do this overriding hackery if doctest has a broken _indent
113+ # function
114+ patcher = MonkeyPatcher((doctest, '_indent', _indent))
115+ return patcher.run_with_patches(
116+ doctest.OutputChecker.output_difference,
117+ self, example, got, optionflags)
118+
119+
120+class SmartOutputChecker(_NonManglingOutputChecker):
121+ """OutputChecker that mangles the 'got' based on matching options.
122+
123+ One problem with doctests is that when you use, say, NORMALIZE_WHITESPACE,
124+ and an example fails, then the diff will display as if the differences in
125+ whitespace matter.
126+
127+ Likewise, the diff will treat lines that match correctly with the ELLIPSIS
128+ option as differing.
129+
130+ This makes it difficult to determine which lines are the *actual* cause of
131+ the difference.
132+
133+ This ``OutputChecker`` mangles the output of the doctest example so that
134+ it normalizes whitespace and ellipses if needed.
135+ """
136+
137+ def ellipsize(self, want, got):
138+ """Turn 'got' into a version of itself, but replaced with ellipses."""
139+ if doctest.ELLIPSIS_MARKER not in want:
140+ return got
141+
142+ wants = want.split(doctest.ELLIPSIS_MARKER)
143+ gots = []
144+
145+ startpos = 0
146+ for w in wants:
147+ index = got.find(w, startpos)
148+ if index < 0:
149+ if gots:
150+ gots[-1] += got[startpos:]
151+ else:
152+ gots = [got]
153+ break
154+ else:
155+ gots.append(w)
156+ startpos = index + len(w)
157+ return doctest.ELLIPSIS_MARKER.join(gots)
158+
159+ def normalize_whitespace(self, output):
160+ """Take 'output', preserve its lines, but normalize other whitespace.
161+ """
162+ lines = []
163+ for line in output.splitlines():
164+ normalized = ' '.join(line.split())
165+ if normalized or not (lines and lines[-1] == ''):
166+ lines.append(normalized)
167+ return '\n'.join(lines)
168+
169+ def output_difference(self, example, got, optionflags):
170+ """Describe the difference between 'example' and 'got'.
171+
172+ Instead of the default ``OutputChecker``, this one applies the same
173+ transformations to the 'got' string that ``check_output`` does. This
174+ means that the diff comparison is easier.
175+ """
176+ if optionflags & doctest.ELLIPSIS:
177+ got = self.ellipsize(example.want, got)
178+
179+ if optionflags & doctest.NORMALIZE_WHITESPACE:
180+ example.want = self.normalize_whitespace(example.want)
181+ got = self.normalize_whitespace(got)
182+
183+ return _NonManglingOutputChecker.output_difference(
184+ self, example, got, optionflags)
185
186=== modified file 'testtools/matchers.py'
187--- testtools/matchers.py 2011-08-15 13:22:29 +0000
188+++ testtools/matchers.py 2011-09-09 17:39:51 +0000
189@@ -38,7 +38,6 @@
190 'StartsWith',
191 ]
192
193-import doctest
194 import operator
195 from pprint import pformat
196 import re
197@@ -51,6 +50,7 @@
198 isbaseexception,
199 istext,
200 )
201+from testtools.helpers import SmartOutputChecker
202
203
204 class Matcher(object):
205@@ -156,44 +156,6 @@
206 return self.original.get_details()
207
208
209-class _NonManglingOutputChecker(doctest.OutputChecker):
210- """Doctest checker that works with unicode rather than mangling strings
211-
212- This is needed because current Python versions have tried to fix string
213- encoding related problems, but regressed the default behaviour with unicode
214- inputs in the process.
215-
216- In Python 2.6 and 2.7 `OutputChecker.output_difference` is was changed to
217- return a bytestring encoded as per `sys.stdout.encoding`, or utf-8 if that
218- can't be determined. Worse, that encoding process happens in the innocent
219- looking `_indent` global function. Because the `DocTestMismatch.describe`
220- result may well not be destined for printing to stdout, this is no good
221- for us. To get a unicode return as before, the method is monkey patched if
222- `doctest._encoding` exists.
223-
224- Python 3 has a different problem. For some reason both inputs are encoded
225- to ascii with 'backslashreplace', making an escaped string matches its
226- unescaped form. Overriding the offending `OutputChecker._toAscii` method
227- is sufficient to revert this.
228- """
229-
230- def _toAscii(self, s):
231- """Return `s` unchanged rather than mangling it to ascii"""
232- return s
233-
234- # Only do this overriding hackery if doctest has a broken _input function
235- if getattr(doctest, "_encoding", None) is not None:
236- from types import FunctionType as __F
237- __f = doctest.OutputChecker.output_difference.im_func
238- __g = dict(__f.func_globals)
239- def _indent(s, indent=4, _pattern=re.compile("^(?!$)", re.MULTILINE)):
240- """Prepend non-empty lines in `s` with `indent` number of spaces"""
241- return _pattern.sub(indent*" ", s)
242- __g["_indent"] = _indent
243- output_difference = __F(__f.func_code, __g, "output_difference")
244- del __F, __f, __g, _indent
245-
246-
247 class DocTestMatches(object):
248 """See if a string matches a doctest example."""
249
250@@ -208,7 +170,7 @@
251 example += '\n'
252 self.want = example # required variable name by doctest.
253 self.flags = flags
254- self._checker = _NonManglingOutputChecker()
255+ self._checker = SmartOutputChecker()
256
257 def __str__(self):
258 if self.flags:
259
260=== modified file 'testtools/tests/test_helpers.py'
261--- testtools/tests/test_helpers.py 2011-08-15 13:48:10 +0000
262+++ testtools/tests/test_helpers.py 2011-09-09 17:39:51 +0000
263@@ -1,7 +1,10 @@
264 # Copyright (c) 2010-2011 testtools developers. See LICENSE for details.
265
266+import doctest
267+
268 from testtools import TestCase
269 from testtools.helpers import (
270+ SmartOutputChecker,
271 try_import,
272 try_imports,
273 )
274@@ -235,6 +238,82 @@
275 self.assertThat(self.modules, StackHidden(False))
276
277
278+class TestSmartOutputChecker(TestCase):
279+
280+ def test_ellipsis_matches(self):
281+ checker = SmartOutputChecker()
282+ self.assertTrue(checker.check_output('x...', 'xyz', doctest.ELLIPSIS))
283+
284+ def test_normalize_whitespace_multiple_blank_lines(self):
285+ checker = SmartOutputChecker()
286+ output = checker.normalize_whitespace('a\n\n\n\nb')
287+ self.assertEqual('a\n\nb', output)
288+
289+ def test_diff_with_normalize_whitespace(self):
290+ # If we're checking with normalized whitespace, then normalize the
291+ # whitespace in the diff.
292+ checker = SmartOutputChecker()
293+ example = doctest.Example('f()', 'a b\nc\n')
294+ diff = checker.output_difference(
295+ example, 'a b\nd\n',
296+ doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF)
297+ self.assertEqual(
298+ ('Differences (ndiff with -expected +actual):\n'
299+ ' a b\n'
300+ ' - c\n'
301+ ' + d\n'), diff)
302+
303+ def test_ellipsis(self):
304+ # If we're checking with ellipsis, then lines that match based on
305+ # ellipsis matching should not appear as different in the diff.
306+ checker = SmartOutputChecker()
307+ example = doctest.Example('f()', 'a...\nb...\nc...\n')
308+ diff = checker.output_difference(
309+ example, 'apple\nbobble\ndog\n',
310+ doctest.ELLIPSIS | doctest.REPORT_NDIFF)
311+ self.assertEqual(
312+ ('Differences (ndiff with -expected +actual):\n'
313+ ' a...\n'
314+ ' - b...\n'
315+ ' - c...\n'
316+ ' + bobble\n'
317+ ' + dog\n'), diff)
318+
319+ def test_no_ellipsis(self):
320+ got = SmartOutputChecker().ellipsize('xxx', 'apple\ncat\n')
321+ self.assertEqual('apple\ncat\n', got)
322+
323+ def test_starts_with_literal(self):
324+ got = SmartOutputChecker().ellipsize('app...', 'apple\ncat\n')
325+ self.assertEqual('app...', got)
326+
327+ def test_starts_with_literal_mismatch(self):
328+ got = SmartOutputChecker().ellipsize('app...', 'cattle\n')
329+ self.assertEqual('cattle\n', got)
330+
331+ def test_ends_with_literal(self):
332+ got = SmartOutputChecker().ellipsize('...cat\n', 'apple\ncat\n')
333+ self.assertEqual('...cat\n', got)
334+
335+ def test_ends_with_literal_mismatch(self):
336+ got = SmartOutputChecker().ellipsize('...cat\n', 'apple\ndog\n')
337+ self.assertEqual('apple\ndog\n', got)
338+
339+ def test_surrounded_by_literals(self):
340+ got = SmartOutputChecker().ellipsize(
341+ 'app...cat\n', 'apple\ncat\n')
342+ self.assertEqual('app...cat\n', got)
343+
344+ def test_partial_match(self):
345+ got = SmartOutputChecker().ellipsize(
346+ 'app...\nc...\nd...\n', 'apple\ncat\negg\n')
347+ self.assertEqual('app...\ncat\negg\n', got)
348+
349+ def test_too_many_literals(self):
350+ got = SmartOutputChecker().ellipsize('aa...aa', 'aaa')
351+ self.assertEqual('aaa', got)
352+
353+
354 def test_suite():
355 from unittest import TestLoader
356 return TestLoader().loadTestsFromName(__name__)

Subscribers

People subscribed via source and target branches