Merge lp:~pyflakes-dev/pyflakes/1172130-doctest-syntax-error into lp:pyflakes

[Florent]

Fixes the regression in 0.7
Needs tests.

[lp:~pyflakes-dev/pyflakes/1172130-doctest-syntax-error]

99. By Florent on 2013-04-24

Add boolean attribute Checker.withDoctest to ignore doctests

100. By Florent on 2013-04-24

Add environment variables PYFLAKES_NODOCTEST and PYFLAKES_BUILTINS

101. By Florent on 2013-04-24

Add tests for DoctestSyntaxError lineno and col

102. By Florent on 2013-04-24

More tests for doctest (lineno, col) calculation

=== modified file 'NEWS.txt'
--- NEWS.txt	2013-04-23 07:17:03 +0000
+++ NEWS.txt	2013-04-24 12:04:24 +0000
@@ -1,4 +1,15 @@
-0.7.1 (2013-04-23):
+<<<<<<< TREE
+0.7.1 (2013-04-23):
+=======
+0.7.x (unreleased):
+  - Fix computation of `DoctestSyntaxError.lineno` and `col`.
+  - Add boolean attribute `Checker.withDoctest` to ignore doctests.
+  - If environment variable `PYFLAKES_NODOCTEST` is set, skip doctests.
+  - Environment variable `PYFLAKES_BUILTINS` accepts a comma-separated
+    list of additional built-in names.
+
+0.7.1 (2013-04-23):
+>>>>>>> MERGE-SOURCE
   - File `bin/pyflakes` was missing in tarball generated with distribute.
   - Fix reporting errors in non-ASCII filenames (Python 2.x).
 

=== modified file 'pyflakes/__init__.py'
--- pyflakes/__init__.py	2013-04-23 07:17:03 +0000
+++ pyflakes/__init__.py	2013-04-24 12:04:24 +0000
@@ -1,2 +1,6 @@
 
+<<<<<<< TREE
 __version__ = '0.7.1'
+=======
+__version__ = '0.7.2a0'
+>>>>>>> MERGE-SOURCE

=== modified file 'pyflakes/checker.py'
--- pyflakes/checker.py	2013-04-21 16:07:22 +0000
+++ pyflakes/checker.py	2013-04-24 12:04:24 +0000
@@ -5,7 +5,7 @@
 Also, it models the Bindings and Scopes.
 """
 import doctest
-import os.path
+import os
 import sys
 try:
     builtin_vars = dir(__import__('builtins'))
@@ -225,7 +225,13 @@
     nodeDepth = 0
     offset = None
     traceTree = False
+    withDoctest = ('PYFLAKES_NODOCTEST' not in os.environ)
+
     builtIns = set(builtin_vars).union(_MAGIC_GLOBALS)
+    _customBuiltIns = os.environ.get('PYFLAKES_BUILTINS')
+    if _customBuiltIns:
+        builtIns.update(_customBuiltIns.split(','))
+    del _customBuiltIns
 
     def __init__(self, tree, filename='(none)', builtins=None):
         self._nodeHandlers = {}
@@ -523,7 +529,11 @@
     def getDocstring(self, node):
         if isinstance(node, ast.Expr):
             node = node.value
-        return node.s if isinstance(node, ast.Str) else None
+        if not isinstance(node, ast.Str):
+            return (None, None)
+        # Computed incorrectly if the docstring has backslash
+        doctest_lineno = node.lineno - node.s.count('\n') - 1
+        return (node.s, doctest_lineno)
 
     def handleNode(self, node, parent):
         if node is None:
@@ -550,12 +560,12 @@
     _getDoctestExamples = doctest.DocTestParser().get_examples
 
     def handleDoctests(self, node):
-        docstring = node.body and self.getDocstring(node.body[0])
-        if not docstring:
-            return
         try:
+            docstring, node_lineno = self.getDocstring(node.body[0])
+            if not docstring:
+                return
             examples = self._getDoctestExamples(docstring)
-        except ValueError:
+        except (ValueError, IndexError):
             # e.g. line 6 of the docstring for <string> has inconsistent
             # leading whitespace: ...
             return
@@ -566,11 +576,11 @@
                 tree = compile(example.source, "<doctest>", "exec", ast.PyCF_ONLY_AST)
             except SyntaxError:
                 e = sys.exc_info()[1]
-                node.lineno += example.lineno + e.lineno
-                node.col_offset += example.indent + 4 + e.offset
-                self.report(messages.DoctestSyntaxError, node)
+                position = (node_lineno + example.lineno + e.lineno,
+                            example.indent + 4 + e.offset)
+                self.report(messages.DoctestSyntaxError, node, position)
             else:
-                self.offset = (node_offset[0] + node.lineno + example.lineno,
+                self.offset = (node_offset[0] + node_lineno + example.lineno,
                                node_offset[1] + example.indent + 4)
                 self.handleChildren(tree)
                 self.offset = node_offset
@@ -689,7 +699,8 @@
             self.handleNode(deco, node)
         self.addBinding(node, FunctionDefinition(node.name, node))
         self.LAMBDA(node)
-        self.deferFunction(lambda: self.handleDoctests(node))
+        if self.withDoctest:
+            self.deferFunction(lambda: self.handleDoctests(node))
 
     def LAMBDA(self, node):
         args = []
@@ -767,7 +778,8 @@
             for keywordNode in node.keywords:
                 self.handleNode(keywordNode, node)
         self.pushClassScope()
-        self.deferFunction(lambda: self.handleDoctests(node))
+        if self.withDoctest:
+            self.deferFunction(lambda: self.handleDoctests(node))
         for stmt in node.body:
             self.handleNode(stmt, node)
         self.popScope()

=== modified file 'pyflakes/messages.py'
--- pyflakes/messages.py	2013-04-21 15:32:22 +0000
+++ pyflakes/messages.py	2013-04-24 12:04:24 +0000
@@ -68,8 +68,10 @@
 class DoctestSyntaxError(Message):
     message = 'syntax error in doctest'
 
-    def __init__(self, filename, loc):
+    def __init__(self, filename, loc, position=None):
         Message.__init__(self, filename, loc)
+        if position:
+            (self.lineno, self.col) = position
         self.message_args = ()
 
 

=== modified file 'pyflakes/test/test_doctests.py'
--- pyflakes/test/test_doctests.py	2013-04-04 23:04:43 +0000
+++ pyflakes/test/test_doctests.py	2013-04-24 12:04:24 +0000
@@ -138,14 +138,27 @@
         self.assertEqual(exc.col, 0)
 
     def test_syntaxErrorInDoctest(self):
-        exc = super(Test, self).flakes('''
-        def doctest_stuff():
-            """
-                >>> from # line 4
-            """
-        ''', m.DoctestSyntaxError).messages[0]
+        exceptions = super(Test, self).flakes(
+            '''
+            def doctest_stuff():
+                """
+                    >>> from # line 4
+                    >>>     fortytwo = 42
+                    >>> except Exception:
+                """
+            ''',
+            m.DoctestSyntaxError,
+            m.DoctestSyntaxError,
+            m.DoctestSyntaxError).messages
+        exc = exceptions[0]
         self.assertEqual(exc.lineno, 4)
         self.assertEqual(exc.col, 26)
+        exc = exceptions[1]
+        self.assertEqual(exc.lineno, 5)
+        self.assertEqual(exc.col, 16)
+        exc = exceptions[2]
+        self.assertEqual(exc.lineno, 6)
+        self.assertEqual(exc.col, 18)
 
     def test_indentationErrorInDoctest(self):
         exc = super(Test, self).flakes('''
@@ -158,6 +171,24 @@
         self.assertEqual(exc.lineno, 5)
         self.assertEqual(exc.col, 16)
 
+    def test_offsetWithMultiLineArgs(self):
+        (exc1, exc2) = super(Test, self).flakes(
+            '''
+            def doctest_stuff(arg1,
+                              arg2,
+                              arg3):
+                """
+                    >>> assert
+                    >>> this
+                """
+            ''',
+            m.DoctestSyntaxError,
+            m.UndefinedName).messages
+        self.assertEqual(exc1.lineno, 6)
+        self.assertEqual(exc1.col, 19)
+        self.assertEqual(exc2.lineno, 7)
+        self.assertEqual(exc2.col, 12)
+
     def test_doctestCanReferToFunction(self):
         super(Test, self).flakes("""
         def foo():