fix: catch TokenError on parse (#1788)

* catch TokenError on parse * typo, use env.PYVERSION * Update tests/test_parser.py --------- Co-authored-by: Ned Batchelder <[email protected]>
nedbat · May 20, 2024 · 364282e · 364282e
1 parent 81089de
commit 364282e
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 2 deletions.
diff --git a/coverage/parser.py b/coverage/parser.py
@@ -263,10 +263,14 @@ def parse_source(self) -> None:
         try:
             self._ast_root = ast.parse(self.text)
             self._raw_parse()
-        except (IndentationError, SyntaxError) as err:
+        except (tokenize.TokenError, IndentationError, SyntaxError) as err:
+            if hasattr(err, "lineno"):
+                lineno = err.lineno         # IndentationError
+            else:
+                lineno = err.args[1][0]     # TokenError
             raise NotPython(
                 f"Couldn't parse '{self.filename}' as Python source: " +
-                f"{err.args[0]!r} at line {err.lineno}",
+                f"{err.args[0]!r} at line {lineno}",
             ) from err
 
         ignore = self.excluded | self.raw_docstrings

diff --git a/tests/test_parser.py b/tests/test_parser.py
@@ -128,6 +128,14 @@ def foo():
         pytest.param("0 spaces\n  2\n 1", id="bad_indent"),
         pytest.param("'''", id="string_eof"),
         pytest.param("$hello", id="dollar"),
+        # on 3.10 this passes ast.parse but fails on tokenize.generate_tokens
+        pytest.param(
+            "\r'\\\n'''",
+            id="leading_newline_eof",
+            marks=[
+                pytest.mark.skipif(env.PYVERSION >= (3, 12), reason="parses fine in 3.12"),
+            ]
+        )
     ])
     def test_not_python(self, text: str) -> None:
         msg = r"Couldn't parse '<code>' as Python source: '.*' at line \d+"