Fix parser when using Python 3.9 (#8716)

The structure of the AST for subscripts was changed in python/cpython#9605.

This also fixes some systemic issues with the tests under Python 3.9, but not all.

Fixes #8627

mypy/fastparse.py

@@ -49,6 +49,8 @@
         import ast as ast3
         assert 'kind' in ast3.Constant._fields, \
                "This 3.8.0 alpha (%s) is too old; 3.8.0a3 required" % sys.version.split()[0]
+        # TODO: Num, Str, Bytes, NameConstant, Ellipsis are deprecated in 3.8.
+        # TODO: Index, ExtSlice are deprecated in 3.9.
         from ast import (
             AST,
             Call,
@@ -1503,19 +1505,30 @@ def visit_Bytes(self, n: Bytes) -> Type:
         contents = bytes_to_human_readable_repr(n.s)
         return RawExpressionType(contents, 'builtins.bytes', self.line, column=n.col_offset)
 
-    # Subscript(expr value, slice slice, expr_context ctx)
+    # Subscript(expr value, slice slice, expr_context ctx)  # Python 3.8 and before
+    # Subscript(expr value, expr slice, expr_context ctx)  # Python 3.9 and later
     def visit_Subscript(self, n: ast3.Subscript) -> Type:
-        if not isinstance(n.slice, Index):
-            self.fail(TYPE_COMMENT_SYNTAX_ERROR, self.line, getattr(n, 'col_offset', -1))
-            return AnyType(TypeOfAny.from_error)
+        if sys.version_info >= (3, 9):  # Really 3.9a5 or later
+            sliceval = n.slice  # type: Any
+            if (isinstance(sliceval, ast3.Slice) or
+                (isinstance(sliceval, ast3.Tuple) and
+                 any(isinstance(x, ast3.Slice) for x in sliceval.elts))):
+                self.fail(TYPE_COMMENT_SYNTAX_ERROR, self.line, getattr(n, 'col_offset', -1))
+                return AnyType(TypeOfAny.from_error)
+        else:
+            # Python 3.8 or earlier use a different AST structure for subscripts
+            if not isinstance(n.slice, Index):
+                self.fail(TYPE_COMMENT_SYNTAX_ERROR, self.line, getattr(n, 'col_offset', -1))
+                return AnyType(TypeOfAny.from_error)
+            sliceval = n.slice.value
 
         empty_tuple_index = False
-        if isinstance(n.slice.value, ast3.Tuple):
-            params = self.translate_expr_list(n.slice.value.elts)
-            if len(n.slice.value.elts) == 0:
+        if isinstance(sliceval, ast3.Tuple):
+            params = self.translate_expr_list(sliceval.elts)
+            if len(sliceval.elts) == 0:
                 empty_tuple_index = True
         else:
-            params = [self.visit(n.slice.value)]
+            params = [self.visit(sliceval)]
 
         value = self.visit(n.value)
         if isinstance(value, UnboundType) and not value.args:

mypy/test/helpers.py

@@ -233,6 +233,8 @@ def clean_up(a: List[str]) -> List[str]:
     remove trailing carriage returns.
     """
     res = []
+    pwd = os.getcwd()
+    driver = pwd + '/driver.py'
     for s in a:
         prefix = os.sep
         ss = s
@@ -241,6 +243,8 @@ def clean_up(a: List[str]) -> List[str]:
                 ss = ss.replace(p, '')
         # Ignore spaces at end of line.
         ss = re.sub(' +$', '', ss)
+        # Remove pwd from driver.py's path
+        ss = ss.replace(driver, 'driver.py')
         res.append(re.sub('\\r$', '', ss))
     return res
 

mypyc/test-data/run.test

@@ -4352,7 +4352,10 @@ import sys
 
 # We lie about the version we are running in tests if it is 3.5, so
 # that hits a crash case.
-if sys.version_info[:2] == (3, 8):
+if sys.version_info[:2] == (3, 9):
+    def version() -> int:
+        return 9
+elif sys.version_info[:2] == (3, 8):
     def version() -> int:
         return 8
 elif sys.version_info[:2] == (3, 7):