From e04a4b41333d4011f17e927ebcecc0340b37e7a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Perrin?= Date: Thu, 10 Mar 2022 10:26:00 +0000 Subject: [PATCH 1/2] Test for undefined FuncDef.arguments after deserialize from cache --- test-data/unit/check-modules.test | 32 +++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test-data/unit/check-modules.test b/test-data/unit/check-modules.test index 6c32c088255d..4f7b9f322df1 100644 --- a/test-data/unit/check-modules.test +++ b/test-data/unit/check-modules.test @@ -3190,3 +3190,35 @@ from dir1 import * from .test2 import * [file dir1/test2.py] from test1 import aaaa # E: Module "test1" has no attribute "aaaa" + +[case testIncompatibleOverrideFromCachedModuleIncremental] +import b +[file a.py] +class Foo: + def frobnicate(self, *args, **kwargs): pass +[file b.py] +from a import Foo +class Bar(Foo): + def frobnicate(self) -> None: pass +[file b.py.2] +from a import Foo +class Bar(Foo): + def frobnicate(self, *args) -> None: pass +[file b.py.3] +from a import Foo +class Bar(Foo): + def frobnicate(self, *args) -> None: pass # type: ignore[override] # I know +[builtins fixtures/tuple.pyi] +[builtins fixtures/dict.pyi] +[out1] +tmp/b.py:3: error: Signature of "frobnicate" incompatible with supertype "Foo" +tmp/b.py:3: note: Superclass: +tmp/b.py:3: note: def frobnicate(self, *args: Any, **kwargs: Any) -> Any +tmp/b.py:3: note: Subclass: +tmp/b.py:3: note: def frobnicate(self) -> None +[out2] +tmp/b.py:3: error: Signature of "frobnicate" incompatible with supertype "Foo" +tmp/b.py:3: note: Superclass: +tmp/b.py:3: note: def frobnicate(self, *args: Any, **kwargs: Any) -> Any +tmp/b.py:3: note: Subclass: +tmp/b.py:3: note: def frobnicate(self, *args: Any) -> None From 370a035cbf56f7d04681c50261c353f7dec91b30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Perrin?= Date: Thu, 10 Mar 2022 10:28:16 +0000 Subject: [PATCH 2/2] Make FuncItem.arguments optional When FuncDef is deserialized we don't reconstruct the arguments. Previous code was deleting the attribute, leading to #11899; instead, always set the attribute, maybe to None, and mypy can remind us to check for not None before use. --- mypy/checker.py | 2 ++ mypy/checkexpr.py | 1 + mypy/messages.py | 6 ++++-- mypy/nodes.py | 9 +++++---- mypy/renaming.py | 2 ++ mypy/semanal.py | 5 +++++ mypy/strconv.py | 2 ++ mypy/stubgen.py | 1 + mypy/stubtest.py | 4 ++++ mypy/suggestions.py | 4 ++++ mypy/treetransform.py | 3 +++ mypy/types.py | 17 +++++++---------- mypyc/irbuild/builder.py | 1 + mypyc/irbuild/env_class.py | 1 + mypyc/irbuild/function.py | 2 ++ mypyc/irbuild/mapper.py | 3 ++- 16 files changed, 46 insertions(+), 17 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index de33f6259297..760edd60bda9 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1005,6 +1005,7 @@ def check_func_def(self, defn: FuncItem, typ: CallableType, name: Optional[str]) arg_type = self.named_generic_type('builtins.dict', [self.str_type(), arg_type]) + assert item.arguments is not None item.arguments[i].variable.type = arg_type # Type check initialization expressions. @@ -1051,6 +1052,7 @@ def check_func_def(self, defn: FuncItem, typ: CallableType, name: Optional[str]) self.binder = old_binder def check_default_args(self, item: FuncItem, body_is_trivial: bool) -> None: + assert item.arguments is not None for arg in item.arguments: if arg.initializer is None: continue diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 4ece9ac4c94b..6d752d366206 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -3618,6 +3618,7 @@ def infer_lambda_type_using_context(self, e: LambdaExpr) -> Tuple[Optional[Calla # See https://github.com/python/mypy/issues/9927 return None, None + assert e.arguments is not None arg_kinds = [arg.kind for arg in e.arguments] if callable_ctx.is_ellipsis_args or ctx.param_spec() is not None: diff --git a/mypy/messages.py b/mypy/messages.py index f067e7e06bd4..f71f24e25b4e 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1950,10 +1950,12 @@ def [T <: int] f(self, x: int, y: T) -> None s += ' = ...' # If we got a "special arg" (i.e: self, cls, etc...), prepend it to the arg list - if isinstance(tp.definition, FuncDef) and tp.definition.name is not None: + if isinstance(tp.definition, FuncDef) and \ + tp.definition.name is not None and \ + tp.definition.arguments is not None: definition_args = [arg.variable.name for arg in tp.definition.arguments] if definition_args and tp.arg_names != definition_args \ - and len(definition_args) > 0 and definition_args[0]: + and len(definition_args) > 0 and definition_args[0]: if s: s = ', ' + s s = definition_args[0] + s diff --git a/mypy/nodes.py b/mypy/nodes.py index 7680641e659f..742743ff17b3 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -642,7 +642,7 @@ def set_line(self, class FuncItem(FuncBase): """Base class for nodes usable as overloaded function items.""" - __slots__ = ('arguments', # Note that can be None if deserialized (type is a lie!) + __slots__ = ('arguments', # Note that can be None if deserialized 'arg_names', # Names of arguments 'arg_kinds', # Kinds of arguments 'min_args', # Minimum number of arguments @@ -658,14 +658,14 @@ class FuncItem(FuncBase): 'expanded', # Variants of function with type variables with values expanded ) - __deletable__ = ('arguments', 'max_pos', 'min_args') + __deletable__ = ('max_pos', 'min_args') def __init__(self, arguments: List[Argument], body: 'Block', typ: 'Optional[mypy.types.FunctionLike]' = None) -> None: super().__init__() - self.arguments = arguments + self.arguments: Optional[List[Argument]] = arguments self.arg_names = [None if arg.pos_only else arg.variable.name for arg in arguments] self.arg_kinds: List[ArgKind] = [arg.kind for arg in self.arguments] self.max_pos: int = ( @@ -693,6 +693,7 @@ def set_line(self, column: Optional[int] = None, end_line: Optional[int] = None) -> None: super().set_line(target, column, end_line) + assert self.arguments is not None for arg in self.arguments: arg.set_line(self.line, self.column, self.end_line) @@ -772,7 +773,7 @@ def deserialize(cls, data: JsonDict) -> 'FuncDef': ret.arg_names = data['arg_names'] ret.arg_kinds = [ArgKind(x) for x in data['arg_kinds']] # Leave these uninitialized so that future uses will trigger an error - del ret.arguments + ret.arguments = None del ret.max_pos del ret.min_args return ret diff --git a/mypy/renaming.py b/mypy/renaming.py index ae21631f0f0a..d9d425ce1647 100644 --- a/mypy/renaming.py +++ b/mypy/renaming.py @@ -86,6 +86,7 @@ def visit_func_def(self, fdef: FuncDef) -> None: self.reject_redefinition_of_vars_in_scope() with self.enter_scope(FUNCTION), self.enter_block(): + assert fdef.arguments is not None for arg in fdef.arguments: name = arg.variable.name # 'self' can't be redefined since it's special as it allows definition of @@ -442,6 +443,7 @@ def visit_mypy_file(self, file_node: MypyFile) -> None: def visit_func_def(self, fdef: FuncDef) -> None: self.reject_redefinition_of_vars_in_scope() with self.enter_scope(): + assert fdef.arguments is not None for arg in fdef.arguments: self.record_skipped(arg.variable.name) super().visit_func_def(fdef) diff --git a/mypy/semanal.py b/mypy/semanal.py index 60127961b60e..9e4945f6c478 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -607,6 +607,7 @@ def visit_func_def(self, defn: FuncDef) -> None: self.statement = defn # Visit default values because they may contain assignment expressions. + assert defn.arguments is not None for arg in defn.arguments: if arg.initializer: arg.initializer.accept(self) @@ -980,6 +981,7 @@ def add_function_to_symbol_table(self, func: Union[FuncDef, OverloadedFuncDef]) def analyze_arg_initializers(self, defn: FuncItem) -> None: with self.tvar_scope_frame(self.tvar_scope.method_frame()): # Analyze default arguments + assert defn.arguments is not None for arg in defn.arguments: if arg.initializer: arg.initializer.accept(self) @@ -993,6 +995,7 @@ def analyze_function_body(self, defn: FuncItem) -> None: a.bind_function_type_variables(cast(CallableType, defn.type), defn) self.function_stack.append(defn) with self.enter(defn): + assert defn.arguments is not None for arg in defn.arguments: self.add_local(arg.variable, defn) @@ -1021,6 +1024,7 @@ def check_classvar_in_signature(self, typ: ProperType) -> None: def check_function_signature(self, fdef: FuncItem) -> None: sig = fdef.type assert isinstance(sig, CallableType) + assert fdef.arguments is not None if len(sig.arg_types) < len(fdef.arguments): self.fail('Type signature has too few arguments', fdef) # Add dummy Any arguments to prevent crashes later. @@ -1073,6 +1077,7 @@ def visit_decorator(self, dec: Decorator) -> None: elif refers_to_fullname(d, 'functools.cached_property'): dec.var.is_settable_property = True self.check_decorated_function_is_method('property', dec) + assert dec.func.arguments is not None if len(dec.func.arguments) > 1: self.fail('Too many arguments', dec.func) elif refers_to_fullname(d, 'typing.no_type_check'): diff --git a/mypy/strconv.py b/mypy/strconv.py index 22534a44971d..7a1f356aa2d6 100644 --- a/mypy/strconv.py +++ b/mypy/strconv.py @@ -64,6 +64,7 @@ def func_helper(self, o: 'mypy.nodes.FuncItem') -> List[object]: """ args: List[Union[mypy.nodes.Var, Tuple[str, List[mypy.nodes.Node]]]] = [] extra: List[Tuple[str, List[mypy.nodes.Var]]] = [] + assert o.arguments is not None for arg in o.arguments: kind: mypy.nodes.ArgKind = arg.kind if kind.is_required(): @@ -131,6 +132,7 @@ def visit_import_all(self, o: 'mypy.nodes.ImportAll') -> str: def visit_func_def(self, o: 'mypy.nodes.FuncDef') -> str: a = self.func_helper(o) a.insert(0, o.name) + assert o.arguments is not None arg_kinds = {arg.kind for arg in o.arguments} if len(arg_kinds & {mypy.nodes.ARG_NAMED, mypy.nodes.ARG_NAMED_OPT}) > 0: a.insert(1, 'MaxPos({})'.format(o.max_pos)) diff --git a/mypy/stubgen.py b/mypy/stubgen.py index 6db5aa75d102..91c343780db6 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -633,6 +633,7 @@ def visit_func_def(self, o: FuncDef, is_abstract: bool = False, self.add("%s%sdef %s(" % (self._indent, 'async ' if o.is_coroutine else '', o.name)) self.record_name(o.name) args: List[str] = [] + assert o.arguments is not None for i, arg_ in enumerate(o.arguments): var = arg_.variable kind = arg_.kind diff --git a/mypy/stubtest.py b/mypy/stubtest.py index ae9c71801dfb..aa55c76e58a8 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -495,6 +495,7 @@ def get_desc(arg: Any) -> str: @staticmethod def from_funcitem(stub: nodes.FuncItem) -> "Signature[nodes.Argument]": stub_sig: Signature[nodes.Argument] = Signature() + assert stub.arguments is not None stub_args = maybe_strip_cls(stub.name, stub.arguments) for stub_arg in stub_args: if stub_arg.kind.is_positional(): @@ -544,6 +545,7 @@ def from_overloadedfuncdef(stub: nodes.OverloadedFuncDef) -> "Signature[nodes.Ar all_args: Dict[str, List[Tuple[nodes.Argument, int]]] = {} for func in map(_resolve_funcitem_from_decorator, stub.items): assert func is not None + assert func.arguments is not None args = maybe_strip_cls(stub.name, func.arguments) for index, arg in enumerate(args): # For positional-only args, we allow overloads to have different names for the same @@ -916,9 +918,11 @@ def apply_decorator_to_funcitem( ): return func if decorator.fullname == "builtins.classmethod": + assert func.arguments is not None assert func.arguments[0].variable.name in ("cls", "metacls") ret = copy.copy(func) # Remove the cls argument, since it's not present in inspect.signature of classmethods + assert ret.arguments ret.arguments = ret.arguments[1:] return ret # Just give up on any other decorators. After excluding properties, we don't run into diff --git a/mypy/suggestions.py b/mypy/suggestions.py index af42cca8a216..e27ef6b14178 100644 --- a/mypy/suggestions.py +++ b/mypy/suggestions.py @@ -144,6 +144,7 @@ class ArgUseFinder(TraverserVisitor): """ def __init__(self, func: FuncDef, typemap: Dict[Expression, Type]) -> None: self.typemap = typemap + assert func.arguments is not None self.arg_types: Dict[SymbolNode, List[Type]] = {arg.variable: [] for arg in func.arguments} def visit_call_expr(self, o: CallExpr) -> None: @@ -179,6 +180,7 @@ def test(x, y): """ finder = ArgUseFinder(func, typemap) func.body.accept(finder) + assert func.arguments is not None return [finder.arg_types[arg.variable] for arg in func.arguments] @@ -345,6 +347,7 @@ def get_args(self, is_method: bool, return types def get_default_arg_types(self, fdef: FuncDef) -> List[Optional[Type]]: + assert fdef.arguments is not None return [ self.manager.all_types[arg.initializer] if arg.initializer else None for arg in fdef.arguments @@ -418,6 +421,7 @@ def get_guesses_from_parent(self, node: FuncDef) -> List[CallableType]: if pnode and isinstance(pnode.node, (FuncDef, Decorator)): typ = get_proper_type(pnode.node.type) # FIXME: Doesn't work right with generic tyeps + assert node.arguments is not None if isinstance(typ, CallableType) and len(typ.arg_types) == len(node.arguments): # Return the first thing we find, since it probably doesn't make sense # to grab things further up in the chain if an earlier parent has it. diff --git a/mypy/treetransform.py b/mypy/treetransform.py index cdd4f604be86..11c543b89e1c 100644 --- a/mypy/treetransform.py +++ b/mypy/treetransform.py @@ -110,6 +110,7 @@ def visit_func_def(self, node: FuncDef) -> FuncDef: for stmt in node.body.body: stmt.accept(init) + assert node.arguments is not None new = FuncDef(node.name, [self.copy_argument(arg) for arg in node.arguments], self.block(node.body), @@ -139,6 +140,7 @@ def visit_func_def(self, node: FuncDef) -> FuncDef: return new def visit_lambda_expr(self, node: LambdaExpr) -> LambdaExpr: + assert node.arguments is not None new = LambdaExpr([self.copy_argument(arg) for arg in node.arguments], self.block(node.body), cast(Optional[FunctionLike], self.optional_type(node.type))) @@ -630,6 +632,7 @@ def __init__(self, transformer: TransformVisitor) -> None: def visit_func_def(self, node: FuncDef) -> None: if node not in self.transformer.func_placeholder_map: # Haven't seen this FuncDef before, so create a placeholder node. + assert node.arguments is not None self.transformer.func_placeholder_map[node] = FuncDef( node.name, node.arguments, node.body, None) super().visit_func_def(node) diff --git a/mypy/types.py b/mypy/types.py index 4d56ee9359af..e2465b4f221f 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1262,16 +1262,13 @@ def __init__(self, # after serialization, but it is useful in error messages. # TODO: decide how to add more info here (file, line, column) # without changing interface hash. - self.def_extras = { - 'first_arg': ( - definition.arguments[0].variable.name - if (getattr(definition, 'arguments', None) - and definition.arg_names - and definition.info - and not definition.is_static) - else None - ), - } + first_arg: Optional[str] = None + if definition.info and not definition.is_static: + if definition.arguments: + first_arg = definition.arguments[0].variable.name + elif definition.arg_names: + first_arg = definition.arg_names[0] + self.def_extras = {'first_arg': first_arg} else: self.def_extras = {} self.type_guard = type_guard diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 57baa8dbf574..c03579d65f9f 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -1184,6 +1184,7 @@ def gen_arg_defaults(builder: IRBuilder) -> None: value to the argument. """ fitem = builder.fn_info.fitem + assert fitem.arguments is not None for arg in fitem.arguments: if arg.initializer: target = builder.lookup(arg.variable) diff --git a/mypyc/irbuild/env_class.py b/mypyc/irbuild/env_class.py index 44bcccb507d0..dc2f14d3680a 100644 --- a/mypyc/irbuild/env_class.py +++ b/mypyc/irbuild/env_class.py @@ -160,6 +160,7 @@ def add_args_to_env(builder: IRBuilder, base: Optional[Union[FuncInfo, ImplicitClass]] = None, reassign: bool = True) -> None: fn_info = builder.fn_info + assert fn_info.fitem.arguments is not None if local: for arg in fn_info.fitem.arguments: rtype = builder.type_to_rtype(arg.variable.type) diff --git a/mypyc/irbuild/function.py b/mypyc/irbuild/function.py index 5b567251111a..8751b5d6206d 100644 --- a/mypyc/irbuild/function.py +++ b/mypyc/irbuild/function.py @@ -122,6 +122,7 @@ def transform_lambda_expr(builder: IRBuilder, expr: LambdaExpr) -> Value: assert isinstance(typ, CallableType) runtime_args = [] + assert expr.arguments is not None for arg, arg_type in zip(expr.arguments, typ.arg_types): arg.variable.type = arg_type runtime_args.append( @@ -467,6 +468,7 @@ def calculate_arg_defaults(builder: IRBuilder, still stored computed on demand). """ fitem = fn_info.fitem + assert fitem.arguments is not None for arg in fitem.arguments: # Constant values don't get stored but just recomputed if arg.initializer and not is_constant(arg.initializer): diff --git a/mypyc/irbuild/mapper.py b/mypyc/irbuild/mapper.py index 901ea49fc2fa..6b3a9e89bc4a 100644 --- a/mypyc/irbuild/mapper.py +++ b/mypyc/irbuild/mapper.py @@ -128,6 +128,7 @@ def fdef_to_sig(self, fdef: FuncDef) -> FuncSignature: ret = self.type_to_rtype(fdef.type.ret_type) else: # Handle unannotated functions + assert fdef.arguments is not None arg_types = [object_rprimitive for arg in fdef.arguments] arg_pos_onlys = [arg.pos_only for arg in fdef.arguments] # We at least know the return type for __init__ methods will be None. @@ -145,7 +146,7 @@ def fdef_to_sig(self, fdef: FuncDef) -> FuncSignature: # deserialized FuncDef that lacks arguments. We won't ever # need to use those inside of a FuncIR, so we just make up # some crap. - if hasattr(fdef, 'arguments'): + if fdef.arguments is not None: arg_names = [arg.variable.name for arg in fdef.arguments] else: arg_names = [name or '' for name in fdef.arg_names]