From fc29b93d7487a798773f9bd87b98d1c5cadc34c6 Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Tue, 10 Jan 2023 14:37:15 +0100 Subject: [PATCH 1/3] Fix checking multiple assignments based on tuple unpacking involving partially initialised variables (Fixes #12915). This commit introduces two changes: * It converts unions of tuples to tuples of unions during multi-assignment checking when all (original) tuples have the same length. This fixes issue #12915. * It removes the `undefined_rvalue` logic completely. This logic bothered me because it introduced the necessity to make the mentioned conversion twice (which was not always successful due to reasons I do not fully understand). Test new case `testDefinePartiallyInitialisedVariableDuringTupleUnpacking` covers issue #12915. I had to adjust some other test cases. In my opinion, the error messages are now more transparent, consistent, and complete. Removing the `undefined_rvalue` logic also seems to fix one more bug. In the test cases `testInferenceWithTypeVariableTwiceInReturnType` and `testInferenceWithTypeVariableTwiceInReturnTypeAndMultipleVariables`, there were three assignments for which Mypy did not report errors but, in my opinion, should. So, for the current test data set, removing the `undefined_rvalue` logic seems to introduce mere benefits than drawbacks (meaning, different error messages than before). I hope it does not break any Mypy functionalities not covered by the tests. --- mypy/checker.py | 78 +++++++++++---------- test-data/unit/check-inference-context.test | 8 +-- test-data/unit/check-inference.test | 48 +++++++++++++ test-data/unit/check-tuples.test | 3 +- test-data/unit/check-unions.test | 25 ++++--- test-data/unit/check-varargs.test | 32 +++++---- 6 files changed, 127 insertions(+), 67 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 065758cd2be9..3e1a5282a340 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -213,6 +213,7 @@ is_named_instance, is_optional, remove_optional, + remove_trivial, store_argument_type, strip_type, ) @@ -3375,7 +3376,6 @@ def check_multi_assignment( context: Context, infer_lvalue_type: bool = True, rv_type: Type | None = None, - undefined_rvalue: bool = False, ) -> None: """Check the assignment of one rvalue to a number of lvalues.""" @@ -3386,11 +3386,7 @@ def check_multi_assignment( if isinstance(rvalue_type, TypeVarLikeType): rvalue_type = get_proper_type(rvalue_type.upper_bound) - if isinstance(rvalue_type, UnionType): - # If this is an Optional type in non-strict Optional code, unwrap it. - relevant_items = rvalue_type.relevant_items() - if len(relevant_items) == 1: - rvalue_type = get_proper_type(relevant_items[0]) + rvalue_type = self.simplify_union_during_multi_assignment_checks(rvalue_type) if isinstance(rvalue_type, AnyType): for lv in lvalues: @@ -3402,7 +3398,7 @@ def check_multi_assignment( self.check_assignment(lv, temp_node, infer_lvalue_type) elif isinstance(rvalue_type, TupleType): self.check_multi_assignment_from_tuple( - lvalues, rvalue, rvalue_type, context, undefined_rvalue, infer_lvalue_type + lvalues, rvalue, rvalue_type, context, infer_lvalue_type ) elif isinstance(rvalue_type, UnionType): self.check_multi_assignment_from_union( @@ -3415,6 +3411,44 @@ def check_multi_assignment( lvalues, rvalue_type, context, infer_lvalue_type ) + def simplify_union_during_multi_assignment_checks(self, rvalue_type: ProperType) -> ProperType: + """Two simplifications: + + 1. If `rvalue_type` is Optional type in non-strict Optional code, unwrap it. + 2. If `rvalue_type` is a union of tuples and all tuples have the same number of items + (more than one), convert it to a tuple of (simplified) unions. + + The second simplification is because the multi-assignment checks currently do not work + for unions of tuples in combination with partially initialised lvalues (issue 12915). + """ + if not isinstance(rvalue_type, UnionType): + return rvalue_type + + # try unwrapping: + relevant_items = rvalue_type.relevant_items() + if len(relevant_items) == 1: + return get_proper_type(relevant_items[0]) + + # try union to tuple conversion: + if len(rvalue_type.items) < 2: + return rvalue_type + tupletype = rvalue_type.items[0] + tupletype = get_proper_type(tupletype) + if not isinstance(tupletype, TupleType): + return rvalue_type + items = [tupletype.items] + nmb_subitems = len(items[0]) + for tupletype in rvalue_type.items[1:]: + tupletype = get_proper_type(tupletype) + if not isinstance(tupletype, TupleType) or (len(tupletype.items) != nmb_subitems): + return rvalue_type + items.append(tupletype.items) + items_transposed = zip(*items) + return TupleType( + [UnionType.make_union(remove_trivial(subitems)) for subitems in items_transposed], + fallback=self.named_type("builtins.tuple"), + ) + def check_multi_assignment_from_union( self, lvalues: list[Expression], @@ -3448,7 +3482,6 @@ def check_multi_assignment_from_union( context, infer_lvalue_type=infer_lvalue_type, rv_type=item, - undefined_rvalue=True, ) for t, lv in zip(transposed, self.flatten_lvalues(lvalues)): # We can access _type_maps directly since temporary type maps are @@ -3500,7 +3533,6 @@ def check_multi_assignment_from_tuple( rvalue: Expression, rvalue_type: TupleType, context: Context, - undefined_rvalue: bool, infer_lvalue_type: bool = True, ) -> None: if self.check_rvalue_count_in_assignment(lvalues, len(rvalue_type.items), context): @@ -3512,34 +3544,6 @@ def check_multi_assignment_from_tuple( star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None right_lvs = lvalues[star_index + 1 :] - if not undefined_rvalue: - # Infer rvalue again, now in the correct type context. - lvalue_type = self.lvalue_type_for_inference(lvalues, rvalue_type) - reinferred_rvalue_type = get_proper_type( - self.expr_checker.accept(rvalue, lvalue_type) - ) - - if isinstance(reinferred_rvalue_type, UnionType): - # If this is an Optional type in non-strict Optional code, unwrap it. - relevant_items = reinferred_rvalue_type.relevant_items() - if len(relevant_items) == 1: - reinferred_rvalue_type = get_proper_type(relevant_items[0]) - if isinstance(reinferred_rvalue_type, UnionType): - self.check_multi_assignment_from_union( - lvalues, rvalue, reinferred_rvalue_type, context, infer_lvalue_type - ) - return - if isinstance(reinferred_rvalue_type, AnyType): - # We can get Any if the current node is - # deferred. Doing more inference in deferred nodes - # is hard, so give up for now. We can also get - # here if reinferring types above changes the - # inferred return type for an overloaded function - # to be ambiguous. - return - assert isinstance(reinferred_rvalue_type, TupleType) - rvalue_type = reinferred_rvalue_type - left_rv_types, star_rv_types, right_rv_types = self.split_around_star( rvalue_type.items, star_index, len(lvalues) ) diff --git a/test-data/unit/check-inference-context.test b/test-data/unit/check-inference-context.test index f80f93eb2615..266de52f0583 100644 --- a/test-data/unit/check-inference-context.test +++ b/test-data/unit/check-inference-context.test @@ -168,9 +168,8 @@ if int(): ab, ao = f(b) # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") if int(): ao, ab = f(b) # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") - if int(): - ao, ao = f(b) + ao, ao = f(b) # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") if int(): ab, ab = f(b) if int(): @@ -199,11 +198,10 @@ if int(): ao, ab, ab, ab = h(b, b) # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") if int(): ab, ab, ao, ab = h(b, b) # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") - if int(): - ao, ab, ab = f(b, b) + ao, ab, ab = f(b, b) # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") if int(): - ab, ab, ao = g(b, b) + ab, ab, ao = g(b, b) # E: Incompatible types in assignment (expression has type "A[B]", variable has type "A[object]") if int(): ab, ab, ab, ab = h(b, b) diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index 45a833e5210c..8e997d6bf9eb 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -1919,6 +1919,54 @@ class C: a = 42 [out] +[case testDefinePartiallyInitialisedVariableDuringTupleUnpacking] +# flags: --strict-optional +from typing import Tuple, Union + +t1: Union[Tuple[None], Tuple[str]] +x1 = None +x1, = t1 +reveal_type(x1) # N: Revealed type is "Union[None, builtins.str]" + +t2: Union[Tuple[str], Tuple[None]] +x2 = None +x2, = t2 +reveal_type(x2) # N: Revealed type is "Union[builtins.str, None]" + +t3: Union[Tuple[int], Tuple[str]] +x3 = None +x3, = t3 +reveal_type(x3) # N: Revealed type is "Union[builtins.int, builtins.str]" + +def f() -> Union[ + Tuple[None, None, None, int, int, int, int, int, int], + Tuple[None, None, None, int, int, int, str, str, str] + ]: ... +a1 = None +b1 = None +c1 = None +a2: object +b2: object +c2: object +a1, a2, a3, b1, b2, b3, c1, c2, c3 = f() +reveal_type(a1) # N: Revealed type is "None" +reveal_type(a2) # N: Revealed type is "None" +reveal_type(a3) # N: Revealed type is "None" +reveal_type(b1) # N: Revealed type is "builtins.int" +reveal_type(b2) # N: Revealed type is "builtins.int" +reveal_type(b3) # N: Revealed type is "builtins.int" +reveal_type(c1) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(c2) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(c3) # N: Revealed type is "Union[builtins.int, builtins.str]" + +tt: Tuple[Union[Tuple[None], Tuple[str], Tuple[int]]] +z = None +z, = tt[0] +reveal_type(z) # N: Revealed type is "Union[None, builtins.str, builtins.int]" + +[builtins fixtures/tuple.pyi] + + -- More partial type errors -- ------------------------ diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test index cdb27d10fe0c..155fd270ed4b 100644 --- a/test-data/unit/check-tuples.test +++ b/test-data/unit/check-tuples.test @@ -1054,7 +1054,8 @@ def g(x: T) -> Tuple[T, T]: return (x, x) z = 1 -x, y = g(z) # E: Argument 1 to "g" has incompatible type "int"; expected "Tuple[B1, B2]" +x, y = g(z) # E: Incompatible types in assignment (expression has type "int", variable has type "Tuple[A, ...]") \ + # E: Incompatible types in assignment (expression has type "int", variable has type "Tuple[Union[B1, C], Union[B2, C]]") [builtins fixtures/tuple.pyi] [out] diff --git a/test-data/unit/check-unions.test b/test-data/unit/check-unions.test index 4c4fbc32ec3f..80864877d6e2 100644 --- a/test-data/unit/check-unions.test +++ b/test-data/unit/check-unions.test @@ -619,7 +619,7 @@ from typing import Union, Tuple a: Union[Tuple[int, int], Tuple[int, float]] a1: object a2: int -(a1, a2) = a # E: Incompatible types in assignment (expression has type "float", variable has type "int") +(a1, a2) = a # E: Incompatible types in assignment (expression has type "Union[int, float]", variable has type "int") b: Union[Tuple[float, int], Tuple[int, int]] b1: object @@ -746,8 +746,10 @@ from typing import Union, Tuple bad: Union[Tuple[int, int, int], Tuple[str, str, str]] x, y = bad # E: Too many values to unpack (2 expected, 3 provided) -reveal_type(x) # N: Revealed type is "Any" -reveal_type(y) # N: Revealed type is "Any" +reveal_type(x) # E: Cannot determine type of "x" \ + # N: Revealed type is "Any" +reveal_type(y) # E: Cannot determine type of "y" \ + # N: Revealed type is "Any" [builtins fixtures/tuple.pyi] [out] @@ -756,10 +758,14 @@ from typing import Union, Tuple bad: Union[Tuple[int, int, int], Tuple[str, str, str]] x, y, z, w = bad # E: Need more than 3 values to unpack (4 expected) -reveal_type(x) # N: Revealed type is "Any" -reveal_type(y) # N: Revealed type is "Any" -reveal_type(z) # N: Revealed type is "Any" -reveal_type(w) # N: Revealed type is "Any" +reveal_type(x) # E: Cannot determine type of "x" \ + # N: Revealed type is "Any" +reveal_type(y) # E: Cannot determine type of "y" \ + # N: Revealed type is "Any" +reveal_type(z) # E: Cannot determine type of "z" \ + # N: Revealed type is "Any" +reveal_type(w) # E: Cannot determine type of "w" \ + # N: Revealed type is "Any" [builtins fixtures/tuple.pyi] [out] @@ -977,8 +983,9 @@ from typing import Dict, Tuple, List, Any a: Any d: Dict[str, Tuple[List[Tuple[str, str]], str]] -x, _ = d.get(a, ([], [])) -reveal_type(x) # N: Revealed type is "Union[builtins.list[Tuple[builtins.str, builtins.str]], builtins.list[]]" +x, _ = d.get(a, ([], [])) # E: Need type annotation for "x" \ + # E: Need type annotation for "_" +reveal_type(x) # N: Revealed type is "Union[builtins.list[Tuple[builtins.str, builtins.str]], builtins.list[Any]]" for y in x: pass [builtins fixtures/dict.pyi] diff --git a/test-data/unit/check-varargs.test b/test-data/unit/check-varargs.test index 00ac7df320d2..52c8af004d54 100644 --- a/test-data/unit/check-varargs.test +++ b/test-data/unit/check-varargs.test @@ -531,19 +531,19 @@ T = TypeVar('T') a, b, aa = None, None, None # type: (A, B, List[A]) if int(): - a, b = f(*aa) # E: Argument 1 to "f" has incompatible type "*List[A]"; expected "B" + a, b = f(*aa) # E: Incompatible types in assignment (expression has type "A", variable has type "B") if int(): - b, b = f(*aa) # E: Argument 1 to "f" has incompatible type "*List[A]"; expected "B" + b, b = f(*aa) # E: Incompatible types in assignment (expression has type "A", variable has type "B") if int(): - a, a = f(b, *aa) # E: Argument 1 to "f" has incompatible type "B"; expected "A" + a, a = f(b, *aa) # E: Incompatible types in assignment (expression has type "B", variable has type "A") if int(): - b, b = f(b, *aa) # E: Argument 2 to "f" has incompatible type "*List[A]"; expected "B" + b, b = f(b, *aa) # E: Incompatible types in assignment (expression has type "A", variable has type "B") if int(): - b, b = f(b, b, *aa) # E: Argument 3 to "f" has incompatible type "*List[A]"; expected "B" + b, b = f(b, b, *aa) # E: Incompatible types in assignment (expression has type "object", variable has type "B") if int(): - a, b = f(a, *a) # E: List or tuple expected as variadic arguments + a, b = f(a, *a) # E: List or tuple expected as variadic arguments if int(): - a, b = f(*a) # E: List or tuple expected as variadic arguments + a, b = f(*a) # E: List or tuple expected as variadic arguments if int(): a, a = f(*aa) @@ -566,13 +566,13 @@ T = TypeVar('T') a, b = None, None # type: (A, B) if int(): - a, a = f(*(a, b)) # E: Argument 1 to "f" has incompatible type "*Tuple[A, B]"; expected "A" + a, a = f(*(a, b)) # E: Incompatible types in assignment (expression has type "B", variable has type "A") if int(): - b, b = f(a, *(b,)) # E: Argument 1 to "f" has incompatible type "A"; expected "B" + b, b = f(a, *(b,)) # E: Incompatible types in assignment (expression has type "A", variable has type "B") if int(): - a, a = f(*(a, b)) # E: Argument 1 to "f" has incompatible type "*Tuple[A, B]"; expected "A" + a, a = f(*(a, b)) # E: Incompatible types in assignment (expression has type "B", variable has type "A") if int(): - b, b = f(a, *(b,)) # E: Argument 1 to "f" has incompatible type "A"; expected "B" + b, b = f(a, *(b,)) # E: Incompatible types in assignment (expression has type "A", variable has type "B") if int(): a, b = f(*(a, b, b)) # E: Too many arguments for "f" if int(): @@ -606,16 +606,18 @@ if int(): aa, a = G().f(*[a]) # E: Incompatible types in assignment (expression has type "List[]", variable has type "A") if int(): ab, aa = G().f(*[a]) \ + # E: Incompatible types in assignment (expression has type "List[A]", variable has type "List[B]") \ # E: Incompatible types in assignment (expression has type "List[]", variable has type "List[A]") \ # N: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance \ - # N: Consider using "Sequence" instead, which is covariant \ - # E: Argument 1 to "f" of "G" has incompatible type "*List[A]"; expected "B" + # N: Consider using "Sequence" instead, which is covariant if int(): ao, ao = G().f(*[a]) \ - # E: Incompatible types in assignment (expression has type "List[]", variable has type "List[object]") \ + # E: Incompatible types in assignment (expression has type "List[A]", variable has type "List[object]") \ # N: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance \ - # N: Consider using "Sequence" instead, which is covariant + # N: Consider using "Sequence" instead, which is covariant \ + # E: Incompatible types in assignment (expression has type "List[]", variable has type "List[object]") + if int(): aa, aa = G().f(*[a]) \ # E: Incompatible types in assignment (expression has type "List[]", variable has type "List[A]") \ From 4754999e7405addcf6d008974eab26dd95515ee4 Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Tue, 10 Jan 2023 16:03:24 +0100 Subject: [PATCH 2/3] black --- mypy/checker.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 3e1a5282a340..68d62eb2c2ae 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3477,11 +3477,7 @@ def check_multi_assignment_from_union( # Type check the assignment separately for each union item and collect # the inferred lvalue types for each union item. self.check_multi_assignment( - lvalues, - rvalue, - context, - infer_lvalue_type=infer_lvalue_type, - rv_type=item, + lvalues, rvalue, context, infer_lvalue_type=infer_lvalue_type, rv_type=item ) for t, lv in zip(transposed, self.flatten_lvalues(lvalues)): # We can access _type_maps directly since temporary type maps are From 7759837409c66ec55eeafbe07ad06ff699e5e7ef Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Mon, 16 Jan 2023 06:21:59 +0100 Subject: [PATCH 3/3] Remove method `lvalue_type_for_inference` (not used anymore). --- mypy/checker.py | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 68d62eb2c2ae..affbd0576062 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3555,44 +3555,6 @@ def check_multi_assignment_from_tuple( for lv, rv_type in zip(right_lvs, right_rv_types): self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) - def lvalue_type_for_inference(self, lvalues: list[Lvalue], rvalue_type: TupleType) -> Type: - star_index = next( - (i for i, lv in enumerate(lvalues) if isinstance(lv, StarExpr)), len(lvalues) - ) - left_lvs = lvalues[:star_index] - star_lv = cast(StarExpr, lvalues[star_index]) if star_index != len(lvalues) else None - right_lvs = lvalues[star_index + 1 :] - left_rv_types, star_rv_types, right_rv_types = self.split_around_star( - rvalue_type.items, star_index, len(lvalues) - ) - - type_parameters: list[Type] = [] - - def append_types_for_inference(lvs: list[Expression], rv_types: list[Type]) -> None: - for lv, rv_type in zip(lvs, rv_types): - sub_lvalue_type, index_expr, inferred = self.check_lvalue(lv) - if sub_lvalue_type and not isinstance(sub_lvalue_type, PartialType): - type_parameters.append(sub_lvalue_type) - else: # index lvalue - # TODO Figure out more precise type context, probably - # based on the type signature of the _set method. - type_parameters.append(rv_type) - - append_types_for_inference(left_lvs, left_rv_types) - - if star_lv: - sub_lvalue_type, index_expr, inferred = self.check_lvalue(star_lv.expr) - if sub_lvalue_type and not isinstance(sub_lvalue_type, PartialType): - type_parameters.extend([sub_lvalue_type] * len(star_rv_types)) - else: # index lvalue - # TODO Figure out more precise type context, probably - # based on the type signature of the _set method. - type_parameters.extend(star_rv_types) - - append_types_for_inference(right_lvs, right_rv_types) - - return TupleType(type_parameters, self.named_type("builtins.tuple")) - def split_around_star( self, items: list[T], star_index: int, length: int ) -> tuple[list[T], list[T], list[T]]: