[wip] Improve reachability in narrowing logic, avoid losing narrowing constraints

.github/workflows/mypy_primer.yml

@@ -65,7 +65,7 @@ jobs:
             --new $GITHUB_SHA --old base_commit \
             --num-shards 6 --shard-index ${{ matrix.shard-index }} \
             --debug \
-            --additional-flags="--debug-serialize" \
+            --additional-flags="--debug-serialize --warn-unreachable" \
             --output concise \
             --mypy-install-librt \
             | tee diff_${{ matrix.shard-index }}.txt

mypy/checker.py

@@ -6714,7 +6714,8 @@ def narrow_type_by_identity_equality(
                 target = TypeRange(target_type, is_upper_bound=False)
 
                 if_map, else_map = conditional_types_to_typemaps(
-                    operands[i], *conditional_types(expr_type, [target])
+                    operands[i],
+                    *conditional_types(expr_type, [target], consider_promotion_overlap=True),
                 )
                 if is_target_for_value_narrowing(get_proper_type(target_type)):
                     all_if_maps.append(if_map)
@@ -6725,8 +6726,7 @@ def narrow_type_by_identity_equality(
                     # However, for non-value targets, we cannot do this narrowing,
                     # and so we ignore else_map
                     # e.g. if (x: str | None) != (y: str), we cannot narrow x to None
-                    if if_map is not None:  # TODO: this gate is incorrect and should be removed
-                        all_if_maps.append(if_map)
+                    all_if_maps.append(if_map)
 
         # Handle narrowing for operands with custom __eq__ methods specially
         # In most cases, we won't be able to do any narrowing
@@ -6750,7 +6750,10 @@ def narrow_type_by_identity_equality(
                     target = TypeRange(target_type, is_upper_bound=False)
                     if is_target_for_value_narrowing(get_proper_type(target_type)):
                         if_map, else_map = conditional_types_to_typemaps(
-                            operands[i], *conditional_types(expr_type, [target])
+                            operands[i],
+                            *conditional_types(
+                                expr_type, [target], consider_promotion_overlap=True
+                            ),
                         )
                         if else_map:
                             all_else_maps.append(else_map)
@@ -6777,7 +6780,10 @@ def narrow_type_by_identity_equality(
                     target = TypeRange(target_type, is_upper_bound=False)
 
                     if_map, else_map = conditional_types_to_typemaps(
-                        operands[i], *conditional_types(expr_type, [target], default=expr_type)
+                        operands[i],
+                        *conditional_types(
+                            expr_type, [target], default=expr_type, consider_promotion_overlap=True
+                        ),
                     )
                     or_if_maps.append(if_map)
                     if is_target_for_value_narrowing(get_proper_type(target_type)):
@@ -8242,6 +8248,7 @@ def conditional_types(
     default: None = None,
     *,
     consider_runtime_isinstance: bool = True,
+    consider_promotion_overlap: bool = False,
 ) -> tuple[Type | None, Type | None]: ...
 
 
@@ -8252,6 +8259,7 @@ def conditional_types(
     default: Type,
     *,
     consider_runtime_isinstance: bool = True,
+    consider_promotion_overlap: bool = False,
 ) -> tuple[Type, Type]: ...
 
 
@@ -8261,6 +8269,7 @@ def conditional_types(
     default: Type | None = None,
     *,
     consider_runtime_isinstance: bool = True,
+    consider_promotion_overlap: bool = False,
 ) -> tuple[Type | None, Type | None]:
     """Takes in the current type and a proposed type of an expression.
 
@@ -8300,6 +8309,7 @@ def conditional_types(
                 proposed_type_ranges,
                 default=union_item,
                 consider_runtime_isinstance=consider_runtime_isinstance,
+                consider_promotion_overlap=consider_promotion_overlap,
             )
             for union_item in get_proper_types(proper_type.items)
         ]
@@ -8335,9 +8345,15 @@ def conditional_types(
                 consider_runtime_isinstance=consider_runtime_isinstance,
             )
             return default, remainder
-    if not is_overlapping_types(current_type, proposed_type, ignore_promotions=True):
+    if not is_overlapping_types(
+        current_type, proposed_type, ignore_promotions=not consider_promotion_overlap
+    ):
         # Expression is never of any type in proposed_type_ranges
         return UninhabitedType(), default
+    if consider_promotion_overlap and not is_overlapping_types(
+        current_type, proposed_type, ignore_promotions=True
+    ):
+        return default, default
     # we can only restrict when the type is precise, not bounded
     proposed_precise_type = UnionType.make_union(
         [type_range.item for type_range in proposed_type_ranges if not type_range.is_upper_bound]

mypy/server/aststrip.py

@@ -137,7 +137,9 @@ def visit_class_def(self, node: ClassDef) -> None:
         node.base_type_exprs.extend(node.removed_base_type_exprs)
         node.removed_base_type_exprs = []
         node.defs.body = [
-            s for s in node.defs.body if s not in to_delete  # type: ignore[comparison-overlap]
+            s
+            for s in node.defs.body
+            if s not in to_delete  # type: ignore[comparison-overlap, redundant-expr]
         ]
         with self.enter_class(node.info):
             super().visit_class_def(node)

mypy/stubutil.py

@@ -31,7 +31,7 @@
 )
 
 # Modules that may fail when imported, or that may have side effects (fully qualified).
-NOT_IMPORTABLE_MODULES = ()
+NOT_IMPORTABLE_MODULES: tuple[str, ...] = ()
 
 # Typing constructs to be replaced by their builtin equivalents.
 TYPING_BUILTIN_REPLACEMENTS: Final = {

mypy/typeshed/stubs/mypy-extensions/mypy_extensions.pyi

@@ -111,6 +111,7 @@ class i64:
     def __ge__(self, x: i64) -> bool: ...
     def __gt__(self, x: i64) -> bool: ...
     def __index__(self) -> int: ...
+    def __eq__(self, x: object) -> bool: ...
 
 class i32:
     @overload
@@ -146,6 +147,7 @@ class i32:
     def __ge__(self, x: i32) -> bool: ...
     def __gt__(self, x: i32) -> bool: ...
     def __index__(self) -> int: ...
+    def __eq__(self, x: object) -> bool: ...
 
 class i16:
     @overload
@@ -181,6 +183,7 @@ class i16:
     def __ge__(self, x: i16) -> bool: ...
     def __gt__(self, x: i16) -> bool: ...
     def __index__(self) -> int: ...
+    def __eq__(self, x: object) -> bool: ...
 
 class u8:
     @overload
@@ -216,3 +219,4 @@ class u8:
     def __ge__(self, x: u8) -> bool: ...
     def __gt__(self, x: u8) -> bool: ...
     def __index__(self) -> int: ...
+    def __eq__(self, x: object) -> bool: ...

test-data/unit/check-isinstance.test

@@ -1995,17 +1995,17 @@ else:
 [out]
 
 [case testNarrowTypeAfterInListNonOverlapping]
+# flags: --warn-unreachable
 from typing import List, Optional
 
 x: List[str]
 y: Optional[int]
 
 if y in x:
-    reveal_type(y)  # N: Revealed type is "builtins.int | None"
+    reveal_type(y)  # E: Statement is unreachable
 else:
     reveal_type(y)  # N: Revealed type is "builtins.int | None"
 [builtins fixtures/list.pyi]
-[out]
 
 [case testNarrowTypeAfterInListNested]
 # flags: --warn-unreachable

test-data/unit/check-narrowing.test

@@ -2984,7 +2984,7 @@ def narrow_tuple(x: Literal['c'], overlap: list[Literal['b', 'c']], no_overlap:
         reveal_type(x)  # N: Revealed type is "Literal['c']"
 
     if x in no_overlap:
-        reveal_type(x)  # N: Revealed type is "Literal['c']"
+        reveal_type(x)  # E: Statement is unreachable
     else:
         reveal_type(x)  # N: Revealed type is "Literal['c']"
 [builtins fixtures/tuple.pyi]
@@ -3025,8 +3025,8 @@ def f2(x: Any) -> None:
 
 def bad_compare_has_key(has_key: bool, key: str, s: tuple[str, ...]) -> None:
     if has_key == key in s:  # E: Non-overlapping equality check (left operand type: "bool", right operand type: "str")
-        reveal_type(has_key)  # N: Revealed type is "builtins.bool"
-        reveal_type(key)  # N: Revealed type is "builtins.str"
+        reveal_type(has_key)  # E: Statement is unreachable
+        reveal_type(key)
 
 def bad_but_should_pass(has_key: bool, key: bool, s: tuple[bool, ...]) -> None:
     if has_key == key in s:
@@ -3340,6 +3340,69 @@ def bar(y: Any):
         reveal_type(y) # N: Revealed type is "Any"
 [builtins fixtures/dict-full.pyi]
 
+
+[case testNarrowingConstrainedTypeVarType]
+# flags: --strict-equality --warn-unreachable
+from typing import TypeVar, Any, Type
+
+TargetType = TypeVar("TargetType", int, float, str)
+
+def convert_type(target_type: Type[TargetType]) -> TargetType:
+    if target_type == str:
+        return str()
+    if target_type == int:
+        return int()
+    if target_type == float:
+        return float()  # E: Incompatible return value type (got "float", expected "int")
+    raise
+[builtins fixtures/primitives.pyi]
+
+
+[case testNarrowingEqualityWithPromotions]
+# flags: --strict-equality --warn-unreachable
+from __future__ import annotations
+from typing import Literal
+
+def f1(number: float, i: int):
+    if number == i:
+        reveal_type(number)  # N: Revealed type is "builtins.float"
+        reveal_type(i)  # N: Revealed type is "builtins.int"
+
+def f2(number: float, five: Literal[5]):
+    if number == five:
+        reveal_type(number)  # N: Revealed type is "builtins.float"
+        reveal_type(five)  # N: Revealed type is "Literal[5]"
+
+def f3(number: float | int, five: Literal[5]):
+    if number == five:
+        reveal_type(number)  # N: Revealed type is "builtins.float | Literal[5]"
+        reveal_type(five)  # N: Revealed type is "Literal[5]"
+
+def f4(number: float | None, i: int):
+    if number == i:
+        reveal_type(number)  # N: Revealed type is "builtins.float"
+        reveal_type(i)  # N: Revealed type is "builtins.int"
+
+def f5(number: float | int, i: int):
+    if number == i:
+        reveal_type(number)  # N: Revealed type is "builtins.float | builtins.int"
+        reveal_type(i)  # N: Revealed type is "builtins.int"
+
+def f6(number: float | complex, i: int):
+    if number == i:
+        reveal_type(number)  # N: Revealed type is "builtins.float | builtins.complex"
+        reveal_type(i)  # N: Revealed type is "builtins.int"
+
+class Custom:
+    def __eq__(self, other: object) -> bool: return True
+
+def f7(number: float, x: Custom | int):
+    if number == x:
+        reveal_type(number)  # N: Revealed type is "builtins.float"
+        reveal_type(x)  # N: Revealed type is "__main__.Custom | builtins.int"
+[builtins fixtures/primitives.pyi]
+
+
 [case testNarrowTypeVarType]
 # flags: --strict-equality --warn-unreachable
 from typing import TypeVar

test-data/unit/check-optional.test

@@ -493,11 +493,11 @@ from typing import Optional
 
 def main(x: Optional[str]):
     if x == 0:
-        reveal_type(x)  # N: Revealed type is "builtins.str | None"
+        reveal_type(x)  # E: Statement is unreachable
     else:
         reveal_type(x)  # N: Revealed type is "builtins.str | None"
     if x is 0:
-        reveal_type(x)  # N: Revealed type is "builtins.str | None"
+        reveal_type(x)  # E: Statement is unreachable
     else:
         reveal_type(x)  # N: Revealed type is "builtins.str | None"
 [builtins fixtures/ops.pyi]

test-data/unit/check-python310.test

@@ -56,6 +56,7 @@ match m:
 -- Value Pattern --
 
 [case testMatchValuePatternNarrows]
+# flags: --warn-unreachable
 import b
 m: object
 
@@ -66,6 +67,7 @@ match m:
 b: int
 
 [case testMatchValuePatternAlreadyNarrower]
+# flags: --warn-unreachable
 import b
 m: bool
 
@@ -76,27 +78,28 @@ match m:
 b: int
 
 [case testMatchValuePatternIntersect]
+# flags: --warn-unreachable
 import b
 
 class A: ...
 m: A
 
 match m:
     case b.b:
-        reveal_type(m)  # N: Revealed type is "__main__.A"
+        reveal_type(m)  # E: Statement is unreachable
 [file b.py]
 class B: ...
 b: B
 
 [case testMatchValuePatternUnreachable]
-# primitives are needed because otherwise mypy doesn't see that int and str are incompatible
+# flags: --warn-unreachable
 import b
 
 m: int
 
 match m:
     case b.b:
-        reveal_type(m)  # N: Revealed type is "builtins.int"
+        reveal_type(m)  # E: Statement is unreachable
 [file b.py]
 b: str
 [builtins fixtures/primitives.pyi]
@@ -2769,7 +2772,7 @@ def x() -> tuple[Literal["test"]]: ...
 
 match x():
     case (x,) if x == "test":  # E: Incompatible types in capture pattern (pattern captures type "Literal['test']", variable has type "Callable[[], tuple[Literal['test']]]")
-        reveal_type(x)  # N: Revealed type is "def () -> tuple[Literal['test']]"
+        reveal_type(x)  # E: Statement is unreachable
     case foo:
         foo
 

test-data/unit/check-tuples.test

@@ -1533,17 +1533,24 @@ reveal_type(x) # N: Revealed type is "tuple[builtins.int, builtins.int]"
 [builtins fixtures/tuple.pyi]
 
 [case testTupleOverlapDifferentTuples]
+# flags: --warn-unreachable
 from typing import Optional, Tuple
 class A: pass
 class B: pass
 
-possibles: Tuple[int, Tuple[A]]
-x: Optional[Tuple[B]]
+def f1(possibles: Tuple[int, Tuple[A]], x: Optional[Tuple[B]]):
+    if x in possibles:
+        reveal_type(x)  # E: Statement is unreachable
+    else:
+        reveal_type(x) # N: Revealed type is "tuple[__main__.B] | None"
+
+class AA(A): pass
 
-if x in possibles:
-    reveal_type(x) # N: Revealed type is "tuple[__main__.B]"
-else:
-    reveal_type(x) # N: Revealed type is "tuple[__main__.B] | None"
+def f2(possibles: Tuple[int, Tuple[A]], x: Optional[Tuple[AA]]):
+    if x in possibles:
+        reveal_type(x)  # N: Revealed type is "tuple[__main__.AA]"
+    else:
+        reveal_type(x) # N: Revealed type is "tuple[__main__.AA] | None"
 
 [builtins fixtures/tuple.pyi]