diff --git a/mypy/checker.py b/mypy/checker.py index fa531daba798..7d0b5dbde09d 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -130,6 +130,7 @@ RefExpr, ReturnStmt, SetExpr, + SplittingVisitor, StarExpr, Statement, StrExpr, @@ -319,7 +320,7 @@ def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> Literal return False -class TypeChecker(NodeVisitor[None], TypeCheckerSharedApi): +class TypeChecker(NodeVisitor[None], TypeCheckerSharedApi, SplittingVisitor): """Mypy type checker. Type check mypy source files that have been semantically analyzed. @@ -454,9 +455,6 @@ def __init__( or self.path in self.msg.errors.ignored_files or (self.options.test_env and self.is_typeshed_stub) ) - - # If True, process function definitions. If False, don't. This is used - # for processing module top levels in fine-grained incremental mode. self.recurse_into_functions = True # This internal flag is used to track whether we a currently type-checking # a final declaration (assignment), so that some errors should be suppressed. @@ -719,23 +717,10 @@ def accept_loop( # Definitions # - @contextmanager - def set_recurse_into_functions(self) -> Iterator[None]: - """Temporarily set recurse_into_functions to True. - - This is used to process top-level functions/methods as a whole. - """ - old_recurse_into_functions = self.recurse_into_functions - self.recurse_into_functions = True - try: - yield - finally: - self.recurse_into_functions = old_recurse_into_functions - def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: # If a function/method can infer variable types, it should be processed as part # of the module top level (i.e. module interface). - if not self.recurse_into_functions and not defn.can_infer_vars: + if not self.recurse_into_functions and not defn.def_or_infer_vars: return with self.tscope.function_scope(defn), self.set_recurse_into_functions(): self._visit_overloaded_func_def(defn) @@ -1211,7 +1196,7 @@ def get_generator_return_type(self, return_type: Type, is_coroutine: bool) -> Ty return NoneType() def visit_func_def(self, defn: FuncDef) -> None: - if not self.recurse_into_functions and not defn.can_infer_vars: + if not self.recurse_into_functions and not defn.def_or_infer_vars: return with self.tscope.function_scope(defn), self.set_recurse_into_functions(): self.check_func_item(defn, name=defn.name) @@ -1452,8 +1437,7 @@ def check_func_def( not self.can_skip_diagnostics or self.options.preserve_asts or not isinstance(defn, FuncDef) - or defn.has_self_attr_def - or defn.can_infer_vars + or defn.def_or_infer_vars ): self.accept(item.body) unreachable = self.binder.is_unreachable() @@ -5620,7 +5604,7 @@ def visit_decorator(self, e: Decorator) -> None: def visit_decorator_inner( self, e: Decorator, allow_empty: bool = False, skip_first_item: bool = False ) -> None: - if self.recurse_into_functions or e.func.can_infer_vars: + if self.recurse_into_functions or e.func.def_or_infer_vars: with self.tscope.function_scope(e.func), self.set_recurse_into_functions(): self.check_func_item(e.func, name=e.func.name, allow_empty=allow_empty) diff --git a/mypy/nodes.py b/mypy/nodes.py index 589da3d240fb..37ea4d3b0d56 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -6,6 +6,7 @@ from abc import abstractmethod from collections import defaultdict from collections.abc import Callable, Iterator, Sequence +from contextlib import contextmanager from enum import Enum, unique from typing import ( TYPE_CHECKING, @@ -659,7 +660,7 @@ class FuncBase(Node): "is_final", # Uses "@final" "is_explicit_override", # Uses "@override" "is_type_check_only", # Uses "@type_check_only" - "can_infer_vars", + "def_or_infer_vars", "_fullname", ) @@ -680,8 +681,8 @@ def __init__(self) -> None: self.is_final = False self.is_explicit_override = False self.is_type_check_only = False - # Can this function/method infer types of variables defined outside? Currently, - # we only set this in cases like: + # Can this function/method define variables or infer variables defined outside? + # In particular, we set this in cases like: # x = None # def foo() -> None: # global x @@ -691,7 +692,7 @@ def __init__(self) -> None: # x = None # def foo(self) -> None: # self.x = 1 - self.can_infer_vars = False + self.def_or_infer_vars = False # Name with module prefix self._fullname = "" @@ -1035,7 +1036,6 @@ class FuncDef(FuncItem, SymbolNode, Statement): "original_def", "is_trivial_body", "is_trivial_self", - "has_self_attr_def", "is_mypy_only", # Present only when a function is decorated with @typing.dataclass_transform or similar "dataclass_transform_spec", @@ -1074,8 +1074,6 @@ def __init__( # the majority). In cases where self is not annotated and there are no Self # in the signature we can simply drop the first argument. self.is_trivial_self = False - # Keep track of functions where self attributes are defined. - self.has_self_attr_def = False # This is needed because for positional-only arguments the name is set to None, # but we sometimes still want to show it in error messages. if arguments: @@ -5089,6 +5087,26 @@ def read(cls, data: ReadBuffer) -> DataclassTransformSpec: return ret +@trait +class SplittingVisitor: + # If True, process function definitions. If False, don't. This is used + # for processing module top levels in fine-grained incremental mode. + recurse_into_functions: bool + + @contextmanager + def set_recurse_into_functions(self) -> Iterator[None]: + """Temporarily set recurse_into_functions to True. + + This is used to process top-level functions/methods as a whole. + """ + old_recurse_into_functions = self.recurse_into_functions + self.recurse_into_functions = True + try: + yield + finally: + self.recurse_into_functions = old_recurse_into_functions + + def get_flags(node: Node, names: list[str]) -> list[str]: return [name for name in names if getattr(node, name)] diff --git a/mypy/semanal.py b/mypy/semanal.py index bf21e057345f..aa64d21b8e7d 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -159,6 +159,7 @@ SetComprehension, SetExpr, SliceExpr, + SplittingVisitor, StarExpr, Statement, StrExpr, @@ -372,7 +373,7 @@ class SemanticAnalyzer( - NodeVisitor[None], SemanticAnalyzerInterface, SemanticAnalyzerPluginInterface + NodeVisitor[None], SemanticAnalyzerInterface, SemanticAnalyzerPluginInterface, SplittingVisitor ): """Semantically analyze parsed mypy files. @@ -497,8 +498,6 @@ def __init__( self.incomplete_namespaces = incomplete_namespaces self.all_exports: list[str] = [] self.plugin = plugin - # If True, process function definitions. If False, don't. This is used - # for processing module top levels in fine-grained incremental mode. self.recurse_into_functions = True self.scope = Scope() @@ -981,10 +980,10 @@ def visit_func_def(self, defn: FuncDef) -> None: if not defn.is_decorated and not defn.is_overload: self.add_function_to_symbol_table(defn) - if not self.recurse_into_functions: + if not self.recurse_into_functions and not defn.def_or_infer_vars: return - with self.scope.function_scope(defn): + with self.scope.function_scope(defn), self.set_recurse_into_functions(): with self.inside_except_star_block_set(value=False): self.analyze_func_def(defn) @@ -1272,14 +1271,14 @@ def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: self.statement = defn self.add_function_to_symbol_table(defn) - if not self.recurse_into_functions: + if not self.recurse_into_functions and not defn.def_or_infer_vars: return # NB: Since _visit_overloaded_func_def will call accept on the # underlying FuncDefs, the function might get entered twice. # This is fine, though, because only the outermost function is # used to compute targets. - with self.scope.function_scope(defn): + with self.scope.function_scope(defn), self.set_recurse_into_functions(): self.analyze_overloaded_func_def(defn) @contextmanager @@ -1809,8 +1808,9 @@ def visit_decorator(self, dec: Decorator) -> None: dec.var.is_initialized_in_class = True if no_type_check: erase_func_annotations(dec.func) - if not no_type_check and self.recurse_into_functions: - dec.func.accept(self) + if not no_type_check and (self.recurse_into_functions or dec.func.def_or_infer_vars): + with self.set_recurse_into_functions(): + dec.func.accept(self) if could_be_decorated_property and dec.decorators and dec.var.is_property: self.fail( "Decorators on top of @property are not supported", dec, code=PROPERTY_DECORATOR @@ -4597,7 +4597,7 @@ def make_name_lvalue_point_to_existing_def( and original_def.node.is_inferred ): for func in self.scope.functions: - func.can_infer_vars = True + func.def_or_infer_vars = True def analyze_tuple_or_list_lvalue(self, lval: TupleExpr, explicit_type: bool = False) -> None: """Analyze an lvalue or assignment target that is a list or tuple.""" @@ -4682,8 +4682,7 @@ def analyze_member_lvalue( # TODO: should we also set lval.kind = MDEF? self.type.names[lval.name] = SymbolTableNode(MDEF, v, implicit=True) for func in self.scope.functions: - if isinstance(func, FuncDef): - func.has_self_attr_def = True + func.def_or_infer_vars = True if ( cur_node and isinstance(cur_node.node, Var) @@ -4691,7 +4690,7 @@ def analyze_member_lvalue( and cur_node.node.is_initialized_in_class ): for func in self.scope.functions: - func.can_infer_vars = True + func.def_or_infer_vars = True self.check_lvalue_validity(lval.node, lval) def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: @@ -7561,7 +7560,7 @@ def already_defined( if isinstance(original_ctx, SymbolTableNode) and isinstance(original_ctx.node, MypyFile): # Since this is an import, original_ctx.node points to the module definition. - # Therefore its line number is always 1, which is not useful for this + # Therefore, its line number is always 1, which is not useful for this # error message. extra_msg = " (by an import)" elif node and node.line != -1 and self.is_local_name(node.fullname): diff --git a/mypy/semanal_main.py b/mypy/semanal_main.py index 6c2f51b39eb1..edc6ee4143f2 100644 --- a/mypy/semanal_main.py +++ b/mypy/semanal_main.py @@ -34,7 +34,7 @@ import mypy.state from mypy.checker import FineGrainedDeferredNode from mypy.errors import Errors -from mypy.nodes import Decorator, FuncDef, MypyFile, OverloadedFuncDef, TypeInfo, Var +from mypy.nodes import Decorator, FuncDef, MypyFile, OverloadedFuncDef, TypeInfo from mypy.options import Options from mypy.plugin import ClassDefContext from mypy.plugins import dataclasses as dataclasses_plugin @@ -52,7 +52,6 @@ from mypy.semanal_infer import infer_decorator_signature_if_simple from mypy.semanal_shared import find_dataclass_transform_spec from mypy.semanal_typeargs import TypeArgumentAnalyzer -from mypy.server.aststrip import SavedAttributes if TYPE_CHECKING: from mypy.build import Graph, State @@ -129,23 +128,18 @@ def cleanup_builtin_scc(state: State) -> None: def semantic_analysis_for_targets( - state: State, nodes: list[FineGrainedDeferredNode], graph: Graph, saved_attrs: SavedAttributes + state: State, nodes: list[FineGrainedDeferredNode], graph: Graph ) -> None: """Semantically analyze only selected nodes in a given module. This essentially mirrors the logic of semantic_analysis_for_scc() - except that we process only some targets. This is used in fine grained + except that we process only some targets. This is used in fine-grained incremental mode, when propagating an update. - - The saved_attrs are implicitly declared instance attributes (attributes - defined on self) removed by AST stripper that may need to be reintroduced - here. They must be added before any methods are analyzed. """ patches: Patches = [] if any(isinstance(n.node, MypyFile) for n in nodes): # Process module top level first (if needed). process_top_levels(graph, [state.id], patches) - restore_saved_attrs(saved_attrs) analyzer = state.manager.semantic_analyzer for n in nodes: if isinstance(n.node, MypyFile): @@ -160,30 +154,6 @@ def semantic_analysis_for_targets( calculate_class_properties(graph, [state.id], state.manager.errors) -def restore_saved_attrs(saved_attrs: SavedAttributes) -> None: - """Restore instance variables removed during AST strip that haven't been added yet.""" - for (cdef, name), sym in saved_attrs.items(): - info = cdef.info - existing = info.get(name) - defined_in_this_class = name in info.names - assert isinstance(sym.node, Var) - # This needs to mimic the logic in SemanticAnalyzer.analyze_member_lvalue() - # regarding the existing variable in class body or in a superclass: - # If the attribute of self is not defined in superclasses, create a new Var. - if ( - existing is None - or - # (An abstract Var is considered as not defined.) - (isinstance(existing.node, Var) and existing.node.is_abstract_var) - or - # Also an explicit declaration on self creates a new Var unless - # there is already one defined in the class body. - sym.node.explicit_self_type - and not defined_in_this_class - ): - info.names[name] = sym - - def process_top_levels(graph: Graph, scc: list[str], patches: Patches) -> None: # Process top levels until everything has been bound. @@ -240,6 +210,9 @@ def process_top_levels(graph: Graph, scc: list[str], patches: Patches) -> None: # processing the same target twice in a row, which is inefficient. worklist = list(reversed(all_deferred)) final_iteration = not any_progress + # Functions/methods that define/infer attributes are processed as part of top-levels. + # We need to clear the locals for those between fine-grained iterations. + analyzer.saved_locals.clear() def order_by_subclassing(targets: list[FullTargetInfo]) -> Iterator[FullTargetInfo]: diff --git a/mypy/server/aststrip.py b/mypy/server/aststrip.py index 828e51895f0f..dd6a106e22bf 100644 --- a/mypy/server/aststrip.py +++ b/mypy/server/aststrip.py @@ -35,7 +35,6 @@ from collections.abc import Iterator from contextlib import contextmanager, nullcontext -from typing import TypeAlias as _TypeAlias from mypy.nodes import ( CLASSDEF_NO_INFO, @@ -57,54 +56,37 @@ OpExpr, OverloadedFuncDef, RefExpr, + SplittingVisitor, StarExpr, SuperExpr, - SymbolTableNode, TupleExpr, TypeInfo, - Var, ) from mypy.traverser import TraverserVisitor from mypy.types import CallableType from mypy.typestate import type_state -SavedAttributes: _TypeAlias = dict[tuple[ClassDef, str], SymbolTableNode] - -def strip_target( - node: MypyFile | FuncDef | OverloadedFuncDef, saved_attrs: SavedAttributes -) -> None: +def strip_target(node: MypyFile | FuncDef | OverloadedFuncDef) -> None: """Reset a fine-grained incremental target to state before semantic analysis. - All TypeInfos are killed. Therefore we need to preserve the variables - defined as attributes on self. This is done by patches (callbacks) - returned from this function that re-add these variables when called. - Args: node: node to strip - saved_attrs: collect attributes here that may need to be re-added to - classes afterwards if stripping a class body (this dict is mutated) """ - visitor = NodeStripVisitor(saved_attrs) + visitor = NodeStripVisitor() if isinstance(node, MypyFile): visitor.strip_file_top_level(node) else: node.accept(visitor) -class NodeStripVisitor(TraverserVisitor): - def __init__(self, saved_class_attrs: SavedAttributes) -> None: +class NodeStripVisitor(TraverserVisitor, SplittingVisitor): + def __init__(self) -> None: # The current active class. self.type: TypeInfo | None = None # This is True at class scope, but not in methods. self.is_class_body = False - # By default, process function definitions. If False, don't -- this is used for - # processing module top levels. self.recurse_into_functions = True - # These attributes were removed from top-level classes during strip and - # will be added afterwards (if no existing definition is found). These - # must be added back before semantically analyzing any methods. - self.saved_class_attrs = saved_class_attrs def strip_file_top_level(self, file_node: MypyFile) -> None: """Strip a module top-level (don't recursive into functions).""" @@ -124,12 +106,6 @@ def visit_block(self, b: Block) -> None: def visit_class_def(self, node: ClassDef) -> None: """Strip class body and type info, but don't strip methods.""" - # We need to save the implicitly defined instance variables, - # i.e. those defined as attributes on self. Otherwise, they would - # be lost if we only reprocess top-levels (this kills TypeInfos) - # but not the methods that defined those variables. - if not self.recurse_into_functions: - self.save_implicit_attributes(node) # We need to delete any entries that were generated by plugins, # since they will get regenerated. to_delete = {v.node for v in node.info.names.values() if v.plugin_generated} @@ -150,14 +126,8 @@ def visit_class_def(self, node: ClassDef) -> None: node.info = CLASSDEF_NO_INFO node.analyzed = None - def save_implicit_attributes(self, node: ClassDef) -> None: - """Produce callbacks that re-add attributes defined on self.""" - for name, sym in node.info.names.items(): - if isinstance(sym.node, Var) and sym.implicit: - self.saved_class_attrs[node, name] = sym - def visit_func_def(self, node: FuncDef) -> None: - if not self.recurse_into_functions: + if not self.recurse_into_functions and not node.def_or_infer_vars: return node.expanded = [] node.type = node.unanalyzed_type @@ -168,15 +138,19 @@ def visit_func_def(self, node: FuncDef) -> None: # See also #4814. assert isinstance(node.type, CallableType) node.type.variables = () - with self.enter_method(node.info) if node.info else nullcontext(): + with ( + self.enter_method(node.info) if node.info else nullcontext(), + self.set_recurse_into_functions(), + ): super().visit_func_def(node) def visit_decorator(self, node: Decorator) -> None: node.var.type = None for expr in node.decorators: expr.accept(self) - if self.recurse_into_functions: - node.func.accept(self) + if self.recurse_into_functions or node.func.def_or_infer_vars: + with self.set_recurse_into_functions(): + node.func.accept(self) else: # Only touch the final status if we re-process # the top level, since decorators are processed there. @@ -184,13 +158,14 @@ def visit_decorator(self, node: Decorator) -> None: node.func.is_final = False def visit_overloaded_func_def(self, node: OverloadedFuncDef) -> None: - if not self.recurse_into_functions: + if not self.recurse_into_functions and not node.def_or_infer_vars: return # Revert change made during semantic analysis main pass. node.items = node.unanalyzed_items.copy() node.impl = None node.is_final = False - super().visit_overloaded_func_def(node) + with self.set_recurse_into_functions(): + super().visit_overloaded_func_def(node) def visit_assignment_stmt(self, node: AssignmentStmt) -> None: node.type = node.unanalyzed_type @@ -253,9 +228,6 @@ def process_lvalue_in_method(self, lvalue: Node) -> None: assert self.type is not None if lvalue.name in self.type.names: del self.type.names[lvalue.name] - key = (self.type.defn, lvalue.name) - if key in self.saved_class_attrs: - del self.saved_class_attrs[key] elif isinstance(lvalue, (TupleExpr, ListExpr)): for item in lvalue.items: self.process_lvalue_in_method(item) diff --git a/mypy/server/update.py b/mypy/server/update.py index 3c7a8d7f820e..64ce0fb5d3f8 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -155,7 +155,7 @@ snapshot_symbol_table, ) from mypy.server.astmerge import merge_asts -from mypy.server.aststrip import SavedAttributes, strip_target +from mypy.server.aststrip import strip_target from mypy.server.deps import get_dependencies_of_target, merge_dependencies from mypy.server.target import trigger_to_target from mypy.server.trigger import WILDCARD_TAG, make_trigger @@ -873,7 +873,7 @@ def propagate_changes_using_dependencies( if id not in todo: todo[id] = set() manager.log_fine_grained(f"process target with error: {target}") - more_nodes, _ = lookup_target(manager, target) + more_nodes, _ = lookup_target(manager, target, id) todo[id].update(more_nodes) triggered = set() # First invalidate subtype caches in all stale protocols. @@ -951,23 +951,9 @@ def find_targets_recursive( if module_id not in result: result[module_id] = set() manager.log_fine_grained(f"process: {target}") - deferred, stale_proto = lookup_target(manager, target) + deferred, stale_proto = lookup_target(manager, target, module_id) if stale_proto: stale_protos.add(stale_proto) - - # If there are function targets that can infer outer variables, they should - # be re-processed as part of the module top-level instead (for consistency). - regular = [] - shared = [] - for d in deferred: - if isinstance(d.node, FuncBase) and d.node.can_infer_vars: - shared.append(d) - else: - regular.append(d) - deferred = regular - if shared: - deferred.append(FineGrainedDeferredNode(manager.modules[module_id], None)) - result[module_id].update(deferred) return result, unloaded_files, stale_protos @@ -1024,11 +1010,10 @@ def key(node: FineGrainedDeferredNode) -> int: manager.errors.add_error_info(info, file=graph[module_id].xpath) # Strip semantic analysis information. - saved_attrs: SavedAttributes = {} for deferred in nodes: processed_targets.append(deferred.node.fullname) - strip_target(deferred.node, saved_attrs) - semantic_analysis_for_targets(graph[module_id], nodes, graph, saved_attrs) + strip_target(deferred.node) + semantic_analysis_for_targets(graph[module_id], nodes, graph) # Merge symbol tables to preserve identities of AST nodes. The file node will remain # the same, but other nodes may have been recreated with different identities, such as # NamedTuples defined using assignment statements. @@ -1112,7 +1097,7 @@ def update_deps( def lookup_target( - manager: BuildManager, target: str + manager: BuildManager, target: str, module_id: str ) -> tuple[list[FineGrainedDeferredNode], TypeInfo | None]: """Look up a target by fully-qualified name. @@ -1120,7 +1105,26 @@ def lookup_target( needs to be reprocessed. If the target represents a TypeInfo corresponding to a protocol, return it as a second item in the return tuple, otherwise None. """ + deferred, stale_proto = _lookup_target_impl(manager, target) + + # If there are function targets that can infer outer variables, they should + # be re-processed as part of the module top-level instead (for consistency). + regular = [] + shared = [] + for d in deferred: + if isinstance(d.node, FuncBase) and d.node.def_or_infer_vars: + shared.append(d) + else: + regular.append(d) + deferred = regular + if shared: + deferred.append(FineGrainedDeferredNode(manager.modules[module_id], None)) + return deferred, stale_proto + +def _lookup_target_impl( + manager: BuildManager, target: str +) -> tuple[list[FineGrainedDeferredNode], TypeInfo | None]: def not_found() -> None: manager.log_fine_grained(f"Can't find matching target for {target} (stale dependency?)") @@ -1170,7 +1174,7 @@ def not_found() -> None: for name, symnode in node.names.items(): node = symnode.node if isinstance(node, FuncDef): - method, _ = lookup_target(manager, target + "." + name) + method, _ = _lookup_target_impl(manager, target + "." + name) result.extend(method) return result, stale_info if isinstance(node, Decorator):