Skip to content

Refactor State creation logic #20691

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ilevkivskyi merged 1 commit into from
Jan 29, 2026
Merged

Refactor State creation logic #20691

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
201 changes: 138 additions & 63 deletions mypy/build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2122,7 +2122,7 @@ class State:
"""The state for a module.

The source is only used for the -c command line option; in that
case path is None. Otherwise source is None and path isn't.
case path is None. Otherwise, source is None and path isn't.
"""

manager: BuildManager
Expand All @@ -2136,7 +2136,6 @@ class State:
source_hash: str | None = None # Hash calculated based on the source code
meta_source_hash: str | None = None # Hash of the source given in the meta, if any
meta: CacheMeta | None = None
data: str | None = None
tree: MypyFile | None = None
# We keep both a list and set of dependencies. A set because it makes it efficient to
# prevent duplicates and the list because I am afraid of changing the order of
Expand All @@ -2152,10 +2151,10 @@ class State:
dep_line_map: dict[str, int]

# Map from dependency id to its last observed interface hash
dep_hashes: dict[str, bytes] = {}
dep_hashes: dict[str, bytes]

# List of errors reported for this file last time.
error_lines: list[SerializedError] = []
error_lines: list[SerializedError]

# Parent package, its parent, etc.
ancestors: list[str] | None = None
Expand Down Expand Up @@ -2196,8 +2195,9 @@ class State:
# we use file size as a proxy for complexity.
size_hint: int

def __init__(
self,
@classmethod
def new_state(
cls,
id: str | None,
path: str | None,
source: str | None,
Expand All @@ -2211,29 +2211,26 @@ def __init__(
# process it. With this flag, any changes to external state as well
# as error reporting should be avoided.
temporary: bool = False,
) -> None:
) -> State:
if not temporary:
assert id or path or source is not None, "Neither id, path nor source given"
self.manager = manager
State.order_counter += 1
self.order = State.order_counter
self.caller_line = caller_line
if caller_state:
self.import_context = caller_state.import_context.copy()
self.import_context.append((caller_state.xpath, caller_line))
import_context = caller_state.import_context.copy()
import_context.append((caller_state.xpath, caller_line))
else:
self.import_context = []
self.id = id or "__main__"
self.options = manager.options.clone_for_module(self.id)
self.early_errors = []
self._type_checker = None
import_context = []
id = id or "__main__"
options = manager.options.clone_for_module(id)

ignore_all = False
if not path and source is None:
assert id is not None
try:
path, follow_imports = find_module_and_diagnose(
manager,
id,
self.options,
options,
caller_state,
caller_line,
ancestor_for,
Expand All @@ -2245,47 +2242,72 @@ def __init__(
manager.missing_modules.add(id)
raise
if follow_imports == "silent":
self.ignore_all = True
ignore_all = True
elif path and is_silent_import_module(manager, path) and not root_source:
self.ignore_all = True
self.path = path
if path:
self.abspath = os.path.abspath(path)
self.xpath = path or "<string>"
if path and source is None and self.manager.cache_enabled:
self.meta = find_cache_meta(self.id, path, manager)
ignore_all = True

meta = None
interface_hash = b""
meta_source_hash = None
if path and source is None and manager.cache_enabled:
meta = find_cache_meta(id, path, manager)
# TODO: Get mtime if not cached.
if self.meta is not None:
self.interface_hash = self.meta.interface_hash
self.meta_source_hash = self.meta.hash
if path and source is None and self.manager.fscache.isdir(path):
if meta is not None:
interface_hash = meta.interface_hash
meta_source_hash = meta.hash
if path and source is None and manager.fscache.isdir(path):
source = ""
self.source = source
self.add_ancestors()
self.per_line_checking_time_ns = collections.defaultdict(int)

t0 = time.time()
self.meta = validate_meta(self.meta, self.id, self.path, self.ignore_all, manager)
self.manager.add_stats(validate_meta_time=time.time() - t0)
if self.meta:
meta = validate_meta(meta, id, path, ignore_all, manager)
manager.add_stats(validate_meta_time=time.time() - t0)

if meta:
# Make copies, since we may modify these and want to
# compare them to the originals later.
self.dependencies = list(self.meta.dependencies)
self.dependencies_set = set(self.dependencies)
self.suppressed = list(self.meta.suppressed)
self.suppressed_set = set(self.suppressed)
all_deps = self.dependencies + self.suppressed
assert len(all_deps) == len(self.meta.dep_prios)
self.priorities = {id: pri for id, pri in zip(all_deps, self.meta.dep_prios)}
assert len(all_deps) == len(self.meta.dep_lines)
self.dep_line_map = {id: line for id, line in zip(all_deps, self.meta.dep_lines)}
assert len(self.meta.dep_hashes) == len(self.meta.dependencies)
self.dep_hashes = {
k: v for (k, v) in zip(self.meta.dependencies, self.meta.dep_hashes)
}
dependencies = list(meta.dependencies)
suppressed = list(meta.suppressed)
all_deps = dependencies + suppressed
assert len(all_deps) == len(meta.dep_prios)
priorities = {id: pri for id, pri in zip(all_deps, meta.dep_prios)}
assert len(all_deps) == len(meta.dep_lines)
dep_line_map = {id: line for id, line in zip(all_deps, meta.dep_lines)}
assert len(meta.dep_hashes) == len(meta.dependencies)
dep_hashes = {k: v for (k, v) in zip(meta.dependencies, meta.dep_hashes)}
# Only copy `error_lines` if the module is not silently imported.
self.error_lines = [] if self.ignore_all else self.meta.error_lines
error_lines = [] if ignore_all else meta.error_lines
else:
dependencies = []
suppressed = []
priorities = {}
dep_line_map = {}
dep_hashes = {}
error_lines = []

state = cls(
manager=manager,
order=State.order_counter,
id=id,
path=path,
source=source,
options=options,
ignore_all=ignore_all,
caller_line=caller_line,
import_context=import_context,
meta=meta,
interface_hash=interface_hash,
meta_source_hash=meta_source_hash,
dependencies=dependencies,
suppressed=suppressed,
priorities=priorities,
dep_line_map=dep_line_map,
dep_hashes=dep_hashes,
error_lines=error_lines,
)

if meta:
if temporary:
self.load_tree(temporary=True)
state.load_tree(temporary=True)
if not manager.use_fine_grained_cache():
# Special case: if there were a previously missing package imported here
# and it is not present, then we need to re-calculate dependencies.
Expand All @@ -2296,10 +2318,10 @@ def __init__(
# suppressed dependencies. Therefore, when the package with module is added,
# we need to re-calculate dependencies.
# NOTE: see comment below for why we skip this in fine grained mode.
if exist_added_packages(self.suppressed, manager, self.options):
self.parse_file() # This is safe because the cache is anyway stale.
self.compute_dependencies()
self.size_hint = self.meta.size
if exist_added_packages(suppressed, manager, options):
state.parse_file() # This is safe because the cache is anyway stale.
state.compute_dependencies()
state.size_hint = meta.size
else:
# When doing a fine-grained cache load, pretend we only
# know about modules that have cache information and defer
Expand All @@ -2309,12 +2331,65 @@ def __init__(
raise ModuleNotFound

# Parse the file (and then some) to get the dependencies.
self.parse_file(temporary=temporary)
self.compute_dependencies()
if self.manager.workers:
state.parse_file(temporary=temporary)
state.compute_dependencies()
if manager.workers:
# We don't need parsed trees in coordinator process, we parse only to
# compute dependencies.
self.tree = None
state.tree = None

return state

def __init__(
self,
manager: BuildManager,
order: int,
id: str,
path: str | None,
source: str | None,
options: Options,
ignore_all: bool,
caller_line: int,
import_context: list[tuple[str, int]],
meta: CacheMeta | None,
interface_hash: bytes,
meta_source_hash: str | None,
dependencies: list[str],
suppressed: list[str],
priorities: dict[str, int],
dep_line_map: dict[str, int],
dep_hashes: dict[str, bytes],
error_lines: list[SerializedError],
size_hint: int = 0,
) -> None:
self.manager = manager
self.order = order
self.id = id
self.path = path
if path:
self.abspath = os.path.abspath(path)
self.xpath = path or "<string>"
self.source = source
self.options = options
self.ignore_all = ignore_all
self.caller_line = caller_line
self.import_context = import_context
self.meta = meta
self.interface_hash = interface_hash
self.meta_source_hash = meta_source_hash
self.dependencies = dependencies
self.suppressed = suppressed
self.dependencies_set = set(dependencies)
self.suppressed_set = set(suppressed)
self.priorities = priorities
self.dep_line_map = dep_line_map
self.dep_hashes = dep_hashes
self.error_lines = error_lines
self.per_line_checking_time_ns = collections.defaultdict(int)
self.early_errors = []
self._type_checker = None
self.add_ancestors()
self.size_hint = size_hint

def reload_meta(self) -> None:
"""Force reload of cache meta.
Expand Down Expand Up @@ -3070,7 +3145,7 @@ def in_partial_package(id: str, manager: BuildManager) -> bool:
else:
# Parent is not in build, try quickly if we can find it.
try:
parent_st = State(
parent_st = State.new_state(
id=parent, path=None, source=None, manager=manager, temporary=True
)
except (ModuleNotFound, CompileError):
Expand Down Expand Up @@ -3419,7 +3494,7 @@ def load_graph(
# Seed the graph with the initial root sources.
for bs in sources:
try:
st = State(
st = State.new_state(
id=bs.module,
path=bs.path,
source=bs.text,
Expand Down Expand Up @@ -3497,11 +3572,11 @@ def load_graph(
if dep in st.ancestors:
# TODO: Why not 'if dep not in st.dependencies' ?
# Ancestors don't have import context.
newst = State(
newst = State.new_state(
id=dep, path=None, source=None, manager=manager, ancestor_for=st
)
else:
newst = State(
newst = State.new_state(
id=dep,
path=None,
source=None,
Expand Down
16 changes: 8 additions & 8 deletions mypy/test/testgraph.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,21 +60,21 @@ def _make_manager(self) -> BuildManager:
def test_sorted_components(self) -> None:
manager = self._make_manager()
graph = {
"a": State("a", None, "import b, c", manager),
"d": State("d", None, "pass", manager),
"b": State("b", None, "import c", manager),
"c": State("c", None, "import b, d", manager),
"a": State.new_state("a", None, "import b, c", manager),
"d": State.new_state("d", None, "pass", manager),
"b": State.new_state("b", None, "import c", manager),
"c": State.new_state("c", None, "import b, d", manager),
}
res = [scc.mod_ids for scc in sorted_components(graph)]
assert_equal(res, [{"d"}, {"c", "b"}, {"a"}])

def test_order_ascc(self) -> None:
manager = self._make_manager()
graph = {
"a": State("a", None, "import b, c", manager),
"d": State("d", None, "def f(): import a", manager),
"b": State("b", None, "import c", manager),
"c": State("c", None, "import b, d", manager),
"a": State.new_state("a", None, "import b, c", manager),
"d": State.new_state("d", None, "def f(): import a", manager),
"b": State.new_state("b", None, "import c", manager),
"c": State.new_state("c", None, "import b, d", manager),
}
res = [scc.mod_ids for scc in sorted_components(graph)]
assert_equal(res, [frozenset({"a", "d", "c", "b"})])
Expand Down