Skip to content

Commit 28751e9

Browse files
Optimize SCC scheduler not_ready_deps tracking (#21389)
The `not_ready_deps` was populated with each direct dependency ID to do the two operations below: - mark a dependency as completed by discarding it - check if all dependences are done based on if it's empty All of these could be done with a counter. The ids are never actually being used/inspected, just the size. So with a counter, we can just populate a counter with `len(dep_sccs)`, and rather than discarding an id we can just decrement the counter, and we can check if the counter is zero for when all dependencies have been completed. This is roughly a ~2% speed up on a warm run. --------- Co-authored-by: Ivan Levkivskyi <levkivskyi@gmail.com>
1 parent 5e7e91b commit 28751e9

1 file changed

Lines changed: 12 additions & 10 deletions

File tree

mypy/build.py

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -230,10 +230,10 @@ def __init__(
230230
self.mod_ids = ids
231231
# Direct dependencies, should be populated by the caller.
232232
self.deps: set[int] = set(deps) if deps is not None else set()
233-
# Direct dependencies that have not been processed yet.
234-
# Should be populated by the caller. This set may change during graph
235-
# processing, while the above stays constant.
236-
self.not_ready_deps: set[int] = set()
233+
# Count of direct dependencies that have not been processed yet.
234+
# Populated by the caller from len(deps); decremented during graph
235+
# processing as each dep completes. self.deps above stays constant.
236+
self.not_ready_count: int = 0
237237
# SCCs that (directly) depend on this SCC. Note this is a list to
238238
# make processing order more predictable. Dependents will be notified
239239
# that they may be ready in the order in this list.
@@ -4619,10 +4619,11 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
46194619
ready = []
46204620
for done_scc in done:
46214621
for dependent in done_scc.direct_dependents:
4622-
scc_by_id[dependent].not_ready_deps.discard(done_scc.id)
4623-
if not scc_by_id[dependent].not_ready_deps:
4624-
not_ready.remove(scc_by_id[dependent])
4625-
ready.append(scc_by_id[dependent])
4622+
dep_scc = scc_by_id[dependent]
4623+
dep_scc.not_ready_count -= 1
4624+
if not dep_scc.not_ready_count:
4625+
not_ready.remove(dep_scc)
4626+
ready.append(dep_scc)
46264627
manager.trace(f"Transitive deps cache size: {sys.getsizeof(manager.transitive_deps_cache)}")
46274628

46284629

@@ -5003,9 +5004,10 @@ def prepare_sccs_full(
50035004
for scc in sccs:
50045005
# Remove trivial dependency on itself.
50055006
scc_deps_map[scc].discard(scc)
5006-
for dep_scc in scc_deps_map[scc]:
5007+
dep_sccs = scc_deps_map[scc]
5008+
for dep_scc in dep_sccs:
50075009
scc.deps.add(dep_scc.id)
5008-
scc.not_ready_deps.add(dep_scc.id)
5010+
scc.not_ready_count = len(dep_sccs)
50095011
return scc_deps_map
50105012

50115013

0 commit comments

Comments
 (0)