Add even more detailed stats

Author: ilevkivskyi

mypy/build.py

@@ -1256,7 +1256,10 @@ def wait_for_done_workers(
 
         done_sccs = []
         results = {}
-        for idx in ready_to_read([w.conn for w in self.workers], WORKER_DONE_TIMEOUT):
+        t0 = time.time()
+        ready = ready_to_read([w.conn for w in self.workers], WORKER_DONE_TIMEOUT)
+        t1 = time.time()
+        for idx in ready:
             buf = receive(self.workers[idx].conn)
             assert read_tag(buf) == SCC_RESPONSE_MESSAGE
             data = SccResponseMessage.read(buf)
@@ -1267,6 +1270,7 @@ def wait_for_done_workers(
             assert data.result is not None
             results.update(data.result)
             done_sccs.append(self.scc_by_id[scc_id])
+        self.add_stats(scc_wait_time=t1 - t0, scc_receive_time=time.time() - t1)
         self.submit_to_workers(graph)  # advance after some workers are free.
         return (
             done_sccs,

mypy/build_worker/worker.py

@@ -45,10 +45,10 @@
     process_stale_scc,
 )
 from mypy.cache import Tag, read_int_opt
-from mypy.defaults import RECURSION_LIMIT, WORKER_CONNECTION_TIMEOUT
+from mypy.defaults import RECURSION_LIMIT, WORKER_CONNECTION_TIMEOUT, WORKER_IDLE_TIMEOUT
 from mypy.errors import CompileError, ErrorInfo, Errors, report_internal_error
 from mypy.fscache import FileSystemCache
-from mypy.ipc import IPCException, IPCServer, receive, send
+from mypy.ipc import IPCException, IPCServer, ready_to_read, receive, send
 from mypy.modulefinder import BuildSource, BuildSourceSet, compute_search_paths
 from mypy.nodes import FileRawData
 from mypy.options import Options
@@ -170,9 +170,13 @@ def serve(server: IPCServer, ctx: ServerContext) -> None:
     # Notify coordinator we are ready to start processing SCCs.
     send(server, AckMessage())
     while True:
+        t0 = time.time()
+        ready_to_read([server], WORKER_IDLE_TIMEOUT)
+        t1 = time.time()
         buf = receive(server)
         assert read_tag(buf) == SCC_REQUEST_MESSAGE
         scc_message = SccRequestMessage.read(buf)
+        manager.add_stats(scc_wait_time=t1 - t0, scc_receive_time=time.time() - t1)
         scc_id = scc_message.scc_id
         if scc_id is None:
             manager.dump_stats()

mypy/defaults.py

@@ -48,4 +48,5 @@
 WORKER_START_INTERVAL: Final = 0.01
 WORKER_START_TIMEOUT: Final = 3
 WORKER_CONNECTION_TIMEOUT: Final = 10
+WORKER_IDLE_TIMEOUT: Final = 600
 WORKER_DONE_TIMEOUT: Final = 600

mypy/ipc.py

@@ -16,7 +16,7 @@
 from collections.abc import Callable
 from select import select
 from types import TracebackType
-from typing import Final
+from typing import Final, Sequence
 from typing_extensions import Self
 
 from librt.base64 import urlsafe_b64encode
@@ -365,7 +365,7 @@ def read_status(status_file: str) -> dict[str, object]:
     return data
 
 
-def ready_to_read(conns: list[IPCClient], timeout: float | None = None) -> list[int]:
+def ready_to_read(conns: Sequence[IPCBase], timeout: float | None = None) -> list[int]:
     """Wait until some connections are readable.
 
     Return index of each readable connection in the original list.