pythongh-150114: Log the memory usage in regrtest (python#150255)

On Linux, log the total memory usage of all Python test processes.
Read the private memory in /proc/pid/smaps.

Author: vstinner

Lib/test/libregrtest/logger.py

@@ -1,5 +1,6 @@
 import os
 import time
+from typing import Callable
 
 from test.support import MS_WINDOWS
 from .results import TestResults
@@ -19,16 +20,27 @@ def __init__(self, results: TestResults, quiet: bool, pgo: bool):
         self._results: TestResults = results
         self._quiet: bool = quiet
         self._pgo: bool = pgo
+        self.get_mem_usage: Callable[[], int | None] | None = None
 
     def log(self, line: str = '') -> None:
         empty = not line
 
-        # add the system load prefix: "load avg: 1.80 "
+        # Add the memory usage: "mem: 1 GiB "
+        if self.get_mem_usage is not None:
+            mem = self.get_mem_usage()
+            if mem:
+                mib = mem / (1024*1024)
+                if mib >= 1024:
+                    line = f"mem: {mib / 1024:.1f} GiB {line}"
+                else:
+                    line = f"mem: {mib:.1f} MiB {line}"
+
+        # Add the system load prefix: "load avg: 1.80 "
         load_avg = self.get_load_avg()
         if load_avg is not None:
             line = f"load avg: {load_avg:.2f} {line}"
 
-        # add the timestamp prefix:  "0:01:05 "
+        # Add the timestamp prefix:  "0:01:05 "
         log_time = time.perf_counter() - self.start_time
 
         mins, secs = divmod(int(log_time), 60)

Lib/test/libregrtest/run_workers.py

@@ -22,7 +22,7 @@
 from .single import PROGRESS_MIN_TIME
 from .utils import (
     StrPath, TestName,
-    format_duration, print_warning, count, plural)
+    format_duration, print_warning, count, plural, get_process_memory_usage)
 from .worker import create_worker_process, USE_PROCESS_GROUP
 
 if MS_WINDOWS:
@@ -452,6 +452,12 @@ def wait_stopped(self, start_time: float) -> None:
                 print_warning(f"Failed to join {self} in {format_duration(dt)}")
                 break
 
+    def get_mem_usage(self):
+        popen = self._popen
+        if popen is None:
+            return
+        return get_process_memory_usage(popen.pid)
+
 
 def get_running(workers: list[WorkerThread]) -> str | None:
     running: list[str] = []
@@ -473,6 +479,7 @@ def __init__(self, num_workers: int, runtests: RunTests,
                  logger: Logger, results: TestResults) -> None:
         self.num_workers = num_workers
         self.runtests = runtests
+        self.logger = logger
         self.log = logger.log
         self.display_progress = logger.display_progress
         self.results: TestResults = results
@@ -598,9 +605,21 @@ def _process_result(self, item: QueueOutput) -> TestResult:
 
         return result
 
+    def get_mem_usage(self):
+        usage = 0
+        main_mem = get_process_memory_usage(os.getpid())
+        if main_mem:
+            usage += main_mem
+        for worker in self.workers:
+            worker_mem = worker.get_mem_usage()
+            if worker_mem:
+                usage += worker_mem
+        return usage
+
     def run(self) -> None:
         fail_fast = self.runtests.fail_fast
         fail_env_changed = self.runtests.fail_env_changed
+        self.logger.get_mem_usage = self.get_mem_usage
 
         self.start_workers()
 
@@ -625,3 +644,4 @@ def run(self) -> None:
             # worker when we exit this function
             self.pending.stop()
             self.stop_workers()
+            self.logger.get_mem_usage = None

Lib/test/libregrtest/utils.py

@@ -752,3 +752,26 @@ def display_title(title):
     print(title)
     print("#" * len(title))
     print(flush=True)
+
+
+def get_process_memory_usage(pid: int) -> int | None:
+    """
+    Read the private memory in bytes from /proc/pid/smaps.
+    """
+    try:
+        fp = open(f"/proc/{pid}/smaps", "rb")
+    except OSError:
+        return None
+
+    try:
+        total = 0
+        with fp:
+            for line in fp:
+                # Include both Private_Clean and Private_Dirty sections.
+                line = line.rstrip()
+                if line.startswith(b"Private_") and line.endswith(b'kB'):
+                    parts = line.split()
+                    total += int(parts[1]) * 1024
+        return total
+    except ProcessLookupError:
+        return None

Lib/test/test_regrtest.py

@@ -41,7 +41,11 @@
 
 ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..')
 ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR))
-LOG_PREFIX = r'[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?'
+LOG_PREFIX = (
+    r'[0-9]+:[0-9]+:[0-9]+ '
+    r'(?:load avg: [0-9]+\.[0-9]{2} )?'
+    r'(?:mem: [0-9]+\.[0-9] (?:MiB|GiB) )?'
+)
 RESULT_REGEX = (
     'passed',
     'failed',

Misc/NEWS.d/next/Tests/2026-05-22-16-41-45.gh-issue-150114.UdMikH.rst

@@ -0,0 +1,2 @@
+On Linux, regrtest now logs the total memory usage of all Python processes.
+Read the private memory in ``/proc/pid/smaps``. Patch by Victor Stinner.