Keep track of test suite. Change formatting.

Author: mdboom

ci/tools/report_universally_skipped_tests.py

@@ -30,6 +30,8 @@
     "test-windows": r"^Test (win-64|windows) / ",
 }
 
+INDEX_FILENAME = "job_index.json"
+
 ANSI_ESCAPE = re.compile(r"\x1B\[[0-9;]*[A-Za-z]")
 PYTEST_NODE_ID = re.compile(r"tests/\S+\.py::\S+")
 PYTEST_TEST_OUTCOME = re.compile(r"(tests/\S+\.py::\S+)\s+(PASSED|FAILED|ERROR|SKIPPED|XFAIL|XPASS)\b")
@@ -41,13 +43,17 @@ class ConfigResult:
     job_ids: list[int]
     skipped: set[str]
     has_logs: bool
+    # test_id -> suite name (e.g. "cuda_bindings"), empty string if unknown
+    test_suites: dict[str, str] = dataclasses.field(default_factory=dict)
 
 
 @dataclasses.dataclass(frozen=True)
 class ConfigLogs:
     name: str
     job_ids: list[int]
     log_paths: list[Path]
+    # job_id -> suite name extracted from the job name
+    job_names: dict[int, str] = dataclasses.field(default_factory=dict)
 
 
 def run_gh(*args: str, check: bool = True) -> subprocess.CompletedProcess[str]:
@@ -122,42 +128,88 @@ def extract_test_status_sets(text: str) -> tuple[set[str], set[str]]:
     return skipped, non_skipped
 
 
+def extract_suite_name(job_name: str, config_name: str) -> str:
+    """Return the test suite portion of a job name (first word after the config prefix)."""
+    pattern = CONFIG_PATTERNS.get(config_name, "")
+    if pattern:
+        match = re.match(pattern, job_name)
+        if match:
+            remainder = job_name[match.end() :]
+            parts = remainder.split()
+            return parts[0] if parts else job_name
+    return job_name
+
+
+def save_job_index(logs_root: Path, index: dict[str, dict[str, str]]) -> None:
+    (logs_root / INDEX_FILENAME).write_text(json.dumps(index, indent=2), encoding="utf-8")
+
+
+def load_job_index(logs_root: Path) -> dict[str, dict[str, str]]:
+    index_path = logs_root / INDEX_FILENAME
+    if index_path.exists():
+        return json.loads(index_path.read_text(encoding="utf-8"))
+    return {}
+
+
 def match_job_ids(jobs: Iterable[dict], pattern: str) -> list[int]:
     regex = re.compile(pattern)
     return [int(job["id"]) for job in jobs if regex.search(str(job.get("name", "")))]
 
 
 def discover_config_logs(logs_root: Path) -> list[ConfigLogs]:
     configs: list[ConfigLogs] = []
+    index = load_job_index(logs_root)
 
     for config in CONFIG_PATTERNS:
         config_dir = logs_root / config
         log_paths = sorted(config_dir.glob("*.log")) if config_dir.exists() else []
         job_ids: list[int] = []
+        job_names: dict[int, str] = {}
+        config_index = index.get(config, {})
+
         for log_path in log_paths:
             with contextlib.suppress(ValueError):
-                job_ids.append(int(log_path.stem))
-        configs.append(ConfigLogs(name=config, job_ids=job_ids, log_paths=log_paths))
+                job_id = int(log_path.stem)
+                job_ids.append(job_id)
+                suite = config_index.get(str(job_id), "")
+                if suite:
+                    job_names[job_id] = suite
+
+        configs.append(ConfigLogs(name=config, job_ids=job_ids, log_paths=log_paths, job_names=job_names))
 
     return configs
 
 
 def download_config_logs(jobs: list[dict], repo: str, run_id: str, logs_root: Path) -> list[ConfigLogs]:
     configs: list[ConfigLogs] = []
+    index: dict[str, dict[str, str]] = {}
 
     for config, pattern in CONFIG_PATTERNS.items():
         config_dir = logs_root / config
         job_ids = match_job_ids(jobs, pattern)
         log_paths: list[Path] = []
 
+        # Build job_id -> suite_name from job metadata before downloading logs.
+        regex = re.compile(pattern)
+        job_names: dict[int, str] = {}
+        for job in jobs:
+            job_name = str(job.get("name", ""))
+            if not regex.search(job_name):
+                continue
+            job_id = int(job["id"])
+            if job_id in job_ids:
+                job_names[job_id] = extract_suite_name(job_name, config)
+
         for job_id in job_ids:
             log_path = config_dir / f"{job_id}.log"
             if not log_path.exists() and not download_job_log(repo, run_id, job_id, log_path):
                 continue
             log_paths.append(log_path)
 
-        configs.append(ConfigLogs(name=config, job_ids=job_ids, log_paths=log_paths))
+        configs.append(ConfigLogs(name=config, job_ids=job_ids, log_paths=log_paths, job_names=job_names))
+        index[config] = {str(jid): name for jid, name in job_names.items()}
 
+    save_job_index(logs_root, index)
     return configs
 
 
@@ -167,13 +219,23 @@ def analyze_config_logs(config_logs: list[ConfigLogs]) -> list[ConfigResult]:
     for config in config_logs:
         skipped_any: set[str] = set()
         non_skipped_any: set[str] = set()
+        test_suites: dict[str, str] = {}
+
         for log_path in config.log_paths:
             text = log_path.read_text(encoding="utf-8", errors="replace")
 
             skipped_in_log, non_skipped_in_log = extract_test_status_sets(text)
             skipped_any.update(skipped_in_log)
             non_skipped_any.update(non_skipped_in_log)
 
+            # Associate skipped test IDs with the suite derived from the job name.
+            with contextlib.suppress(ValueError):
+                job_id = int(log_path.stem)
+                suite = config.job_names.get(job_id, "")
+                if suite:
+                    for test_id in skipped_in_log:
+                        test_suites.setdefault(test_id, suite)
+
         # For sharded matrices, a test may only appear in one log. Treat it as
         # config-skipped if it is skipped at least once and never non-skipped
         # (passed/failed/error/xpass/xfail) in that config.
@@ -185,6 +247,7 @@ def analyze_config_logs(config_logs: list[ConfigLogs]) -> list[ConfigResult]:
                 job_ids=config.job_ids,
                 skipped=skipped_for_config,
                 has_logs=bool(config.log_paths),
+                test_suites=test_suites,
             )
         )
 
@@ -217,16 +280,22 @@ def build_summary(results: list[ConfigResult]) -> str:
             "_Note: the test `tests/test_cuda.py::test_always_skip` is expected to be skipped in all configurations, but is missing._"
         )
 
+    # Merge test->suite mappings across all configs (first one seen wins).
+    test_suites: dict[str, str] = {}
+    for result in results:
+        for test_id, suite in result.test_suites.items():
+            test_suites.setdefault(test_id, suite)
+
     universal = sorted(intersection or set())
     lines.append(f"Tests skipped across wheel test configurations ({len(results)}):")
     lines.append("")
     if not universal:
         lines.append("_No tests were skipped in all configurations._")
     else:
-        lines.append("| Test |")
-        lines.append("| --- |")
         for test in universal:
-            lines.append(f"| `{test}` |")
+            suite = test_suites.get(test, "")
+            label = f"{suite}/{test}" if suite else test
+            lines.append(f"- [ ] `{label}`")
 
     return "\n".join(lines) + "\n"