Add conformance results invariant validator

Author: JelleZijlstra

.github/workflows/conformance.yml

@@ -32,6 +32,11 @@ jobs:
           uv sync --python 3.12 --frozen
           uv run --python 3.12 --frozen python src/main.py
 
+      - name: Validate conformance invariants
+        working-directory: conformance
+        run: |
+          uv run --python 3.12 --frozen python src/validate_results.py
+
       - name: Assert conformance results are up to date
         run: |
           if [ -n "$(git status --porcelain -- conformance/results)" ]; then

conformance/results/mypy/generics_defaults.toml

@@ -1,4 +1,8 @@
 conformant = "Partial"
+notes = """
+Does not detect a TypeVar with a default used after a TypeVarTuple.
+Does not fully support defaults on TypeVarTuple and ParamSpec.
+"""
 output = """
 generics_defaults.py:24: error: "T" cannot appear after "DefaultStrT" in type parameter list because it has no default type  [misc]
 generics_defaults.py:66: error: "AllTheDefaults" expects between 2 and 5 type arguments, but 1 given  [type-arg]

conformance/results/mypy/generics_defaults_referential.toml

@@ -1,4 +1,7 @@
 conformant = "Partial"
+notes = """
+Does not correctly handle defaults referencing other TypeVars.
+"""
 output = """
 generics_defaults_referential.py:23: error: Expression is of type "type[slice[StartT, StopT, StepT]]", not "type[slice[int, int, int | None]]"  [assert-type]
 generics_defaults_referential.py:37: error: Argument 1 to "Foo" has incompatible type "str"; expected "int"  [arg-type]

conformance/results/mypy/generics_defaults_specialization.toml

@@ -1,4 +1,7 @@
 conformant = "Partial"
+notes = """
+Does not correctly resolve defaults when classes are used directly.
+"""
 output = """
 generics_defaults_specialization.py:30: error: Bad number of arguments for type alias, expected between 0 and 1, given 2  [type-arg]
 generics_defaults_specialization.py:45: error: Expression is of type "type[Bar[DefaultStrT]]", not "type[Bar[str]]"  [assert-type]

conformance/src/validate_results.py

@@ -0,0 +1,86 @@
+"""
+Validate invariants for conformance result files.
+"""
+
+from pathlib import Path
+import sys
+import tomllib
+from typing import Any
+
+
+def main() -> int:
+    results_dir = Path(__file__).resolve().parent.parent / "results"
+    issues: list[str] = []
+    checked = 0
+
+    for type_checker_dir in sorted(results_dir.iterdir()):
+        if not type_checker_dir.is_dir():
+            continue
+        for file in sorted(type_checker_dir.iterdir()):
+            if file.name == "version.toml":
+                continue
+            checked += 1
+            try:
+                with file.open("rb") as f:
+                    info = tomllib.load(f)
+            except Exception as e:
+                issues.append(f"{file.relative_to(results_dir)}: failed to parse TOML ({e})")
+                continue
+
+            issues.extend(_validate_result(file, results_dir, info))
+
+    if issues:
+        print(f"Found {len(issues)} invariant violation(s) across {checked} file(s):")
+        for issue in issues:
+            print(f"- {issue}")
+        return 1
+
+    print(f"Validated {checked} conformance result file(s); no invariant violations found.")
+    return 0
+
+
+def _validate_result(file: Path, results_dir: Path, info: dict[str, Any]) -> list[str]:
+    issues: list[str] = []
+    rel_path = file.relative_to(results_dir)
+
+    automated = info.get("conformance_automated")
+    if automated not in {"Pass", "Fail"}:
+        issues.append(
+            f"{rel_path}: conformance_automated must be 'Pass' or 'Fail' (got {automated!r})"
+        )
+        return issues
+    automated_is_pass = automated == "Pass"
+
+    conformant = info.get("conformant")
+    if conformant is None:
+        if automated_is_pass:
+            conformant_is_pass = True
+        else:
+            issues.append(
+                f"{rel_path}: conformant is required when conformance_automated is 'Fail'"
+            )
+            return issues
+    elif isinstance(conformant, str):
+        conformant_is_pass = conformant == "Pass"
+    else:
+        issues.append(f"{rel_path}: conformant must be a string when present")
+        return issues
+
+    if conformant_is_pass != automated_is_pass:
+        issues.append(
+            f"{rel_path}: conformant={conformant!r} does not match "
+            f"conformance_automated={automated!r}"
+        )
+
+    if not conformant_is_pass:
+        notes = info.get("notes", "")
+        if not isinstance(notes, str) or not notes.strip():
+            issues.append(
+                f"{rel_path}: notes must be present when checker is not fully conformant"
+            )
+
+    return issues
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())