fix(perf): validate profile primary thresholds (#70)

Co-authored-by: Arbousier1 <elderli@foxmail.com>

Author: TypeThe0ry

perf/ab/gate-config.json

@@ -116,7 +116,7 @@
         "display.spawn.time",
         "display.spawn.alloc"
       ],
-      "minimum_passes": 2,
+      "minimum_passes": 1,
       "must_pass": [
         "display.spawn.time"
       ],
@@ -130,7 +130,7 @@
         "region.queue.time",
         "region.queue.alloc"
       ],
-      "minimum_passes": 2,
+      "minimum_passes": 1,
       "must_pass": [
         "region.queue.time"
       ],

perf/ab/gate.py

@@ -451,33 +451,44 @@ def render_markdown(decision: dict[str, Any]) -> str:
     return "\n".join(lines)
 
 
-def evaluate(
+def validate_profile_config(
     config: dict[str, Any],
     profile_name: str,
     benchmark_specs: list[dict[str, Any]],
-    aa_pairs: dict[str, list[tuple[float, float]]],
-    ab_pairs: dict[str, list[tuple[float, float]]],
-    units: dict[str, str | None],
-) -> dict[str, Any]:
-    bootstrap = config["bootstrap"]
-    aa_results: list[dict[str, Any]] = []
-    benchmark_results: list[dict[str, Any]] = []
-    aa_passed = True
-    reasons: list[str] = []
-
+) -> tuple[set[str], int]:
     profile = config["profiles"][profile_name]
-    must_pass = set(profile.get("must_pass", []))
     configured_ids = [entry["id"] for entry in benchmark_specs]
     if len(configured_ids) != len(set(configured_ids)):
         raise ValueError(f"profile {profile_name} contains duplicate benchmark ids")
     primary_ids = {
         entry["id"] for entry in benchmark_specs if entry.get("metric", "primary") == "primary"
     }
+    must_pass = set(profile.get("must_pass", []))
     if not must_pass or not must_pass.issubset(primary_ids):
         raise ValueError(f"profile {profile_name} must_pass must contain only primary metrics")
-    minimum_passes = int(profile["minimum_passes"])
+    minimum_passes = profile.get("minimum_passes")
+    if isinstance(minimum_passes, bool) or not isinstance(minimum_passes, int):
+        raise ValueError(f"profile {profile_name} has a non-integer primary minimum_passes")
     if not 1 <= minimum_passes <= len(primary_ids):
         raise ValueError(f"profile {profile_name} has an invalid primary minimum_passes")
+    return must_pass, minimum_passes
+
+
+def evaluate(
+    config: dict[str, Any],
+    profile_name: str,
+    benchmark_specs: list[dict[str, Any]],
+    aa_pairs: dict[str, list[tuple[float, float]]],
+    ab_pairs: dict[str, list[tuple[float, float]]],
+    units: dict[str, str | None],
+) -> dict[str, Any]:
+    bootstrap = config["bootstrap"]
+    aa_results: list[dict[str, Any]] = []
+    benchmark_results: list[dict[str, Any]] = []
+    aa_passed = True
+    reasons: list[str] = []
+
+    must_pass, minimum_passes = validate_profile_config(config, profile_name, benchmark_specs)
     for entry in benchmark_specs:
         benchmark_id = entry["id"]
         aa_metrics = paired_statistics(
@@ -583,6 +594,7 @@ def run(args: argparse.Namespace) -> int:
     benchmark_specs = [benchmark_index[benchmark_id] for benchmark_id in profile["benchmark_ids"]]
     if not benchmark_specs:
         raise ValueError("at least one benchmark metric must be configured")
+    validate_profile_config(config, args.profile, benchmark_specs)
     config_sha = sha256_file(config_path)
     repetitions = int(config["matrix"]["repetitions_per_order"])
     aa_pairs, aa_units, aa_run = load_pairs(

perf/ab/tests/test_gate.py

@@ -272,6 +272,25 @@ def test_global_states_use_explicit_three_state_semantics(self) -> None:
         self.assertEqual("INCONCLUSIVE_NOISE_OR_NO_GAIN", gate.INCONCLUSIVE)
         self.assertEqual("FAIL_REGRESSION_OR_BEHAVIOR", gate.FAIL)
 
+    def test_every_profile_has_valid_primary_thresholds(self) -> None:
+        benchmark_index = {entry["id"]: entry for entry in self.config["benchmarks"]}
+        for profile_name, profile in self.config["profiles"].items():
+            with self.subTest(profile=profile_name):
+                benchmark_specs = [benchmark_index[entry] for entry in profile["benchmark_ids"]]
+                gate.validate_profile_config(self.config, profile_name, benchmark_specs)
+
+    def test_profile_validation_rejects_minimum_above_primary_count(self) -> None:
+        config = json.loads(json.dumps(self.config))
+        profile_name = "display-spawn"
+        config["profiles"][profile_name]["minimum_passes"] = 2
+        benchmark_index = {entry["id"]: entry for entry in config["benchmarks"]}
+        benchmark_specs = [
+            benchmark_index[entry]
+            for entry in config["profiles"][profile_name]["benchmark_ids"]
+        ]
+        with self.assertRaisesRegex(ValueError, "invalid primary minimum_passes"):
+            gate.validate_profile_config(config, profile_name, benchmark_specs)
+
 
 class MatrixScheduleTest(unittest.TestCase):
     def test_ab_schedule_is_exactly_four_interleaved_ab_and_ba_pairs(self) -> None: