test(bench-orchestrator): drop YAML-parity tests now that inline matrices are gone

The develop/nightly parity tests parsed the inline workflow matrices, which the
previous commit deleted. Replace them with behavior-preservation assertions on
the resolved target sets (TPC-H NVMe full coverage; nightly default lane at
SF=100); the golden files remain the full-output regression guard.

Signed-off-by: Connor Tsui <connor@spiraldb.com>
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PQg1cgPLKJg4QVJvE2xa3Q

Author: connortsui20

bench-orchestrator/tests/test_matrix.py

@@ -3,12 +3,9 @@
 
 """Tests for the declarative benchmark matrix.
 
-The parity tests read the current workflow YAML and assert that the ``develop`` and ``nightly``
-profiles reproduce today's hand-written matrices exactly. They are the proof that introducing the
-registry is behavior-preserving; once the workflows consume ``vx-bench matrix`` and the inline
-matrices are removed, these two tests should be replaced by the golden tests alone.
-
-Regenerate the golden files after an intended change with ``UPDATE_GOLDEN=1 pytest``.
+The golden files (``tests/golden/<profile>.json``) pin each profile's resolved matrix, so any
+change to what-runs surfaces as a reviewable diff. Regenerate them after an intended change with
+``UPDATE_GOLDEN=1 pytest``.
 """
 
 import json
@@ -197,57 +194,43 @@ def test_matrix_matches_golden(profile_name: str) -> None:
 
 
 # --------------------------------------------------------------------------------------------------
-# Parity with the current workflow YAML (behavior preservation)
+# Behavior preservation: develop and nightly reproduce the target sets the (now-deleted) inline
+# workflow matrices used. The golden files pin the full output; these spot the key invariants.
 # --------------------------------------------------------------------------------------------------
 
 
-def _repo_root() -> pathlib.Path:
-    return pathlib.Path(__file__).resolve().parents[2]
-
-
-def _extract_json_block(text: str, key: str) -> list[dict[str, object]]:
-    """Extract the JSON array that follows ``key`` in a ``... | <block>`` workflow scalar."""
-    pipe = text.index("|", text.index(key))
-    body: list[str] = []
-    indent: int | None = None
-    for line in text[pipe + 1 :].splitlines():
-        if line.strip() == "":
-            if indent is not None:
-                body.append(line)
-            continue
-        line_indent = len(line) - len(line.lstrip())
-        if indent is None:
-            indent = line_indent
-        elif line_indent < indent:
-            break
-        body.append(line)
-    return json.loads("\n".join(body))
-
-
-def _target_pairs(targets: list[dict[str, object]]) -> frozenset[tuple[object, object]]:
-    return frozenset((t["engine"], t["format"]) for t in targets)
+def _pairs(entry: dict[str, object]) -> set[tuple[str, str]]:
+    return {(t["engine"], t["format"]) for t in entry["targets"]}
 
 
-def test_develop_profile_matches_current_full_matrix() -> None:
-    text = (_repo_root() / ".github/workflows/sql-benchmarks.yml").read_text(encoding="utf-8")
-    current = {b["id"]: b for b in _extract_json_block(text, "benchmark_matrix:")}
-    resolved = {entry["id"]: entry for entry in resolve_matrix(PROFILES["develop"])}
-
-    assert set(resolved) == set(current)
-    for bid, bench in current.items():
-        assert _target_pairs(resolved[bid]["targets"]) == _target_pairs(bench["develop_targets"]), bid
-        assert resolved[bid]["data_formats"] == bench["data_formats"], bid
-
-
-def test_nightly_profile_matches_current_nightly_matrix() -> None:
-    text = (_repo_root() / ".github/workflows/nightly-bench.yml").read_text(encoding="utf-8")
-    current = _extract_json_block(text, "benchmark_matrix:")
-    resolved = {entry["id"]: entry for entry in resolve_matrix(PROFILES["nightly"])}
-
-    # The SF=100 entries map to the current nightly tpch-nvme / tpch-s3 rows by storage.
-    for resolved_id, bench in (("tpch-nvme-100", current[0]), ("tpch-s3-100", current[1])):
-        assert _target_pairs(resolved[resolved_id]["targets"]) == _target_pairs(bench["develop_targets"])
-        assert resolved[resolved_id]["data_formats"] == bench["data_formats"]
+def test_develop_tpch_nvme_keeps_full_target_coverage() -> None:
+    develop = {entry["id"]: entry for entry in resolve_matrix(PROFILES["develop"])}
+    # TPC-H on NVMe compares both engines across the columnar formats plus arrow, lance, duckdb.
+    assert _pairs(develop["tpch-nvme"]) == {
+        ("datafusion", "arrow"),
+        ("datafusion", "parquet"),
+        ("datafusion", "vortex"),
+        ("datafusion", "vortex-compact"),
+        ("datafusion", "lance"),
+        ("duckdb", "parquet"),
+        ("duckdb", "vortex"),
+        ("duckdb", "vortex-compact"),
+        ("duckdb", "duckdb"),
+    }
+    assert develop["tpch-nvme"]["data_formats"] == ["parquet", "vortex", "vortex-compact", "duckdb"]
+
+
+def test_nightly_runs_default_targets_at_sf100() -> None:
+    nightly = {entry["id"]: entry for entry in resolve_matrix(PROFILES["nightly"])}
+    assert set(nightly) == {"tpch-nvme-100", "tpch-s3-100"}
+    for entry in nightly.values():
+        assert _pairs(entry) == {
+            ("datafusion", "parquet"),
+            ("datafusion", "vortex"),
+            ("duckdb", "parquet"),
+            ("duckdb", "vortex"),
+        }
+        assert entry["scale_factor"] == "100.0"
 
 
 # --------------------------------------------------------------------------------------------------