|
| 1 | +from __future__ import annotations |
| 2 | + |
| 3 | +import json |
| 4 | +from pathlib import Path |
| 5 | + |
| 6 | +ROOT = Path(__file__).resolve().parent.parent |
| 7 | +MANIFEST_PATH = ROOT / "fixtures" / "manifest.json" |
| 8 | +ALLOWED_DEGRADATION_LEVELS = ["baseline", "mild", "moderate", "severe"] |
| 9 | +EXPECTED_FIXTURE_ORDER = [ |
| 10 | + "coding_workflow_pr_review_v1", |
| 11 | + "coding_workflow_pr_review_mild_v1", |
| 12 | + "coding_workflow_pr_review_moderate_v1", |
| 13 | + "coding_workflow_pr_review_degraded_v1", |
| 14 | +] |
| 15 | + |
| 16 | + |
| 17 | +def _load_json(path: Path) -> dict: |
| 18 | + with path.open("r", encoding="utf-8") as f: |
| 19 | + return json.load(f) |
| 20 | + |
| 21 | + |
| 22 | +def _load_manifest() -> dict: |
| 23 | + return _load_json(MANIFEST_PATH) |
| 24 | + |
| 25 | + |
| 26 | +def test_manifest_is_json_serializable_and_sorted() -> None: |
| 27 | + manifest = _load_manifest() |
| 28 | + json.dumps(manifest, sort_keys=True) |
| 29 | + |
| 30 | + fixture_ids = [entry["fixture_id"] for entry in manifest["fixtures"]] |
| 31 | + assert fixture_ids == EXPECTED_FIXTURE_ORDER |
| 32 | + |
| 33 | + |
| 34 | +def test_manifest_paths_exist() -> None: |
| 35 | + manifest = _load_manifest() |
| 36 | + required_paths = [ |
| 37 | + Path("original/trace.json"), |
| 38 | + Path("original/state.json"), |
| 39 | + Path("original/dependency_graph.json"), |
| 40 | + Path("original/contracts"), |
| 41 | + Path("reconstructed/trace.json"), |
| 42 | + Path("reconstructed/state.json"), |
| 43 | + Path("reconstructed/dependency_graph.json"), |
| 44 | + Path("expected/admissibility.json"), |
| 45 | + Path("expected/failures.json"), |
| 46 | + Path("README.md"), |
| 47 | + ] |
| 48 | + |
| 49 | + for entry in manifest["fixtures"]: |
| 50 | + fixture_dir = ROOT / entry["path"] |
| 51 | + assert fixture_dir.exists(), f"Missing fixture directory: {fixture_dir}" |
| 52 | + for rel_path in required_paths: |
| 53 | + assert (fixture_dir / rel_path).exists(), f"Missing path: {fixture_dir / rel_path}" |
| 54 | + |
| 55 | + |
| 56 | +def test_manifest_matches_fixture_admissibility_metadata() -> None: |
| 57 | + manifest = _load_manifest() |
| 58 | + |
| 59 | + for entry in manifest["fixtures"]: |
| 60 | + admissibility = _load_json(ROOT / entry["path"] / "expected" / "admissibility.json") |
| 61 | + assert entry["fixture_id"] == admissibility["fixture_id"] |
| 62 | + assert entry["fixture_version"] == admissibility["fixture_version"] |
| 63 | + assert entry["expected_admissible"] == admissibility["expected_admissible"] |
| 64 | + assert entry["expected_failure_labels"] == sorted(admissibility.get("expected_failure_labels", [])) |
| 65 | + |
| 66 | + |
| 67 | +def test_manifest_contracts_match_contract_files() -> None: |
| 68 | + manifest = _load_manifest() |
| 69 | + |
| 70 | + for entry in manifest["fixtures"]: |
| 71 | + contracts_dir = ROOT / entry["path"] / "original" / "contracts" |
| 72 | + contract_ids = [] |
| 73 | + for contract_file in sorted(contracts_dir.glob("*.json")): |
| 74 | + contract_ids.append(_load_json(contract_file)["contract_id"]) |
| 75 | + assert sorted(contract_ids) == entry["contracts"] |
| 76 | + |
| 77 | + |
| 78 | +def test_manifest_expected_failure_labels_match_failures_file() -> None: |
| 79 | + manifest = _load_manifest() |
| 80 | + |
| 81 | + for entry in manifest["fixtures"]: |
| 82 | + failures = _load_json(ROOT / entry["path"] / "expected" / "failures.json") |
| 83 | + assert entry["expected_failure_labels"] == sorted(failures.get("expected_failures", [])) |
| 84 | + |
| 85 | + |
| 86 | +def test_benchmark_artifact_references_only_manifest_fixtures() -> None: |
| 87 | + manifest = _load_manifest() |
| 88 | + benchmark = _load_json(ROOT / "artifacts" / "layered_admissibility_results.json") |
| 89 | + |
| 90 | + manifest_index = { |
| 91 | + entry["fixture_id"]: { |
| 92 | + "fixture_version": entry["fixture_version"], |
| 93 | + "path": entry["path"], |
| 94 | + } |
| 95 | + for entry in manifest["fixtures"] |
| 96 | + } |
| 97 | + |
| 98 | + for point in benchmark["points"]: |
| 99 | + fixture_id = point["fixture_id"] |
| 100 | + assert fixture_id in manifest_index |
| 101 | + assert point["fixture_version"] == manifest_index[fixture_id]["fixture_version"] |
| 102 | + assert point["fixture_path"] == manifest_index[fixture_id]["path"] |
| 103 | + |
| 104 | + |
| 105 | +def test_degradation_levels_are_known_and_unique_per_family() -> None: |
| 106 | + manifest = _load_manifest() |
| 107 | + family_to_levels: dict[str, set[str]] = {} |
| 108 | + |
| 109 | + for entry in manifest["fixtures"]: |
| 110 | + level = entry["degradation_level"] |
| 111 | + family = entry["family"] |
| 112 | + assert level in ALLOWED_DEGRADATION_LEVELS |
| 113 | + family_to_levels.setdefault(family, set()) |
| 114 | + assert level not in family_to_levels[family] |
| 115 | + family_to_levels[family].add(level) |
| 116 | + |
| 117 | + |
| 118 | +def test_no_unregistered_fixture_directories() -> None: |
| 119 | + manifest = _load_manifest() |
| 120 | + registered_paths = {entry["path"] for entry in manifest["fixtures"]} |
| 121 | + |
| 122 | + discovered_fixture_paths = { |
| 123 | + str(path.parent.parent.relative_to(ROOT)).replace("\\", "/") |
| 124 | + for path in (ROOT / "fixtures").glob("*/expected/admissibility.json") |
| 125 | + } |
| 126 | + |
| 127 | + assert discovered_fixture_paths.issubset(registered_paths) |
0 commit comments