Skip to content

Commit ea997d4

Browse files
authored
Add manifest-driven layered artifact generation command
1 parent f7bdd8a commit ea997d4

3 files changed

Lines changed: 80 additions & 1 deletion

File tree

package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
"test:core": "pytest tests/test_core_foundation_ts.py -q",
1212
"test:replay": "pytest tests/test_paper_replay_bench.py tests/test_agent_trace_replay.py tests/test_replay_continuity.py -q",
1313
"layout": "python scripts/check_repo_layout.py",
14-
"check": "npm run layout && npm run typecheck && npm run validate && npm run build && npm run test"
14+
"check": "npm run layout && npm run typecheck && npm run validate && npm run build && npm run test",
15+
"generate:layered-admissibility": "python scripts/generate_layered_admissibility_artifact.py"
1516
}
1617
}
Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
"""Deterministic entrypoint for layered admissibility artifact regeneration."""
2+
3+
from __future__ import annotations
4+
5+
import sys
6+
from pathlib import Path
7+
8+
REPO_ROOT = Path(__file__).resolve().parents[1]
9+
if str(REPO_ROOT) not in sys.path:
10+
sys.path.insert(0, str(REPO_ROOT))
11+
12+
from src.validation.degradation_curve_generator import DegradationCurveGenerator
13+
14+
CURVE_ID = "coding_workflow_pr_review_curve_v1"
15+
OUTPUT_PATH = Path("artifacts/layered_admissibility_results.json")
16+
17+
18+
def generate_layered_admissibility_artifact(
19+
output_path: Path = OUTPUT_PATH,
20+
*,
21+
curve_id: str = CURVE_ID,
22+
) -> Path:
23+
"""Regenerate layered admissibility artifact from manifest-ordered fixtures."""
24+
25+
generator = DegradationCurveGenerator()
26+
fixtures = generator.fixtures_for_layered_admissibility_curve()
27+
curve = generator.generate(fixtures, curve_id=curve_id)
28+
generator.write_json(curve, output_path)
29+
return output_path
30+
31+
32+
def main() -> int:
33+
output_path = generate_layered_admissibility_artifact()
34+
print(output_path.as_posix())
35+
return 0
36+
37+
38+
if __name__ == "__main__":
39+
raise SystemExit(main())
Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
from __future__ import annotations
2+
3+
import json
4+
from pathlib import Path
5+
6+
from scripts.generate_layered_admissibility_artifact import (
7+
CURVE_ID,
8+
generate_layered_admissibility_artifact,
9+
)
10+
from src.validation.degradation_curve_generator import DegradationCurveGenerator
11+
12+
ARTIFACT_PATH = Path("artifacts/layered_admissibility_results.json")
13+
CANONICAL_FIXTURE_ID = "coding_workflow_pr_review_moderate_v1"
14+
CANONICAL_MODERATE_SCORE = 0.8333333333333334
15+
16+
17+
def test_generated_artifact_matches_committed_artifact(tmp_path: Path) -> None:
18+
output_path = tmp_path / "layered_admissibility_results.json"
19+
generate_layered_admissibility_artifact(output_path)
20+
21+
generated = json.loads(output_path.read_text(encoding="utf-8"))
22+
committed = json.loads(ARTIFACT_PATH.read_text(encoding="utf-8"))
23+
assert generated == committed
24+
25+
26+
def test_generated_artifact_has_no_new_top_level_fields() -> None:
27+
generator = DegradationCurveGenerator()
28+
curve = generator.generate(generator.fixtures_for_layered_admissibility_curve(), curve_id=CURVE_ID)
29+
30+
generated_top_level = set(generator.to_dict(curve).keys())
31+
committed_top_level = set(json.loads(ARTIFACT_PATH.read_text(encoding="utf-8")).keys())
32+
assert generated_top_level == committed_top_level
33+
34+
35+
def test_moderate_fixture_score_remains_canonical() -> None:
36+
payload = json.loads(ARTIFACT_PATH.read_text(encoding="utf-8"))
37+
points = payload["points"]
38+
moderate_point = next(point for point in points if point["fixture_id"] == CANONICAL_FIXTURE_ID)
39+
assert moderate_point["overall_admissibility_score"] == CANONICAL_MODERATE_SCORE

0 commit comments

Comments
 (0)