Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
"test:core": "pytest tests/test_core_foundation_ts.py -q",
"test:replay": "pytest tests/test_paper_replay_bench.py tests/test_agent_trace_replay.py tests/test_replay_continuity.py -q",
"layout": "python scripts/check_repo_layout.py",
"check": "npm run layout && npm run typecheck && npm run validate && npm run build && npm run test"
"check": "npm run layout && npm run typecheck && npm run validate && npm run build && npm run test",
"generate:layered-admissibility": "python scripts/generate_layered_admissibility_artifact.py"
}
}
39 changes: 39 additions & 0 deletions scripts/generate_layered_admissibility_artifact.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
"""Deterministic entrypoint for layered admissibility artifact regeneration."""

from __future__ import annotations

import sys
from pathlib import Path

REPO_ROOT = Path(__file__).resolve().parents[1]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))

from src.validation.degradation_curve_generator import DegradationCurveGenerator

CURVE_ID = "coding_workflow_pr_review_curve_v1"
OUTPUT_PATH = Path("artifacts/layered_admissibility_results.json")

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The OUTPUT_PATH is currently defined as a relative path. Since this script is intended to be a deterministic entrypoint and already calculates REPO_ROOT, it is more robust to anchor the output path to the repository root. This ensures the script behaves consistently regardless of the current working directory.

Suggested change
OUTPUT_PATH = Path("artifacts/layered_admissibility_results.json")
OUTPUT_PATH = REPO_ROOT / "artifacts" / "layered_admissibility_results.json"



def generate_layered_admissibility_artifact(
output_path: Path = OUTPUT_PATH,
*,
curve_id: str = CURVE_ID,
) -> Path:
"""Regenerate layered admissibility artifact from manifest-ordered fixtures."""

generator = DegradationCurveGenerator()
fixtures = generator.fixtures_for_layered_admissibility_curve()

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The call to fixtures_for_layered_admissibility_curve() relies on a default relative path for the manifest file. To ensure portability and consistency with the REPO_ROOT logic, it's better to pass an absolute path anchored to the repository root.

Suggested change
fixtures = generator.fixtures_for_layered_admissibility_curve()
fixtures = generator.fixtures_for_layered_admissibility_curve(manifest_path=REPO_ROOT / "fixtures" / "manifest.json")

curve = generator.generate(fixtures, curve_id=curve_id)
generator.write_json(curve, output_path)
return output_path


def main() -> int:
output_path = generate_layered_admissibility_artifact()
print(output_path.as_posix())
return 0


if __name__ == "__main__":
raise SystemExit(main())
39 changes: 39 additions & 0 deletions tests/test_generate_layered_admissibility_artifact.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
from __future__ import annotations

import json
from pathlib import Path

from scripts.generate_layered_admissibility_artifact import (
CURVE_ID,
generate_layered_admissibility_artifact,
)
Comment on lines +6 to +9

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Import REPO_ROOT from the script to allow anchoring paths in the test file, ensuring tests can be run from any directory.

Suggested change
from scripts.generate_layered_admissibility_artifact import (
CURVE_ID,
generate_layered_admissibility_artifact,
)
from scripts.generate_layered_admissibility_artifact import (
CURVE_ID,
REPO_ROOT,
generate_layered_admissibility_artifact,
)

from src.validation.degradation_curve_generator import DegradationCurveGenerator

ARTIFACT_PATH = Path("artifacts/layered_admissibility_results.json")

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Anchor ARTIFACT_PATH to the repository root using the imported REPO_ROOT. This improves test robustness when executed from different working directories.

Suggested change
ARTIFACT_PATH = Path("artifacts/layered_admissibility_results.json")
ARTIFACT_PATH = REPO_ROOT / "artifacts" / "layered_admissibility_results.json"

CANONICAL_FIXTURE_ID = "coding_workflow_pr_review_moderate_v1"
CANONICAL_MODERATE_SCORE = 0.8333333333333334


def test_generated_artifact_matches_committed_artifact(tmp_path: Path) -> None:
output_path = tmp_path / "layered_admissibility_results.json"
generate_layered_admissibility_artifact(output_path)

generated = json.loads(output_path.read_text(encoding="utf-8"))
committed = json.loads(ARTIFACT_PATH.read_text(encoding="utf-8"))
assert generated == committed


def test_generated_artifact_has_no_new_top_level_fields() -> None:
generator = DegradationCurveGenerator()
curve = generator.generate(generator.fixtures_for_layered_admissibility_curve(), curve_id=CURVE_ID)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Pass the absolute manifest path to fixtures_for_layered_admissibility_curve to ensure the test can locate the manifest regardless of the current working directory.

Suggested change
curve = generator.generate(generator.fixtures_for_layered_admissibility_curve(), curve_id=CURVE_ID)
fixtures = generator.fixtures_for_layered_admissibility_curve(manifest_path=REPO_ROOT / "fixtures" / "manifest.json")
curve = generator.generate(fixtures, curve_id=CURVE_ID)


generated_top_level = set(generator.to_dict(curve).keys())
committed_top_level = set(json.loads(ARTIFACT_PATH.read_text(encoding="utf-8")).keys())
assert generated_top_level == committed_top_level


def test_moderate_fixture_score_remains_canonical() -> None:
payload = json.loads(ARTIFACT_PATH.read_text(encoding="utf-8"))
points = payload["points"]
moderate_point = next(point for point in points if point["fixture_id"] == CANONICAL_FIXTURE_ID)
assert moderate_point["overall_admissibility_score"] == CANONICAL_MODERATE_SCORE
Loading