Skip to content

Commit 52da23a

Browse files
ProfRandom92ProfRandom92
authored
feat: add agent artifact bundle
Adds a deterministic local evidence bundle script for AI-assisted work, including safe_pr_gate result reuse, validation evidence fields, optional MCP context output references, focused tests, and Gemini review fixes for status/exit-code propagation.
1 parent ae63af9 commit 52da23a

3 files changed

Lines changed: 269 additions & 0 deletions

File tree

‎docs/codex_skills/artifact_validation.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ Create or update deterministic artifact checks without changing benchmark meanin
77
## When to use
88

99
Use for artifacts under `artifacts/`, generator scripts under `scripts/`, regeneration parity tests, stable JSON checks, and committed artifact reproducibility.
10+
Use `scripts/agent_artifact_bundle.py` for deterministic evidence bundles that combine safe gate status with explicit validation evidence.
1011

1112
## Allowed actions
1213

‎scripts/agent_artifact_bundle.py‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
#!/usr/bin/env python3
2+
"""Build a deterministic local evidence bundle for AI-assisted work."""
3+
4+
from __future__ import annotations
5+
6+
import argparse
7+
import json
8+
import sys
9+
from pathlib import Path
10+
from typing import Any
11+
12+
REPO_ROOT = Path(__file__).resolve().parents[1]
13+
if str(REPO_ROOT) not in sys.path:
14+
sys.path.insert(0, str(REPO_ROOT))
15+
16+
from scripts.safe_pr_gate import GateState, collect_gate_state, evaluate_gate
17+
18+
19+
def _parse_args(argv: list[str]) -> argparse.Namespace:
20+
parser = argparse.ArgumentParser(description="Build a deterministic local evidence bundle for AI-assisted work.")
21+
parser.add_argument("--allow-main", action="store_true", help="Allow bundle generation on main.")
22+
parser.add_argument(
23+
"--validation-command",
24+
action="append",
25+
default=[],
26+
help="Validation command executed for the bundle evidence. May be repeated.",
27+
)
28+
parser.add_argument(
29+
"--validation-result",
30+
action="append",
31+
default=[],
32+
help="Validation result corresponding to each command. May be repeated.",
33+
)
34+
parser.add_argument(
35+
"--mcp-context-output-ref",
36+
help="Optional reference to a previously generated MCP context tool output.",
37+
)
38+
return parser.parse_args(argv)
39+
40+
41+
def _error_response(exc: RuntimeError) -> dict[str, Any]:
42+
return {
43+
"error": {
44+
"message": str(exc),
45+
"type": exc.__class__.__name__,
46+
},
47+
"ok": False,
48+
"result": "ERROR",
49+
}
50+
51+
52+
def _build_validation_evidence(commands: list[str], results: list[str]) -> list[dict[str, str]]:
53+
if len(commands) != len(results):
54+
raise RuntimeError(
55+
"validation command/result count mismatch: "
56+
f"{len(commands)} command(s), {len(results)} result(s)"
57+
)
58+
return [{"command": command, "result": result} for command, result in zip(commands, results)]
59+
60+
61+
def build_agent_artifact_bundle(
62+
state: GateState,
63+
*,
64+
allow_main: bool,
65+
validation_commands: list[str],
66+
validation_results: list[str],
67+
mcp_context_output_ref: str | None = None,
68+
) -> dict[str, Any]:
69+
if state.branch == "main" and not allow_main:
70+
raise RuntimeError("main branch is not allowed for agent artifact bundling")
71+
72+
safe_pr_gate_result = evaluate_gate(state)
73+
bundle_ok = safe_pr_gate_result.ok
74+
bundle: dict[str, Any] = {
75+
"branch": state.branch,
76+
"changed_files": list(state.changed_paths),
77+
"ok": bundle_ok,
78+
"result": "PASS" if bundle_ok else "FAIL",
79+
"safe_pr_gate": safe_pr_gate_result.to_dict(),
80+
"validation_evidence": _build_validation_evidence(validation_commands, validation_results),
81+
}
82+
if mcp_context_output_ref is not None:
83+
bundle["mcp_context_output_ref"] = mcp_context_output_ref
84+
return bundle
85+
86+
87+
def main(argv: list[str] | None = None) -> int:
88+
args = _parse_args(sys.argv[1:] if argv is None else argv)
89+
try:
90+
state = collect_gate_state()
91+
bundle = build_agent_artifact_bundle(
92+
state,
93+
allow_main=args.allow_main,
94+
validation_commands=list(args.validation_command),
95+
validation_results=list(args.validation_result),
96+
mcp_context_output_ref=args.mcp_context_output_ref,
97+
)
98+
sys.stdout.write(json.dumps(bundle, indent=2, sort_keys=True) + "\n")
99+
return 0 if bundle["ok"] else 1
100+
except RuntimeError as exc:
101+
sys.stdout.write(json.dumps(_error_response(exc), indent=2, sort_keys=True) + "\n")
102+
return 1
103+
104+
105+
if __name__ == "__main__":
106+
raise SystemExit(main())

‎tests/test_agent_artifact_bundle.py‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
from __future__ import annotations
2+
3+
import json
4+
5+
import pytest
6+
7+
import scripts.agent_artifact_bundle as agent_artifact_bundle
8+
from scripts.safe_pr_gate import GateState, evaluate_gate
9+
10+
11+
def test_build_agent_artifact_bundle_is_deterministic_and_includes_optional_metadata() -> None:
12+
state = GateState(
13+
branch="feat/agent-artifact-bundle",
14+
status_short=(),
15+
changed_paths=(),
16+
)
17+
18+
bundle = agent_artifact_bundle.build_agent_artifact_bundle(
19+
state,
20+
allow_main=False,
21+
validation_commands=["python -m compileall -q scripts/agent_artifact_bundle.py", "pytest tests/test_agent_artifact_bundle.py -q"],
22+
validation_results=["pass", "pass"],
23+
mcp_context_output_ref="artifacts/mcp_context_layer_example.json",
24+
)
25+
26+
assert bundle == {
27+
"branch": "feat/agent-artifact-bundle",
28+
"changed_files": [],
29+
"mcp_context_output_ref": "artifacts/mcp_context_layer_example.json",
30+
"ok": True,
31+
"result": "PASS",
32+
"safe_pr_gate": evaluate_gate(state).to_dict(),
33+
"validation_evidence": [
34+
{
35+
"command": "python -m compileall -q scripts/agent_artifact_bundle.py",
36+
"result": "pass",
37+
},
38+
{
39+
"command": "pytest tests/test_agent_artifact_bundle.py -q",
40+
"result": "pass",
41+
},
42+
],
43+
}
44+
45+
first = json.dumps(bundle, indent=2, sort_keys=True)
46+
second = json.dumps(bundle, indent=2, sort_keys=True)
47+
assert first == second
48+
49+
50+
def test_build_agent_artifact_bundle_rejects_main_without_allow_main() -> None:
51+
state = GateState(branch="main", status_short=(), changed_paths=())
52+
53+
with pytest.raises(RuntimeError, match="main branch is not allowed for agent artifact bundling"):
54+
agent_artifact_bundle.build_agent_artifact_bundle(
55+
state,
56+
allow_main=False,
57+
validation_commands=[],
58+
validation_results=[],
59+
)
60+
61+
62+
def test_build_agent_artifact_bundle_reflects_safe_gate_failure() -> None:
63+
state = GateState(
64+
branch="feat/agent-artifact-bundle",
65+
status_short=(" M docs/example.md",),
66+
changed_paths=("docs/example.md",),
67+
)
68+
69+
bundle = agent_artifact_bundle.build_agent_artifact_bundle(
70+
state,
71+
allow_main=False,
72+
validation_commands=["python -m compileall -q scripts/agent_artifact_bundle.py"],
73+
validation_results=["pass"],
74+
)
75+
76+
assert bundle["ok"] is False
77+
assert bundle["result"] == "FAIL"
78+
assert bundle["safe_pr_gate"]["ok"] is False
79+
assert bundle["safe_pr_gate"]["result"] == "FAIL"
80+
81+
82+
def test_main_emits_deterministic_json_and_omits_optional_reference(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
83+
state = GateState(
84+
branch="feat/agent-artifact-bundle",
85+
status_short=(),
86+
changed_paths=("scripts/agent_artifact_bundle.py",),
87+
)
88+
monkeypatch.setattr(agent_artifact_bundle, "collect_gate_state", lambda: state)
89+
90+
exit_code = agent_artifact_bundle.main(
91+
[
92+
"--validation-command",
93+
"python -m compileall -q scripts/agent_artifact_bundle.py",
94+
"--validation-result",
95+
"pass",
96+
]
97+
)
98+
output = json.loads(capsys.readouterr().out)
99+
100+
assert exit_code == 0
101+
assert output == {
102+
"branch": "feat/agent-artifact-bundle",
103+
"changed_files": ["scripts/agent_artifact_bundle.py"],
104+
"ok": True,
105+
"result": "PASS",
106+
"safe_pr_gate": evaluate_gate(state).to_dict(),
107+
"validation_evidence": [
108+
{
109+
"command": "python -m compileall -q scripts/agent_artifact_bundle.py",
110+
"result": "pass",
111+
}
112+
],
113+
}
114+
assert "mcp_context_output_ref" not in output
115+
116+
117+
def test_main_returns_failure_exit_code_when_safe_gate_fails(
118+
monkeypatch: pytest.MonkeyPatch,
119+
capsys: pytest.CaptureFixture[str],
120+
) -> None:
121+
state = GateState(
122+
branch="feat/agent-artifact-bundle",
123+
status_short=(" M docs/example.md",),
124+
changed_paths=("docs/example.md",),
125+
)
126+
monkeypatch.setattr(agent_artifact_bundle, "collect_gate_state", lambda: state)
127+
128+
exit_code = agent_artifact_bundle.main(
129+
[
130+
"--validation-command",
131+
"python -m compileall -q scripts/agent_artifact_bundle.py",
132+
"--validation-result",
133+
"pass",
134+
]
135+
)
136+
output = json.loads(capsys.readouterr().out)
137+
138+
assert exit_code == 1
139+
assert output["ok"] is False
140+
assert output["result"] == "FAIL"
141+
assert output["safe_pr_gate"]["ok"] is False
142+
assert output["safe_pr_gate"]["result"] == "FAIL"
143+
144+
145+
def test_main_reports_main_branch_as_deterministic_error_json(
146+
monkeypatch: pytest.MonkeyPatch,
147+
capsys: pytest.CaptureFixture[str],
148+
) -> None:
149+
monkeypatch.setattr(agent_artifact_bundle, "collect_gate_state", lambda: GateState(branch="main", status_short=(), changed_paths=()))
150+
151+
exit_code = agent_artifact_bundle.main([])
152+
output = json.loads(capsys.readouterr().out)
153+
154+
assert exit_code == 1
155+
assert output == {
156+
"error": {
157+
"message": "main branch is not allowed for agent artifact bundling",
158+
"type": "RuntimeError",
159+
},
160+
"ok": False,
161+
"result": "ERROR",
162+
}

0 commit comments

Comments
 (0)