Skip to content

Commit f6fcb5c

Browse files
tbitcstbitcs
committed
Fix benchmark recovery and archive admissions
1 parent 49b7451 commit f6fcb5c

10 files changed

Lines changed: 186 additions & 5 deletions

File tree

‎.specsmith/requirements.json‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4962,5 +4962,16 @@
49624962
"TEST-518",
49634963
"TEST-519"
49644964
]
4965+
},
4966+
{
4967+
"id": "REQ-492",
4968+
"version": 1,
4969+
"title": "Preregistered real-repository publication evidence",
4970+
"description": "GovernanceBench publication runs MUST fail closed against a byte-hashed, frozen preregistration that fixes task IDs, conditions, repetition counts, exact model/provider routes, controller protocol, metrics, failure accounting, and promotion gates before paid inference begins. The publication suite MUST include a license-preserving, commit-pinned snapshot of an independent upstream repository with public validators and an evaluator-only hidden oracle. Raw rows and compact evidence manifests MUST retain the preregistration identity and digest. Failed model runs MUST remain in token and cost denominators and their absolute expenditure MUST be reported. Open models in the 20B-35B band MUST advance sequentially from n=1 admission to n=5 screening and n=10 confirmation only when each frozen correctness and efficiency gate passes.",
4971+
"source": "Preprint publication-readiness protocol GB-PREPRINT-2026-07-30-V1",
4972+
"status": "implemented",
4973+
"test_ids": [
4974+
"TEST-520"
4975+
]
49654976
}
49664977
]

‎.specsmith/testcases.json‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5686,5 +5686,17 @@
56865686
"input": {},
56875687
"expected_behavior": {},
56885688
"confidence": 1.0
5689+
},
5690+
{
5691+
"id": "TEST-520",
5692+
"version": 1,
5693+
"title": "Frozen real-repository publication protocol",
5694+
"description": "Tests verify the byte-exact preregistration, locked three-task and open-model profiles, exact model/provider routes, fail-closed drift handling before inference, pinned upstream file hashes and license, src-layout execution, hidden acceptance isolation, compact evidence provenance, PEP 735 test dependency provisioning, and explicit failed-run token and cost expenditure.",
5695+
"requirement_id": "REQ-492",
5696+
"type": "integration",
5697+
"verification_method": "pytest tests/test_preprint_publication_readiness.py",
5698+
"input": "Frozen protocol, T30 upstream snapshot, clean starter, representative pass/fail rows",
5699+
"expected_behavior": "Publication evidence cannot drift, hide failed spend, or claim unearned open-model replication",
5700+
"confidence": 1.0
56895701
}
56905702
]

‎paper/data/preprint-qwen3-32b-admission-30578331257/cells.csv‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
workflow_id,commit_sha,task,category,horizon,condition,rep,model,provider,model_tier,benchmark_profile,passed,acceptance_oracle_passed,project_tests_passed,input_tokens,output_tokens,cached_input_tokens,cache_write_tokens,tokens,cost_usd,rework_turns,governance_turns,llm_turns,wall_clock_s,stop_reason,expected_file_count,written_file_count,milestones_completed,milestones_total,tokens_per_completed_milestone,language_count,tool_schema_hashes
2+
30578331257,e9305bb01e5cb861f07ae7f7b01a8bf1a431c574,T30,real_repository_maintenance,standard,SPECSMITH_FULL,1,Qwen/Qwen3-32B:deepinfra,huggingface,open-mid,publication-open-admission,False,False,False,133325,15385,0,0,148710,0.014974,6,5,12,306.742,max_turns,6,6,3,3,49570.0,2,cf28361d99c24f40

‎paper/data/preprint-qwen3-32b-admission-30578331257/manifest.json‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
{
2+
"cell_fields": [
3+
"workflow_id",
4+
"commit_sha",
5+
"task",
6+
"category",
7+
"horizon",
8+
"condition",
9+
"rep",
10+
"model",
11+
"provider",
12+
"model_tier",
13+
"benchmark_profile",
14+
"passed",
15+
"acceptance_oracle_passed",
16+
"project_tests_passed",
17+
"input_tokens",
18+
"output_tokens",
19+
"cached_input_tokens",
20+
"cache_write_tokens",
21+
"tokens",
22+
"cost_usd",
23+
"rework_turns",
24+
"governance_turns",
25+
"llm_turns",
26+
"wall_clock_s",
27+
"stop_reason",
28+
"expected_file_count",
29+
"written_file_count",
30+
"milestones_completed",
31+
"milestones_total",
32+
"tokens_per_completed_milestone",
33+
"language_count",
34+
"tool_schema_hashes"
35+
],
36+
"cells_canonical_sha256": "82e465655505fc24be59f5aede50f3039fe45848cad1f8be57baf63514ce7c2a",
37+
"cells_csv_sha256": "e8a5d4a6b64b02115478beef94bf19842081dc141e10bda431aedc196d6383b7",
38+
"commit_sha": "e9305bb01e5cb861f07ae7f7b01a8bf1a431c574",
39+
"excluded_fields": [
40+
"agent_transcript",
41+
"final_diff",
42+
"governance_decision",
43+
"lint_output",
44+
"test_output",
45+
"verify_result"
46+
],
47+
"protocols": [
48+
{
49+
"protocol_id": "GB-PREPRINT-2026-07-30-V1",
50+
"protocol_sha256": "92545b3923527811f0b7e43d05c2cb728ace475423f93d2f5ee5192288f90f74"
51+
}
52+
],
53+
"row_count": 1,
54+
"schema": "governancebench-evidence-v2",
55+
"source_artifacts": [
56+
{
57+
"name": "bench-results-qwen3-32b.json",
58+
"rows": 1,
59+
"sha256": "ec3137bdd29c67f9f9e6467e657f833e7cfbb7bfcacc031d4e5c9d578194f138"
60+
}
61+
],
62+
"workflow_id": "30578331257"
63+
}

‎paper/data/preprint-qwen3.6-27b-admission-30578321675/cells.csv‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
workflow_id,commit_sha,task,category,horizon,condition,rep,model,provider,model_tier,benchmark_profile,passed,acceptance_oracle_passed,project_tests_passed,input_tokens,output_tokens,cached_input_tokens,cache_write_tokens,tokens,cost_usd,rework_turns,governance_turns,llm_turns,wall_clock_s,stop_reason,expected_file_count,written_file_count,milestones_completed,milestones_total,tokens_per_completed_milestone,language_count,tool_schema_hashes
2+
30578321675,e9305bb01e5cb861f07ae7f7b01a8bf1a431c574,T30,real_repository_maintenance,standard,SPECSMITH_FULL,1,Qwen/Qwen3.6-27B:deepinfra,huggingface,open-mid,publication-open-admission,False,False,False,40755,11710,0,0,52465,0.050514,5,2,7,159.636,text_response,6,2,1,3,52465.0,2,cf28361d99c24f40

‎paper/data/preprint-qwen3.6-27b-admission-30578321675/manifest.json‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
{
2+
"cell_fields": [
3+
"workflow_id",
4+
"commit_sha",
5+
"task",
6+
"category",
7+
"horizon",
8+
"condition",
9+
"rep",
10+
"model",
11+
"provider",
12+
"model_tier",
13+
"benchmark_profile",
14+
"passed",
15+
"acceptance_oracle_passed",
16+
"project_tests_passed",
17+
"input_tokens",
18+
"output_tokens",
19+
"cached_input_tokens",
20+
"cache_write_tokens",
21+
"tokens",
22+
"cost_usd",
23+
"rework_turns",
24+
"governance_turns",
25+
"llm_turns",
26+
"wall_clock_s",
27+
"stop_reason",
28+
"expected_file_count",
29+
"written_file_count",
30+
"milestones_completed",
31+
"milestones_total",
32+
"tokens_per_completed_milestone",
33+
"language_count",
34+
"tool_schema_hashes"
35+
],
36+
"cells_canonical_sha256": "d1d023d17fbd7e94a13dfcfb8116f470c76417f09a573b3ecd6ed0c69bfdc222",
37+
"cells_csv_sha256": "b436ef554de500bb8fc3843a01c4f15d2c1ea38b0d844ecb5fc888aad2d6e13d",
38+
"commit_sha": "e9305bb01e5cb861f07ae7f7b01a8bf1a431c574",
39+
"excluded_fields": [
40+
"agent_transcript",
41+
"final_diff",
42+
"governance_decision",
43+
"lint_output",
44+
"test_output",
45+
"verify_result"
46+
],
47+
"protocols": [
48+
{
49+
"protocol_id": "GB-PREPRINT-2026-07-30-V1",
50+
"protocol_sha256": "92545b3923527811f0b7e43d05c2cb728ace475423f93d2f5ee5192288f90f74"
51+
}
52+
],
53+
"row_count": 1,
54+
"schema": "governancebench-evidence-v2",
55+
"source_artifacts": [
56+
{
57+
"name": "bench-results-qwen3.6-27b.json",
58+
"rows": 1,
59+
"sha256": "52c04751876f2ce0e0fa97526343aee74f8533e2820470ad2b632ec27e21923a"
60+
}
61+
],
62+
"workflow_id": "30578321675"
63+
}

‎pyproject.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ dev = [
5050
"ruff>=0.4",
5151
"mypy>=1.10",
5252
"pre-commit>=3.0",
53+
"tomli>=2.0; python_version < '3.11'",
5354
"types-pyyaml>=6.0",
5455
]
5556
esdb = [

‎scripts/govern_bench/harness.py‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2135,6 +2135,14 @@ def _noop_action_signature(
21352135
]
21362136

21372137

2138+
def _has_milestone_packet_argument_error(tool_results: list[dict[str, Any]]) -> bool:
2139+
"""Return whether a normalized tool result reports an invalid milestone packet."""
2140+
prefix = "ERROR: at least path_1 and content_1 are required"
2141+
return any(
2142+
_stringify_content(result.get("content")).startswith(prefix) for result in tool_results
2143+
)
2144+
2145+
21382146
def _updated_repeated_noop_streak(
21392147
prior: int,
21402148
current_signature: str,
@@ -5212,10 +5220,7 @@ def _run_agent_loop(
52125220
if (
52135221
condition.id == "SPECSMITH_FULL"
52145222
and invalid_milestone_packet_count
5215-
and any(
5216-
result.startswith("ERROR: at least path_1 and content_1 are required")
5217-
for result in tool_results
5218-
)
5223+
and _has_milestone_packet_argument_error(tool_results)
52195224
):
52205225
remaining_paths = _next_incomplete_boundary_paths(task, files_written)
52215226
retry_tool = "write_file" if len(remaining_paths) == 1 else "write_milestone"

‎scripts/govern_bench/install_demo_deps.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,10 @@
1616
import sys
1717
from pathlib import Path
1818

19-
import tomllib
19+
try:
20+
import tomllib
21+
except ModuleNotFoundError: # pragma: no cover - exercised by the Python 3.10 CI jobs
22+
import tomli as tomllib
2023

2124
_PROJECTS = Path(__file__).with_name("projects")
2225

‎tests/test_preprint_publication_readiness.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
_exec_run_command,
2020
_exec_run_validator,
2121
_get_project_dir,
22+
_has_milestone_packet_argument_error,
2223
_install_acceptance_oracle,
2324
_project_pythonpath,
2425
)
@@ -252,6 +253,24 @@ def fake_post(
252253
assert captured["timeout_s"] == 30
253254

254255

256+
def test_milestone_packet_recovery_reads_normalized_tool_content() -> None:
257+
assert _has_milestone_packet_argument_error(
258+
[
259+
{
260+
"role": "tool",
261+
"tool_call_id": "call-1",
262+
"content": (
263+
"ERROR: at least path_1 and content_1 are required; "
264+
"provide a complete milestone packet"
265+
),
266+
}
267+
]
268+
)
269+
assert not _has_milestone_packet_argument_error(
270+
[{"role": "tool", "tool_call_id": "call-2", "content": "OK: wrote files"}]
271+
)
272+
273+
255274
def test_github_collection_matches_src_layout_runtime_path() -> None:
256275
workflow = (Path(__file__).parents[1] / ".github/workflows/bench.yml").read_text(
257276
encoding="utf-8"

0 commit comments

Comments
 (0)