Skip to content

Commit 97756fd

Browse files
committed
fix(examples): verify temporary prompt bundle
1 parent 2630b54 commit 97756fd

2 files changed

Lines changed: 37 additions & 0 deletions

File tree

examples/optimization/eval_optimize_loop/eval_loop/writeback.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,14 @@ def temporary_prompt_bundle(
195195
expected_sha256=expected_hashes[name],
196196
)
197197
written_hashes[name] = candidate_hashes[name]
198+
installed_hashes = _current_hashes(snapshot)
199+
candidate_mismatches = [
200+
name for name, candidate_hash in candidate_hashes.items() if installed_hashes[name] != candidate_hash
201+
]
202+
if candidate_mismatches:
203+
raise ConcurrentPromptUpdateError(
204+
"candidate prompt files changed before temporary evaluation: " f"{', '.join(candidate_mismatches)}"
205+
)
198206
yield
199207
except BaseException as primary_error:
200208
failures = _restore_snapshot(snapshot, written_hashes)

examples/optimization/eval_optimize_loop/tests/test_writeback.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,35 @@ def inject_external_before_second_precondition(file_descriptor: int):
173173
assert paths["user"].read_bytes() == external_content
174174

175175

176+
def test_temporary_verifies_complete_candidate_bundle_before_entering_body(tmp_path: Path, monkeypatch):
177+
paths, baseline = _prompt_files(tmp_path)
178+
snapshot = snapshot_prompt_files(paths)
179+
external_content = b"external after second candidate replace"
180+
original_replace = os.replace
181+
replace_calls = 0
182+
body_entered = False
183+
184+
def update_first_file_after_second_replace(source: str | bytes | Path, destination: str | bytes | Path):
185+
nonlocal replace_calls
186+
original_replace(source, destination)
187+
replace_calls += 1
188+
if replace_calls == 2:
189+
paths["system"].write_bytes(external_content)
190+
191+
monkeypatch.setattr(writeback.os, "replace", update_first_file_after_second_replace)
192+
193+
with pytest.raises((ConcurrentPromptUpdateError, RuntimeError)):
194+
with temporary_prompt_bundle(
195+
snapshot,
196+
{"system": "candidate system", "user": "candidate user"},
197+
):
198+
body_entered = True
199+
200+
assert not body_entered
201+
assert paths["system"].read_bytes() == external_content
202+
assert paths["user"].read_bytes() == baseline["user"]
203+
204+
176205
def test_commit_rejects_concurrent_update_before_any_write(tmp_path: Path, monkeypatch):
177206
paths, baseline = _prompt_files(tmp_path)
178207
snapshot = snapshot_prompt_files(paths)

0 commit comments

Comments
 (0)