Skip to content

Commit 323aad7

Browse files
abrichrclaude
andcommitted
feat: add [s] screenshot refresh to regenerate steps mid-recording
When the model's planned steps diverge from the actual UI (e.g. a menu doesn't have the expected option), the user can press 's' to take a fresh screenshot and regenerate all remaining steps from the current screen state — no need to describe what's wrong. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 57a5f8f commit 323aad7

1 file changed

Lines changed: 30 additions & 1 deletion

File tree

scripts/record_waa_demos.py

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1756,7 +1756,7 @@ def _hard_reset_task_env() -> bytes:
17561756
_display_current_step(step_num, total, remaining_steps[0])
17571757

17581758
user_input = input(
1759-
" [Enter]=done [d]=task done [r]=redo [R]=restart\n"
1759+
" [Enter]=done [d]=task done [r]=redo [R]=restart [s]=refresh steps\n"
17601760
" Or type feedback to correct remaining steps: "
17611761
).strip()
17621762

@@ -1868,6 +1868,35 @@ def _hard_reset_task_env() -> bytes:
18681868
print()
18691869
print(" Task restarted. Continue recording.\n")
18701870

1871+
elif user_input.lower() == "s":
1872+
# REFRESH: regenerate remaining steps from current screenshot
1873+
print(" Taking fresh screenshot and regenerating remaining steps...")
1874+
before_png = _take_screenshot(server)
1875+
new_suggested = _generate_steps(before_png, instruction, task_config)
1876+
new_steps = _parse_step_list(new_suggested)
1877+
if new_steps:
1878+
_display_steps(new_suggested)
1879+
new_suggested = _interactive_step_review(
1880+
before_png, instruction, task_config, new_suggested,
1881+
)
1882+
remaining_steps = _parse_step_list(new_suggested)
1883+
step_plans.append({
1884+
"at_step_idx": step_idx,
1885+
"trigger": "screenshot_refresh",
1886+
"steps": list(remaining_steps),
1887+
})
1888+
for i in range(step_idx, step_idx + len(remaining_steps)):
1889+
refined_indices.add(i)
1890+
1891+
_save_checkpoint(
1892+
task_dir, task_id, instruction,
1893+
completed_steps, remaining_steps,
1894+
step_plans, refined_indices,
1895+
steps_meta, step_idx,
1896+
)
1897+
else:
1898+
print(" WARNING: VLM returned no steps. Keeping previous plan.")
1899+
18711900
else:
18721901
# FEEDBACK: mid-recording step correction
18731902
print(" Taking fresh screenshot and refining remaining steps...")

0 commit comments

Comments
 (0)