Skip to content

Commit d498bca

Browse files
abrichrclaude
andcommitted
fix(agent): replace pyautogui.drag() with mouseDown/moveTo/mouseUp
pyautogui.drag() uses relative coordinates that compound with starting position errors, making it unreliable for small targets like LibreOffice fill handles (~3x3 pixels). Replace with explicit mouseDown/moveTo/mouseUp sequence with timing delays for reliable drag operations. Also adds drag case to _build_pixel_command() for the pixel_action() path. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 9b51aee commit d498bca

1 file changed

Lines changed: 29 additions & 2 deletions

File tree

  • openadapt_evals/adapters/waa

openadapt_evals/adapters/waa/live.py

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1026,7 +1026,7 @@ def _build_pixel_command(
10261026
10271027
Args:
10281028
action_type: One of "click", "double_click", "right_click", "type",
1029-
"key", "scroll", "done", "error", "wait".
1029+
"key", "scroll", "drag", "done", "error", "wait".
10301030
x: X pixel coordinate (absolute).
10311031
y: Y pixel coordinate (absolute).
10321032
text: Text to type (for action_type="type").
@@ -1072,6 +1072,24 @@ def _build_pixel_command(
10721072
if action_type == "scroll":
10731073
return f"import pyautogui; pyautogui.scroll(-3, x={px}, y={py})"
10741074

1075+
if action_type == "drag":
1076+
# drag needs end coordinates passed via text as "end_x,end_y"
1077+
ex, ey = px, py
1078+
if text:
1079+
parts = text.split(",")
1080+
if len(parts) == 2:
1081+
ex, ey = self._clamp_pixel_coords(int(parts[0]), int(parts[1]))
1082+
return (
1083+
f"import pyautogui; import time; "
1084+
f"pyautogui.moveTo({px}, {py}); "
1085+
f"time.sleep(0.3); "
1086+
f"pyautogui.mouseDown(button='left'); "
1087+
f"time.sleep(0.1); "
1088+
f"pyautogui.moveTo({ex}, {ey}, duration=1.0); "
1089+
f"time.sleep(0.1); "
1090+
f"pyautogui.mouseUp(button='left')"
1091+
)
1092+
10751093
logger.warning(f"Unknown pixel action type: {action_type}")
10761094
return None
10771095

@@ -1469,7 +1487,16 @@ def _translate_action(self, action: BenchmarkAction) -> str | None:
14691487
start_x, start_y = self._clamp_pixel_coords(int(start_x), int(start_y))
14701488
end_x, end_y = self._clamp_pixel_coords(int(end_x), int(end_y))
14711489

1472-
return f"import pyautogui; pyautogui.moveTo({start_x}, {start_y}); pyautogui.drag({end_x - start_x}, {end_y - start_y}, duration=0.5)"
1490+
return (
1491+
f"import pyautogui; import time; "
1492+
f"pyautogui.moveTo({start_x}, {start_y}); "
1493+
f"time.sleep(0.3); "
1494+
f"pyautogui.mouseDown(button='left'); "
1495+
f"time.sleep(0.1); "
1496+
f"pyautogui.moveTo({end_x}, {end_y}, duration=1.0); "
1497+
f"time.sleep(0.1); "
1498+
f"pyautogui.mouseUp(button='left')"
1499+
)
14731500

14741501
logger.warning(f"Unknown action type: {action.type}")
14751502
return None

0 commit comments

Comments
 (0)