Merge pull request #404 from hud-evals/yang/gemini-tool-fix-drag-and-drop

Yang/gemini tool fix drag and drop

Author: jdchawla29

hud/tools/computer/gemini.py

@@ -22,6 +22,9 @@
 
 logger = logging.getLogger(__name__)
 
+GEMINI_DRAG_INSET = 25
+DISPLAY_DRAG_INSET_PIXELS = 20
+
 SUPPORTED_GEMINI_COMPUTER_USE_MODELS = (
     "gemini-2.5-computer-use-preview-10-2025",
     "gemini-3-flash-preview",
@@ -168,6 +171,30 @@ def __init__(
             **kwargs,
         )
 
+    def _inset_drag_coordinate(self, value: int) -> int:
+        """Keep Gemini normalized drag endpoints away from display edges."""
+        if (
+            self.coordinate_space is None
+            or not isinstance(value, int | float)
+            or not 0 <= value <= self.coordinate_space
+        ):
+            return value
+
+        max_value = max(self.coordinate_space - GEMINI_DRAG_INSET, GEMINI_DRAG_INSET)
+        return min(max(value, GEMINI_DRAG_INSET), max_value)
+
+    def _inset_scaled_drag_path(self, path: list[tuple[int, int]]) -> list[tuple[int, int]]:
+        """Keep scaled drag points inside the display so they do not hit OS/window edges."""
+        max_x = max(self.environment_width - 1 - DISPLAY_DRAG_INSET_PIXELS, 0)
+        max_y = max(self.environment_height - 1 - DISPLAY_DRAG_INSET_PIXELS, 0)
+        return [
+            (
+                min(max(int(x), DISPLAY_DRAG_INSET_PIXELS), max_x),
+                min(max(int(y), DISPLAY_DRAG_INSET_PIXELS), max_y),
+            )
+            for x, y in path
+        ]
+
     async def __call__(
         self,
         action: str = ACTION_FIELD,
@@ -381,7 +408,16 @@ async def _finalize(
                         message="x, y, destination_x, and destination_y are required",
                     )
                 )
-            path = self._scale_path([(x, y), (destination_x, destination_y)])
+            path = self._scale_path(
+                [
+                    (self._inset_drag_coordinate(x), self._inset_drag_coordinate(y)),
+                    (
+                        self._inset_drag_coordinate(destination_x),
+                        self._inset_drag_coordinate(destination_y),
+                    ),
+                ]
+            )
+            path = self._inset_scaled_drag_path(path)
             result = await self.executor.drag(path=path)
             return await _finalize(result)
 

hud/tools/computer/tests/test_computer.py

@@ -12,7 +12,28 @@
 from hud.tools.computer.openai import OpenAIComputerTool
 from hud.tools.computer.qwen import QwenComputerTool
 from hud.tools.executors.base import BaseExecutor
-from hud.tools.types import Coordinate
+from hud.tools.executors.xdo import XDOExecutor
+from hud.tools.types import ContentResult, Coordinate
+
+
+class RecordingXDOExecutor(XDOExecutor):
+    def __init__(self):
+        super().__init__()
+        self.commands: list[str] = []
+
+    async def execute(self, command: str, take_screenshot: bool = True):
+        self.commands.append(command)
+        return ContentResult(output=command)
+
+
+class RecordingExecutor(BaseExecutor):
+    def __init__(self):
+        super().__init__()
+        self.drag_paths: list[list[tuple[int, int]]] = []
+
+    async def drag(self, path, pattern=None, hold_keys=None, take_screenshot=True):
+        self.drag_paths.append(path)
+        return await super().drag(path, pattern, hold_keys, take_screenshot=False)
 
 
 @pytest.mark.asyncio
@@ -151,6 +172,51 @@ def test_normalized_coordinate_max_stays_in_display_bounds():
     assert int(y) <= comp.environment_height - 1
 
 
+def test_drag_path_interpolation_adds_intermediate_points():
+    executor = BaseExecutor()
+
+    path = executor._interpolate_drag_path([(0, 0), (120, 0)])
+
+    assert path[0] == (0, 0)
+    assert path[-1] == (120, 0)
+    assert len(path) == 11
+
+
+@pytest.mark.asyncio
+async def test_gemini_drag_clamps_edges_and_interpolates_executor_path():
+    executor = RecordingExecutor()
+    comp = GeminiComputerTool(executor=executor, width=1400, height=850)
+
+    blocks = await comp(
+        action="drag_and_drop",
+        x=0,
+        y=500,
+        destination_x=1000,
+        destination_y=500,
+    )
+
+    assert blocks
+    path = executor.drag_paths[0]
+    assert path[0][0] >= 20
+    assert path[-1][0] <= comp.environment_width - 1 - 20
+
+    interpolated = executor._interpolate_drag_path(path)
+    assert len(interpolated) > 2
+
+
+@pytest.mark.asyncio
+async def test_xdo_drag_executes_interpolated_mouse_moves():
+    executor = RecordingXDOExecutor()
+
+    result = await executor.drag([(0, 0), (120, 0)], take_screenshot=False)
+
+    mouse_moves = [command for command in executor.commands if command.startswith("mousemove ")]
+    assert result.output == "Dragged along 11 points"
+    assert len(mouse_moves) == 11
+    assert mouse_moves[0] == "mousemove 0 0"
+    assert mouse_moves[-1] == "mousemove 120 0"
+
+
 class TestHudComputerToolExtended:
     """Extended tests for HudComputerTool covering edge cases and platform logic."""
 

hud/tools/executors/base.py

@@ -3,7 +3,9 @@
 import asyncio
 import base64
 import logging
+import math
 from io import BytesIO
+from itertools import pairwise
 from typing import TYPE_CHECKING, Literal, TypeAlias
 
 from hud.tools.types import ContentResult
@@ -13,6 +15,8 @@
 
 logger = logging.getLogger(__name__)
 
+DRAG_STEP_PIXELS = 12
+
 
 class BaseExecutor:
     """
@@ -42,6 +46,31 @@ def __init__(self, display_num: int | None = None) -> None:
         self._screenshot_delay = 0.5
         logger.info("BaseExecutor initialized")
 
+    def _interpolate_drag_path(
+        self, path: list[tuple[int, int]], step_pixels: int = DRAG_STEP_PIXELS
+    ) -> list[tuple[int, int]]:
+        """Fill long drag segments with intermediate points for pointer-delta UIs."""
+        if len(path) < 2:
+            return path
+
+        interpolated: list[tuple[int, int]] = [path[0]]
+        for start, end in pairwise(path):
+            start_x, start_y = start
+            end_x, end_y = end
+            distance = math.hypot(end_x - start_x, end_y - start_y)
+            steps = max(1, math.ceil(distance / max(step_pixels, 1)))
+
+            for step in range(1, steps + 1):
+                t = step / steps
+                point = (
+                    round(start_x + (end_x - start_x) * t),
+                    round(start_y + (end_y - start_y) * t),
+                )
+                if point != interpolated[-1]:
+                    interpolated.append(point)
+
+        return interpolated
+
     # ===== Core CLA Actions =====
 
     async def click(

hud/tools/executors/pyautogui.py

@@ -478,33 +478,26 @@ async def drag(
             return ContentResult(error="Drag path must have at least 2 points")
 
         try:
+            drag_path = self._interpolate_drag_path(path)
+
             # Hold keys if specified
             self._hold_keys_context(hold_keys)
 
             try:
                 # Move to start
-                start_x, start_y = path[0]
+                start_x, start_y = drag_path[0]
                 self.pyautogui.moveTo(start_x, start_y)
 
-                # Handle multi-point drag
-                if len(path) == 2:
-                    # Simple drag
-                    end_x, end_y = path[1]
-                    self.pyautogui.dragTo(end_x, end_y, duration=0.5, button="left")
-                    result = ContentResult(
-                        output=f"Dragged from ({start_x}, {start_y}) to ({end_x}, {end_y})"
-                    )
-                else:
-                    # Multi-point drag
-                    self.pyautogui.mouseDown(button="left")
-                    for i, (x, y) in enumerate(path[1:], 1):
-                        duration = 0.1
-                        if pattern and i - 1 < len(pattern):
-                            duration = pattern[i - 1] / 1000.0  # Convert ms to seconds
-                        self.pyautogui.moveTo(x, y, duration=duration)
-                    self.pyautogui.mouseUp(button="left")
-
-                    result = ContentResult(output=f"Dragged along {len(path)} points")
+                # Move through enough points for pointer-delta-sensitive UIs.
+                self.pyautogui.mouseDown(button="left")
+                for i, (x, y) in enumerate(drag_path[1:], 1):
+                    duration = 0.01
+                    if pattern and i - 1 < len(pattern):
+                        duration = pattern[i - 1] / 1000.0  # Convert ms to seconds
+                    self.pyautogui.moveTo(x, y, duration=duration)
+                self.pyautogui.mouseUp(button="left")
+
+                result = ContentResult(output=f"Dragged along {len(drag_path)} points")
 
                 if hold_keys:
                     result = ContentResult(output=f"{result.output} while holding {hold_keys}")

hud/tools/executors/tests/test_pyautogui_executor.py

@@ -116,15 +116,22 @@ async def test_drag_with_pyautogui(self):
         """Test drag when pyautogui is available."""
         executor = PyAutoGUIExecutor()
 
-        with patch("pyautogui.dragTo") as mock_drag:
+        with (
+            patch("pyautogui.moveTo") as mock_move,
+            patch("pyautogui.mouseDown") as mock_down,
+            patch("pyautogui.mouseUp") as mock_up,
+        ):
             # drag expects a path (list of coordinate tuples)
             path = [(100, 100), (300, 400)]
             result = await executor.drag(path)
 
             assert isinstance(result, ContentResult)
             assert result.output and "Dragged" in result.output
-            # Implementation uses dragTo to move to each point
-            mock_drag.assert_called()
+            # Implementation holds the button and moves through interpolated points.
+            mock_move.assert_any_call(100, 100)
+            assert mock_move.call_count > len(path)
+            mock_down.assert_called_once_with(button="left")
+            mock_up.assert_called_once_with(button="left")
 
     @pytest.mark.asyncio
     async def test_wait(self):

hud/tools/executors/xdo.py

@@ -425,20 +425,24 @@ async def drag(
         if len(path) < 2:
             return ContentResult(error="Drag path must have at least 2 points")
 
+        drag_path = self._interpolate_drag_path(path)
+
         # Hold keys if specified
         await self._hold_keys_context(hold_keys)
 
         try:
             # Start drag
-            start_x, start_y = path[0]
+            start_x, start_y = drag_path[0]
             await self.execute(f"mousemove {start_x} {start_y}", take_screenshot=False)
             await self.execute("mousedown 1", take_screenshot=False)
 
             # Move through intermediate points
-            for i, (x, y) in enumerate(path[1:], 1):
+            for i, (x, y) in enumerate(drag_path[1:], 1):
                 # Apply delay if pattern is specified
                 if pattern and i - 1 < len(pattern):
                     await asyncio.sleep(pattern[i - 1] / 1000.0)  # Convert ms to seconds
+                else:
+                    await asyncio.sleep(0.008)
 
                 await self.execute(f"mousemove {x} {y}", take_screenshot=False)
 
@@ -449,10 +453,10 @@ async def drag(
             if take_screenshot:
                 screenshot = await self.screenshot()
                 result = ContentResult(
-                    output=f"Dragged along {len(path)} points", base64_image=screenshot
+                    output=f"Dragged along {len(drag_path)} points", base64_image=screenshot
                 )
             else:
-                result = ContentResult(output=f"Dragged along {len(path)} points")
+                result = ContentResult(output=f"Dragged along {len(drag_path)} points")
 
         finally:
             # Release held keys