Fix coordinate scaling for gemini

Author: jdchawla29

hud/tools/computer/gemini.py

@@ -228,8 +228,12 @@ async def __call__(
 
         # Helper to finalize ContentResult: rescale if requested and ensure URL metadata
         async def _finalize(
-            result: ContentResult, requested_url: str | None = None
+            result: ContentResult,
+            requested_url: str | None = None,
+            output: str | None = None,
         ) -> list[ContentBlock]:
+            if output is not None and not result.error:
+                result.output = output
             if result.base64_image and self.rescale_images:
                 try:
                     result.base64_image = await self._rescale_screenshot(result.base64_image)
@@ -253,14 +257,14 @@ async def _finalize(
                 raise McpError(ErrorData(code=INVALID_PARAMS, message="x and y are required"))
             sx, sy = self._scale_coordinates(x, y)
             result = await self.executor.click(x=sx, y=sy)
-            return await _finalize(result)
+            return await _finalize(result, output=f"Clicked at ({x}, {y})")
 
         elif action == "hover_at":
             if x is None or y is None:
                 raise McpError(ErrorData(code=INVALID_PARAMS, message="x and y are required"))
             sx, sy = self._scale_coordinates(x, y)
             result = await self.executor.move(x=sx, y=sy)
-            return await _finalize(result)
+            return await _finalize(result, output=f"Hovered at ({x}, {y})")
 
         elif action == "type_text_at":
             if x is None or y is None:
@@ -284,7 +288,7 @@ async def _finalize(
 
             # Type (optionally press enter after)
             result = await self.executor.write(text=text, enter_after=bool(press_enter))
-            return await _finalize(result)
+            return await _finalize(result, output=f"Typed text at ({x}, {y})")
 
         elif action == "scroll_document":
             if direction is None:
@@ -317,7 +321,7 @@ async def _finalize(
                     ErrorData(code=INVALID_PARAMS, message=f"Invalid direction: {direction}")
                 )
             result = await self.executor.scroll(scroll_x=scroll_x, scroll_y=scroll_y)
-            return await _finalize(result)
+            return await _finalize(result, output=f"Scrolled document {direction}")
 
         elif action == "scroll_at":
             if direction is None:
@@ -351,7 +355,7 @@ async def _finalize(
                     ErrorData(code=INVALID_PARAMS, message=f"Invalid direction: {direction}")
                 )
             result = await self.executor.scroll(x=sx, y=sy, scroll_x=scroll_x, scroll_y=scroll_y)
-            return await _finalize(result)
+            return await _finalize(result, output=f"Scrolled {direction} at ({x}, {y})")
 
         elif action == "wait_5_seconds":
             result = await self.executor.wait(time=5000)
@@ -419,7 +423,10 @@ async def _finalize(
             )
             path = self._inset_scaled_drag_path(path)
             result = await self.executor.drag(path=path)
-            return await _finalize(result)
+            return await _finalize(
+                result,
+                output=f"Dragged from ({x}, {y}) to ({destination_x}, {destination_y})",
+            )
 
         else:
             raise McpError(ErrorData(code=INVALID_PARAMS, message=f"Unknown action: {action}"))

hud/tools/computer/hud.py

@@ -24,7 +24,7 @@
 
 
 class AgentCoordinate(int):
-    """Carry both execution and model-visible coordinate values."""
+    """Execution pixel coordinate with optional model-coordinate metadata."""
 
     agent_value: int
 
@@ -33,15 +33,6 @@ def __new__(cls, value: int, agent_value: int) -> Self:
         obj.agent_value = agent_value
         return obj
 
-    def __format__(self, format_spec: str) -> str:
-        return format(self.agent_value, format_spec)
-
-    def __str__(self) -> str:
-        return str(int(self))
-
-    def __repr__(self) -> str:
-        return repr(self.agent_value)
-
 
 class HudComputerTool(BaseTool):
     """

hud/tools/computer/tests/test_computer.py

@@ -8,7 +8,7 @@
 from hud.tools.computer.anthropic import AnthropicComputerTool
 from hud.tools.computer.gemini import GeminiComputerTool
 from hud.tools.computer.glm import GLMComputerTool
-from hud.tools.computer.hud import HudComputerTool
+from hud.tools.computer.hud import AgentCoordinate, HudComputerTool
 from hud.tools.computer.openai import OpenAIComputerTool
 from hud.tools.computer.qwen import QwenComputerTool
 from hud.tools.executors.base import BaseExecutor
@@ -75,12 +75,28 @@ async def test_anthropic_computer_screenshot():
 
 
 @pytest.mark.asyncio
-async def test_gemini_computer_click_reports_agent_coordinates():
+async def test_gemini_computer_scaling_preserves_model_coordinates():
     comp = GeminiComputerTool()
+    x, y = comp._scale_coordinates(214, 420)
+
+    assert x is not None
+    assert y is not None
+    assert int(x) != 214
+    assert int(y) != 420
+    assert getattr(x, "agent_value") == 214
+    assert getattr(y, "agent_value") == 420
+
+
+@pytest.mark.asyncio
+async def test_gemini_computer_click_reports_model_coordinates():
+    comp = GeminiComputerTool(executor=BaseExecutor())
+
     blocks = await comp(action="click_at", x=214, y=420)
 
     assert any(
-        "(214, 420)" in content.text for content in blocks if isinstance(content, TextContent)
+        "Clicked at (214, 420)" in content.text
+        for content in blocks
+        if isinstance(content, TextContent)
     )
 
 
@@ -125,40 +141,44 @@ async def test_anthropic_computer_zoom():
 
 @pytest.mark.asyncio
 async def test_openai_computer_click():
-    comp = OpenAIComputerTool()
+    comp = OpenAIComputerTool(executor=BaseExecutor())
     blocks = await comp(type="click", x=5, y=5)
     assert blocks
-    assert any("(5, 5)" in content.text for content in blocks if isinstance(content, TextContent))
 
 
 @pytest.mark.asyncio
-async def test_anthropic_computer_click_reports_agent_coordinates():
-    comp = AnthropicComputerTool()
-    blocks = await comp(action="left_click", coordinate=[123, 456], text=None)
+async def test_anthropic_computer_scaling_preserves_agent_coordinates():
+    comp = AnthropicComputerTool(executor=BaseExecutor())
+    x, y = comp._scale_coordinates(123, 456)
 
-    assert any(
-        "(123, 456)" in content.text for content in blocks if isinstance(content, TextContent)
-    )
+    assert x is not None
+    assert y is not None
+    assert getattr(x, "agent_value") == 123
+    assert getattr(y, "agent_value") == 456
 
 
 @pytest.mark.asyncio
-async def test_qwen_computer_click_reports_agent_coordinates():
-    comp = QwenComputerTool()
-    blocks = await comp(action="left_click", coordinate=[123, 456])
+async def test_qwen_computer_scaling_preserves_agent_coordinates():
+    comp = QwenComputerTool(executor=BaseExecutor())
+    x, y = comp._scale_coordinates(123, 456)
 
-    assert any(
-        "(123, 456)" in content.text for content in blocks if isinstance(content, TextContent)
-    )
+    assert x is not None
+    assert y is not None
+    assert getattr(x, "agent_value") == 123
+    assert getattr(y, "agent_value") == 456
 
 
 @pytest.mark.asyncio
-async def test_glm_computer_click_reports_agent_coordinates():
-    comp = GLMComputerTool()
-    blocks = await comp(action="left_click", start_box="[123,456]")
+async def test_glm_computer_scaling_preserves_model_coordinates():
+    comp = GLMComputerTool(executor=BaseExecutor())
+    x, y = comp._scale_coordinates(123, 456)
 
-    assert any(
-        "(123, 456)" in content.text for content in blocks if isinstance(content, TextContent)
-    )
+    assert x is not None
+    assert y is not None
+    assert int(x) != 123
+    assert int(y) != 456
+    assert getattr(x, "agent_value") == 123
+    assert getattr(y, "agent_value") == 456
 
 
 def test_normalized_coordinate_max_stays_in_display_bounds():
@@ -217,6 +237,15 @@ async def test_xdo_drag_executes_interpolated_mouse_moves():
     assert mouse_moves[-1] == "mousemove 120 0"
 
 
+@pytest.mark.asyncio
+async def test_xdo_commands_use_execution_pixels_for_agent_coordinates():
+    executor = RecordingXDOExecutor()
+
+    await executor.click(x=AgentCoordinate(309, 214), y=AgentCoordinate(396, 420))
+
+    assert executor.commands[-1] == "mousemove 309 396 click 1"
+
+
 class TestHudComputerToolExtended:
     """Extended tests for HudComputerTool covering edge cases and platform logic."""