update web agent; increase call limit to 10 and enhance browser tool actions with new coordinates support

Author: dcloud347

argus/tools/ask_web_agent.py

@@ -24,7 +24,7 @@ class AskWebAgentTool(Tool):
         "Ask the web-browsing agent to solve a task by navigating to a URL and describing what it sees. "
         "Use this to verify that a page or server is working as expected. "
         "Returns a description of the page appearance along with a screenshot image when available. "
-        "IMPORTANT: you may call this tool at most 5 times in total — use these calls wisely (e.g. only after the server is confirmed up, and after each meaningful fix)."
+        "IMPORTANT: you may call this tool at most 10 times in total — use these calls wisely (e.g. only after the server is confirmed up, and after each meaningful fix)."
     )
     parameters = {
         "type": "object",
@@ -45,7 +45,7 @@ class AskWebAgentTool(Tool):
         "required": ["url"],
     }
 
-    def __init__(self, web_agent: WebAgent, max_calls: int = 5) -> None:
+    def __init__(self, web_agent: WebAgent, max_calls: int = 10) -> None:
         self._web_agent = web_agent
         self._max_calls = max_calls
         self._call_count = 0
@@ -54,7 +54,7 @@ def execute(self, url: str, question: str = "", **_: Any) -> Content:  # type: i
         if self._call_count >= self._max_calls:
             return Content(
                 text=(
-                    f"[ask_web_agent] Call limit reached ({self._max_calls}/{self._max_calls}). "
+                    f"[ask_web_agent] Call limit reached ({self._call_count}/{self._max_calls}). "
                     "No more web agent calls are allowed. Use curl or inspect HTML to verify instead."
                 )
             )

argus/tools/browser.py

@@ -11,10 +11,12 @@
 class BrowserTool(Tool):
     """Control a headless Chromium browser via Playwright.
 
-    Supports navigate, screenshot, click, type, scroll, get_text, and
-    get_console_logs actions. The ``screenshot`` action returns a ``Content``
-    object with the PNG image attached; all other actions return text-only
-    ``Content``.
+    Supports navigate, screenshot, click, dblclick, hover, drag_and_drop,
+    type, press, scroll, reload, get_text, and get_console_logs actions.
+    For ``hover`` and ``click``, pass ``x``/``y`` pixel coordinates to target
+    canvas elements or precise positions; otherwise pass a CSS ``selector``.
+    The ``screenshot`` action returns a ``Content`` object with the PNG image
+    attached; all other actions return text-only ``Content``.
     """
 
     name = "browser"
@@ -41,15 +43,35 @@ class BrowserTool(Tool):
                     "reload",
                     "get_text",
                     "get_console_logs",
+                    "get_element_bounds",
                 ],
-                "description": "The browser action to perform.",
+                "description": (
+                    "The browser action to perform. "
+                    "Use get_element_bounds to get the pixel position and size of an element "
+                    "(e.g. a canvas), then use those coordinates with hover or click."
+                ),
             },
             "url": {"type": "string", "description": "URL to navigate to (navigate action)."},
             "selector": {
                 "type": "string",
                 "description": (
                     "CSS selector for the target element "
-                    "(click, dblclick, hover, drag_and_drop source, type, press actions)."
+                    "(click, dblclick, hover, drag_and_drop source, type, press actions). "
+                    "For hover and click, x/y coordinates take precedence over selector."
+                ),
+            },
+            "x": {
+                "type": "number",
+                "description": (
+                    "Absolute x coordinate in pixels for hover or click. "
+                    "Use instead of selector for canvas elements or precise positions."
+                ),
+            },
+            "y": {
+                "type": "number",
+                "description": (
+                    "Absolute y coordinate in pixels for hover or click. "
+                    "Use instead of selector for canvas elements or precise positions."
                 ),
             },
             "target_selector": {
@@ -112,6 +134,10 @@ async def _execute_async(self, action: str, **kwargs: Any) -> Content:
             return Content(text="Screenshot captured.", images=[b64])
 
         if action == "click":
+            x, y = kwargs.get("x"), kwargs.get("y")
+            if x is not None and y is not None:
+                await self._page.mouse.click(x, y)
+                return Content(text=f"Clicked at ({x}, {y})")
             selector = kwargs.get("selector", "")
             await self._page.click(selector)
             return Content(text=f"Clicked '{selector}'")
@@ -122,6 +148,10 @@ async def _execute_async(self, action: str, **kwargs: Any) -> Content:
             return Content(text=f"Double-clicked '{selector}'")
 
         if action == "hover":
+            x, y = kwargs.get("x"), kwargs.get("y")
+            if x is not None and y is not None:
+                await self._page.mouse.move(x, y)
+                return Content(text=f"Hovered at ({x}, {y})")
             selector = kwargs.get("selector", "")
             await self._page.hover(selector)
             return Content(text=f"Hovered over '{selector}'")
@@ -160,6 +190,23 @@ async def _execute_async(self, action: str, **kwargs: Any) -> Content:
         if action == "get_text":
             return Content(text=await self._page.inner_text("body"))
 
+        if action == "get_element_bounds":
+            selector = kwargs.get("selector", "")
+            el = await self._page.query_selector(selector)
+            if el is None:
+                return Content(text=f"[error: element not found for selector '{selector}']")
+            box = await el.bounding_box()
+            if box is None:
+                return Content(text=f"[error: element '{selector}' has no bounding box (hidden?)]")
+            return Content(
+                text=(
+                    f"Element '{selector}' bounds: "
+                    f"x={box['x']:.0f}, y={box['y']:.0f}, "
+                    f"width={box['width']:.0f}, height={box['height']:.0f}. "
+                    f"Center: ({box['x'] + box['width'] / 2:.0f}, {box['y'] + box['height'] / 2:.0f})"
+                )
+            )
+
         if action == "get_console_logs":
             if not self._console_logs:
                 return Content(text="No console output.")

argus/tools/checklist.py

@@ -39,6 +39,14 @@ class ChecklistTool(Tool):
                 "type": "integer",
                 "description": "0-based index of the item to complete. Required for action='complete'.",
             },
+            "note": {
+                "type": "string",
+                "description": (
+                    "Evidence that the step was completed. Required for action='complete'. "
+                    "For fix/implementation steps, paste the output of `git diff <filename>` here. "
+                    "For other steps, briefly describe what was done (e.g. 'found bug in src/core/ticks.js:42')."
+                ),
+            },
             "item": {
                 "type": "string",
                 "description": "Description of the new step to append. Required for action='add'.",
@@ -61,6 +69,8 @@ def render(self) -> str:
         for i, it in enumerate(self._items):
             mark = "x" if it["done"] else " "
             lines.append(f"  [{mark}] {i}. {it['text']}")
+            if it.get("note"):
+                lines.append(f"       note: {it['note']}")
         done = sum(1 for it in self._items if it["done"])
         lines.append(f"\nProgress: {done}/{len(self._items)} completed")
         return "\n".join(lines)
@@ -70,6 +80,7 @@ def execute(
         action: str,
         items: list[str] | None = None,
         index: int | None = None,
+        note: str | None = None,
         item: str | None = None,
     ) -> Content:
         if action == "init":
@@ -87,7 +98,16 @@ def execute(
                 return Content(
                     text=f"[error: index {index} out of range (valid: 0–{len(self._items) - 1})]"
                 )
+            if not note:
+                return Content(
+                    text=(
+                        "[error: 'note' is required for action='complete'. "
+                        "For fix/implementation steps, paste the `git diff` output. "
+                        "For other steps, briefly describe what was done.]"
+                    )
+                )
             self._items[index]["done"] = True
+            self._items[index]["note"] = note
             return Content(text=f"Item {index} marked as done.\n\n{self.render()}")
 
         if action == "add":

config.example.yaml

@@ -18,12 +18,14 @@ agent:
     Your plan should not be too long — ideally around 5-8 steps. Do NOT make a plan with 12+ steps as that is likely too detailed and will not be effective.
 
     ## Recommended Workflow Phases (follow in order)
+    The key principle is: **if you are stuck on a phase, skip it and move on — a partial fix is better than no fix.**
     You have a limited number of steps. Spend them wisely across these phases:
-      1. **Explore** (≤12 steps): Understand the codebase and locate the relevant code.
-      2. **Reproduce** (≤5 steps): Build the project if needed and create a minimal reproduction app.
-      3. **Fix** (≤8 steps): Edit the source code to resolve the issue.
-      4. **Verify** (≤10 steps): Confirm the fix works using ask_web_agent and test edge cases.
-    If you are still exploring after 15 steps, stop and move to reproduction immediately. Do NOT spend too many steps exploring!
+      1. **Explore** (≤24 steps): Understand the codebase and locate the relevant code.
+      2. **Reproduce** (≤10 steps): Build the project if needed and create a minimal reproduction app.
+      3. **Fix** (≤16 steps): Edit the source code to resolve the issue.
+      4. **Verify** (≤20 steps): Confirm the fix works using ask_web_agent and test edge cases.
+    If you are still exploring after 20 steps, stop and move to reproduction immediately. Do NOT spend too many steps exploring!
+    If you cannot get a reproduction server working within 10 steps, skip visual reproduction and go directly to Fix. Implement the source-code change and use `git diff` to confirm it — you do not need a running server to fix code.
 
     ## Important Boundaries
       - DO NOT MODIFY: Tests, configuration files