add run_agent; implement general-purpose multimodal code agent with task handling and image support

update ask_web_agent; clarify response instructions for task completion
update shell; modify git diff command to reflect working directory

Author: dcloud347

argus/tools/ask_web_agent.py

@@ -69,7 +69,7 @@ def execute(self, url: str, question: str = "", **_: Any) -> Content:  # type: i
             "2. If the page is blank or shows errors, get console logs and report the exact error messages.\n"
             "3. If the question requires interaction (clicking buttons, typing values, scrolling), perform those actions and take screenshots to capture the result.\n"
             "4. Use get_text to read specific values if visual inspection is not precise enough.\n"
-            "5. End your response with a one-line verdict: PASS, FAIL, or PARTIAL — and explain why with specific evidence (values seen, errors found, etc.)."
+            "5. End your response with a clear answer and explain why with specific evidence (values seen, errors found, etc.)."
         )
         text, screenshot = self._web_agent.run_with_screenshot(Content(text=task))
         return Content(text=text, images=[screenshot] if screenshot else [])

argus/tools/shell.py

@@ -50,11 +50,11 @@ def __init__(
         self._port: int = port
 
     def get_patch(self) -> str | None:
-        """Return the git diff of /testbed via docker exec, bypassing the pexpect shell."""
+        """Return the git diff of the working directory via docker exec, bypassing the pexpect shell."""
         if self._container is None:
             return None
         result = self._container.exec_run(
-            ["bash", "-c", "git -C /testbed -c core.fileMode=false diff"]
+            ["bash", "-c", f"git -C {self.workdir} -c core.fileMode=false diff"]
         )
         return result.output.decode("utf-8", errors="replace")
 

configs/examples/general_purpose.yaml

@@ -0,0 +1,101 @@
+agent:
+  max_steps: 80
+  log_dir: logs/
+  enable_memory: false
+  summarize_threshold: 7000
+  system_prompt: |
+    You are a software engineer that interacts multiple times with a computer shell to solve programming tasks.
+    You operate in a REPL (Read-Eval-Print Loop) environment where you must issue exactly ONE shell tool call at a time.
+    IMPORTANT: To execute commands you MUST call the shell tool — do NOT just write bash code blocks in your response text, as those are not executed.
+    After each tool call, wait for the result before deciding the next step.
+    Please briefly explain your reasoning before each tool call.
+
+    ## Task Planning (MANDATORY FIRST STEP)
+    Before doing anything else, call the `checklist` tool with action='init' to make a detailed plan listing every step you plan to take.
+    After completing each step, you have to call `checklist` with action='complete' and the step's index to mark this step is done.
+    If you discover new steps mid-task, use action='add'.
+    If you find your plan is not working, you may make a new plan by calling `checklist` with action='init' again.
+    Your plan should not be too long — ideally around 5-8 steps. Do NOT make a plan with 12+ steps as that is likely too detailed and will not be effective.
+
+    ## Shell Usage Rules (CRITICAL)
+      - NEVER wrap commands in `bash -lc '...'` or `bash -c '...'`. You are already inside a bash session.
+      - When using grep/find, always search within /testbed (use `.` or `/testbed`). NEVER search from `..` or `/` as it will time out.
+      - Do NOT run blocking commands that take more than 120 seconds. Start servers/processes in the background (append `&`) and return immediately.
+      - If the command timed out, please try a different command or approach and continue. Do NOT repeat the same command again.
+
+    ## Efficient Code Reading
+      - Use `grep -n "pattern" -r src/ --include="*.js"` to find relevant lines first.
+      - Then read only the relevant section with `sed -n 'START,ENDp' file`. Keep ranges to ~100 lines maximum — this is a hard limit, not a suggestion.
+      - Do NOT use `cat` on source files. Do NOT use sed ranges larger than 100 lines. Outputs too long will be summarized and you will lose detail.
+      - Never search for something you already found — track what you know.
+
+    ## Editing Source Code
+      - Do NOT use `applypatch`, `patch`, or any tool not available in a standard bash environment.
+      - After each edit, verify the change took effect with `grep -n` or a small `sed -n` read.
+      - If an edit fails twice in a row, switch to a different approach immediately.
+      - NEVER use `node - << 'EOF'` or `python3 - << 'EOF'` to edit files — heredoc + scripting language causes multi-level escaping bugs where replacements silently fail. Use `sed -i` for simple substitutions.
+
+    ### Edit files with sed:
+      ```bash
+      # Replace all occurrences
+      sed -i 's/old_string/new_string/g' filename
+
+      # Replace only first occurrence
+      sed -i 's/old_string/new_string/' filename
+
+      # Replace first occurrence on line 1
+      sed -i '1s/old_string/new_string/' filename
+
+      # Replace all occurrences in lines 1-10
+      sed -i '1,10s/old_string/new_string/g' filename
+
+    ## Using ask_web_agent
+      - Use ask_web_agent when the task requires visual inspection of a web page (e.g. checking rendered output, UI layout, or interactive behaviour).
+      - Do NOT use it to check whether a server is running — use `curl -s -o /dev/null -w "%{http_code}" http://localhost:PORT` instead.
+      - Ask specific, targeted questions describing exactly what you expect to see. Vague questions produce vague answers.
+      - After every call, read the response carefully. If it reports a problem, adjust your approach before calling again.
+
+    ## MANDATORY: Responding to ask_web_agent Errors (CRITICAL RULE)
+      - After EVERY ask_web_agent call, you MUST read its response.
+      - If the response indicates a FAIL or PARTIAL result, you MUST immediately analyze the failure reason and adjust your approach accordingly.
+      - Ignoring a FAIL/PARTIAL result and continuing anyway is a critical mistake — do not do it.
+
+    ## Tool Output
+      - If a tool result starts with "[Output was ... chars — summarized below]", the raw output was too long and has been automatically summarized for you. Treat the summary as the actual result and continue accordingly.
+
+  LLM:
+    provider: openai
+    model: gpt-4o
+    api_key: ""
+    base_url: ""
+    temperature: 0.5
+  Memory:
+    user_id: default
+    base_url: http://localhost:1995/api/v1
+    api_key: null
+    retrieve_method: hybrid
+    top_k: 5
+
+web_agent:
+  headless: true
+  max_steps: 10
+  log_dir: logs/
+  system_prompt: |
+    You are a web browsing agent. Navigate to URLs, take screenshots, and report what you see clearly and concisely.
+
+    - Always navigate to the URL first, then take a screenshot before drawing conclusions.
+    - If the page is unreachable or blank, check console logs and report immediately.
+    - If the task involves interaction (clicking, typing, scrolling), perform the action and take a screenshot after.
+    - Base your judgement on visual appearance and behaviour, not source code.
+  LLM:
+    provider: openai
+    model: gpt-4o
+    api_key: ""
+    base_url: ""
+    temperature: null
+  Memory:
+    user_id: default
+    base_url: http://localhost:1995/api/v1
+    api_key: null
+    retrieve_method: hybrid
+    top_k: 5

config.example.yaml configs/examples/swebench_multimodal.yamlconfig.example.yaml renamed to configs/examples/swebench_multimodal.yaml

@@ -0,0 +1,184 @@
+#!/usr/bin/env python3
+"""General-purpose multimodal code agent entry point."""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import logging
+import random
+from pathlib import Path
+
+import requests
+
+from argus import Agent, AskWebAgentTool, EverMindMemory, WebAgent
+from argus.config import AgentConfig, Config, LLMConfig, WebAgentConfig
+from argus.data.content import Content
+from argus.llm.anthropic import AnthropicClient
+from argus.llm.base import LLMClient
+from argus.llm.openai import OpenAIClient
+from argus.tools.checklist import ChecklistTool
+from argus.tools.shell import ShellTool
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
+logger = logging.getLogger(__name__)
+
+DEFAULT_DOCKER_IMAGE = "python:3.11"
+
+
+def _load_image(source: str) -> str | None:
+    """Load an image from a local path or URL and return base64-encoded bytes."""
+    if source.startswith("http://") or source.startswith("https://"):
+        try:
+            resp = requests.get(source, timeout=15)
+            resp.raise_for_status()
+            return base64.b64encode(resp.content).decode()
+        except Exception as exc:
+            logger.warning("Failed to download image %s: %s", source, exc)
+            return None
+    path = Path(source)
+    if not path.exists():
+        logger.warning("Image file not found: %s", source)
+        return None
+    return base64.b64encode(path.read_bytes()).decode()
+
+
+def _build_task(text: str, port: int, max_steps: int, images: list[str]) -> Content:
+    """Build the task Content, injecting runtime context and optionally attaching images."""
+    preamble = (
+        f"You have {max_steps} steps available. Use them wisely.\n"
+        f"A port has been reserved for you on the host: {port}. "
+        f"If you need to run a server or web service, bind it to {port} inside the container.\n\n"
+    )
+    text = preamble + text
+    if not images:
+        return Content(text=text)
+    loaded = [b64 for src in images if (b64 := _load_image(src)) is not None]
+    if loaded:
+        text += f"\n\n{len(loaded)} image(s) are attached after this text block."
+    return Content(text=text, images=loaded)
+
+
+def _build_llm(cfg: LLMConfig) -> LLMClient:
+    provider = cfg.provider.lower()
+    if provider == "anthropic":
+        return AnthropicClient(
+            model=cfg.model,
+            api_key=cfg.api_key,
+            base_url=cfg.base_url or None,
+            temperature=cfg.temperature,
+        )
+    if provider == "openai":
+        return OpenAIClient(
+            model=cfg.model,
+            api_key=cfg.api_key,
+            base_url=cfg.base_url or None,
+            temperature=cfg.temperature,
+        )
+    raise ValueError(f"Unknown LLM provider: {cfg.provider!r}. Supported: openai, anthropic")
+
+
+def _build_web_agent(cfg: WebAgentConfig) -> WebAgent:
+    llm = _build_llm(cfg.LLM)
+    log_dir = Path(cfg.log_dir) if cfg.log_dir else None
+    return WebAgent(
+        llm=llm,
+        system_prompt=cfg.system_prompt,
+        max_steps=cfg.max_steps,
+        log_dir=log_dir,
+        headless=cfg.headless,
+    )
+
+
+def _build_agent(
+    cfg: AgentConfig, docker_image: str, workdir: str, port: int, web_agent: WebAgent
+) -> Agent:
+    shell = ShellTool(docker_image, workdir=workdir, remove_on_cleanup=False, port=port)
+    log_dir = Path(cfg.log_dir) if cfg.log_dir else None
+    llm = _build_llm(cfg.LLM)
+
+    memory = None
+    if cfg.enable_memory:
+        memory = EverMindMemory(
+            user_id=cfg.Memory.user_id,
+            base_url=cfg.Memory.base_url,
+            api_key=cfg.Memory.api_key,
+            retrieve_method=cfg.Memory.retrieve_method,
+            top_k=cfg.Memory.top_k,
+        )
+
+    return Agent(
+        llm=llm,
+        tools=[ChecklistTool(), shell, AskWebAgentTool(web_agent)],
+        system_prompt=cfg.system_prompt,
+        max_steps=cfg.max_steps,
+        log_dir=log_dir,
+        memory=memory,
+        summarize_threshold=cfg.summarize_threshold,
+    )
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Run the Argus multimodal code agent on an arbitrary task."
+    )
+    parser.add_argument("--config", help="Path to config.yaml", required=True)
+    parser.add_argument(
+        "--docker",
+        default=DEFAULT_DOCKER_IMAGE,
+        metavar="IMAGE",
+        help=f"Docker image to use (default: {DEFAULT_DOCKER_IMAGE})",
+    )
+    parser.add_argument(
+        "--workdir",
+        default="/workspace",
+        metavar="DIR",
+        help="Working directory inside the container (default: /workspace)",
+    )
+    parser.add_argument("--task", metavar="TEXT", help="Task description.", required=True)
+    parser.add_argument(
+        "--images",
+        nargs="*",
+        metavar="SRC",
+        help="Image files or URLs to attach to the task (local paths or http(s):// URLs).",
+    )
+    parser.add_argument(
+        "--port",
+        type=int,
+        default=None,
+        metavar="PORT",
+        help="Host port to expose from the container. Randomly assigned if omitted.",
+    )
+    args = parser.parse_args()
+
+    # Resolve task text
+    task_text = args.task
+
+    port = args.port if args.port is not None else random.randint(10000, 65535)
+
+    cfg = Config.from_yaml(Path(args.config))
+    task = _build_task(task_text, port, cfg.agent.max_steps, args.images or [])
+
+    logger.info("Docker image : %s", args.docker)
+    logger.info("Workdir      : %s", args.workdir)
+    logger.info("Port         : %d", port)
+    logger.info("Max steps    : %d", cfg.agent.max_steps)
+    logger.info("Images       : %d", len(task.images))
+
+    web_agent = _build_web_agent(cfg.web_agent)
+    agent = _build_agent(cfg.agent, args.docker, args.workdir, port, web_agent)
+
+    agent.run(task)
+
+    shell = next((t for t in agent.tool_list if isinstance(t, ShellTool)), None)
+    if shell is not None:
+        patch = shell.get_patch()
+        if patch:
+            logger.info("=== git diff (patch) ===\n%s", patch)
+        shell.cleanup()
+
+    logger.info("Done.")
+
+
+if __name__ == "__main__":
+    main()

run_agent.py

@@ -129,7 +129,7 @@ def _build_llm(cfg: LLMConfig) -> LLMClient:
 def _build_agent(cfg: AgentConfig, instance: dict, port: int, web_agent: WebAgent) -> Agent:
     """Instantiate the agent with the appropriate tools, LLM, and memory (if enabled)."""
     shell = ShellTool(
-        _docker_image(instance), workdir="/testbed", remove_on_cleanup=False, port=port
+        _docker_image(instance), workdir="/testbed", remove_on_cleanup=True, port=port
     )
     log_dir = Path(cfg.log_dir) / instance["instance_id"] if cfg.log_dir else None
     llm = _build_llm(cfg.LLM)