Merge branch 'master' of https://github.com/elecvoid243/AstrBot

Author: elecvoid243

.github/workflows/build-docs.yml

@@ -13,7 +13,7 @@ jobs:
       - name: checkout
         uses: actions/checkout@v6
       - name: Setup pnpm
-        uses: pnpm/action-setup@v5.0.0
+        uses: pnpm/action-setup@v6.0.3
         with:
           version: 10.28.2
       - name: Setup Node.js

.github/workflows/dashboard_ci.yml

@@ -15,7 +15,7 @@ jobs:
         uses: actions/checkout@v6
 
       - name: Setup pnpm
-        uses: pnpm/action-setup@v5.0.0
+        uses: pnpm/action-setup@v6.0.3
         with:
           version: 10.28.2
 

.github/workflows/release.yml

@@ -51,7 +51,7 @@ jobs:
           echo "tag=$tag" >> "$GITHUB_OUTPUT"
 
       - name: Setup pnpm
-        uses: pnpm/action-setup@v5.0.0
+        uses: pnpm/action-setup@v6.0.3
         with:
           version: 10.28.2
 

astrbot/core/agent/runners/tool_loop_agent_runner.py

@@ -1322,7 +1322,7 @@ async def _resolve_tool_exec(
                     model=self.req.model,
                     session_id=self.req.session_id,
                     extra_user_content_parts=self.req.extra_user_content_parts,
-                    tool_choice="required",
+                    # tool_choice="required",
                     abort_signal=self._abort_signal,
                 )
                 if requery_resp:
@@ -1348,7 +1348,7 @@ async def _resolve_tool_exec(
                         model=self.req.model,
                         session_id=self.req.session_id,
                         extra_user_content_parts=self.req.extra_user_content_parts,
-                        tool_choice="required",
+                        # tool_choice="required",
                         abort_signal=self._abort_signal,
                     )
                     if repair_resp:

astrbot/core/agent/tool_image_cache.py

@@ -52,7 +52,6 @@ def __init__(self) -> None:
         self._initialized = True
         self._cache_dir = os.path.join(get_astrbot_temp_path(), self.CACHE_DIR_NAME)
         os.makedirs(self._cache_dir, exist_ok=True)
-        logger.debug(f"ToolImageCache initialized, cache dir: {self._cache_dir}")
 
     def _get_file_extension(self, mime_type: str) -> str:
         """Get file extension from MIME type."""

astrbot/core/astr_agent_tool_exec.py

@@ -32,6 +32,9 @@
 from astrbot.core.provider.entites import ProviderRequest
 from astrbot.core.provider.register import llm_tools
 from astrbot.core.tools.computer_tools import (
+    CuaKeyboardTypeTool,
+    CuaMouseClickTool,
+    CuaScreenshotTool,
     ExecuteShellTool,
     FileDownloadTool,
     FileEditTool,
@@ -187,7 +190,9 @@ def _get_runtime_computer_tools(
         cls,
         runtime: str,
         tool_mgr,
+        booter: str | None = None,
     ) -> dict[str, FunctionTool]:
+        booter = "" if booter is None else str(booter).lower()
         if runtime == "sandbox":
             shell_tool = tool_mgr.get_builtin_tool(ExecuteShellTool)
             python_tool = tool_mgr.get_builtin_tool(PythonTool)
@@ -197,7 +202,7 @@ def _get_runtime_computer_tools(
             write_tool = tool_mgr.get_builtin_tool(FileWriteTool)
             edit_tool = tool_mgr.get_builtin_tool(FileEditTool)
             grep_tool = tool_mgr.get_builtin_tool(GrepTool)
-            return {
+            tools = {
                 shell_tool.name: shell_tool,
                 python_tool.name: python_tool,
                 upload_tool.name: upload_tool,
@@ -207,6 +212,18 @@ def _get_runtime_computer_tools(
                 edit_tool.name: edit_tool,
                 grep_tool.name: grep_tool,
             }
+            if booter == "cua":
+                screenshot_tool = tool_mgr.get_builtin_tool(CuaScreenshotTool)
+                mouse_click_tool = tool_mgr.get_builtin_tool(CuaMouseClickTool)
+                keyboard_type_tool = tool_mgr.get_builtin_tool(CuaKeyboardTypeTool)
+                tools.update(
+                    {
+                        screenshot_tool.name: screenshot_tool,
+                        mouse_click_tool.name: mouse_click_tool,
+                        keyboard_type_tool.name: keyboard_type_tool,
+                    }
+                )
+            return tools
         if runtime == "local":
             shell_tool = tool_mgr.get_builtin_tool(ExecuteShellTool)
             python_tool = tool_mgr.get_builtin_tool(LocalPythonTool)
@@ -243,6 +260,7 @@ def _build_handoff_toolset(
         runtime_computer_tools = cls._get_runtime_computer_tools(
             runtime,
             tool_mgr,
+            provider_settings.get("sandbox", {}).get("booter"),
         )
 
         # Keep persona semantics aligned with the main agent: tools=None means

astrbot/core/astr_main_agent.py

@@ -48,6 +48,9 @@
     BrowserExecTool,
     CreateSkillCandidateTool,
     CreateSkillPayloadTool,
+    CuaKeyboardTypeTool,
+    CuaMouseClickTool,
+    CuaScreenshotTool,
     EvaluateSkillCandidateTool,
     ExecuteShellTool,
     FileDownloadTool,
@@ -1108,6 +1111,22 @@ def _apply_sandbox_tools(
         req.func_tool.add_tool(tool_mgr.get_builtin_tool(RollbackSkillReleaseTool))
         req.func_tool.add_tool(tool_mgr.get_builtin_tool(SyncSkillReleaseTool))
 
+    if booter == "cua":
+        req.system_prompt += (
+            "\n[CUA Desktop Control]\n"
+            "Use `astrbot_execute_shell` with `background=true` to launch GUI apps. "
+            'Use Firefox for browser tasks, for example `firefox "https://example.com"`. '
+            "After each visible step, call `astrbot_cua_screenshot` with "
+            "`send_to_user=true` and `return_image_to_llm=true` so the user can "
+            "monitor progress. When typing, inspect the screenshot first and confirm "
+            "the target field is focused and empty or safe to append to. Use "
+            "`astrbot_cua_mouse_click` for coordinates and `astrbot_cua_keyboard_type` "
+            "for text input; use text=`\\n` for Enter.\n"
+        )
+        req.func_tool.add_tool(tool_mgr.get_builtin_tool(CuaScreenshotTool))
+        req.func_tool.add_tool(tool_mgr.get_builtin_tool(CuaMouseClickTool))
+        req.func_tool.add_tool(tool_mgr.get_builtin_tool(CuaKeyboardTypeTool))
+
     req.system_prompt = f"{req.system_prompt or ''}\n{SANDBOX_MODE_PROMPT}\n"
 
 

astrbot/core/computer/booters/base.py

@@ -1,6 +1,7 @@
 from ..olayer import (
     BrowserComponent,
     FileSystemComponent,
+    GUIComponent,
     PythonComponent,
     ShellComponent,
 )
@@ -29,9 +30,21 @@ def capabilities(self) -> tuple[str, ...] | None:
     def browser(self) -> BrowserComponent | None:
         return None
 
+    @property
+    def gui(self) -> GUIComponent | None:
+        return None
+
     async def boot(self, session_id: str) -> None: ...
 
-    async def shutdown(self) -> None: ...
+    async def shutdown(self, **kwargs) -> None:
+        """Shut down the computer sandbox.
+
+        Subclasses may accept extra keyword arguments for
+        type-specific cleanup (e.g. ``delete_sandbox`` for
+        ShipyardNeoBooter).  The default implementation ignores
+        them.
+        """
+        ...
 
     async def upload_file(self, path: str, file_name: str) -> dict:
         """Upload file to the computer.