Merge remote-tracking branch 'origin/main' into SOLENG-360-refactor/askui-controller-multi-target

Author: mlikasam-askui

pdm.lock

@@ -5,7 +5,8 @@ authors = [
     {name = "askui GmbH", email = "info@askui.com"},
 ]
 dependencies = [
-    "askui-agent-os>=26.1.1",
+    'askui-agent-os>=26.4.1; sys_platform == "darwin"',
+    'askui-agent-os>=26.5.1; sys_platform != "darwin"',
     "anthropic>=0.86.0",
     "fastapi>=0.115.12",
     "fastmcp>=2.3.0",

pyproject.toml

@@ -79,7 +79,9 @@ class ComputerAgent(Agent):
             `display`.
         settings (AgentSettings | None, optional): Provider-based model settings. If `None`, uses the default AskUI model stack.
         retry (Retry, optional): The retry instance to use for retrying failed actions. Defaults to `ConfigurableRetry` with exponential backoff. Currently only supported for `locate()` method.
-        act_tools (list[Tool] | None, optional): Additional tools to make available for the `act()` method.
+        act_tools (list[Tool] | None, optional): Additional tools to make available for
+            the `act()` method for every call. Same tools can instead be passed per call
+            via `act(..., tools=[...])` (see example below).
 
     Example:
         Single local machine (the default):
@@ -136,6 +138,26 @@ class ComputerAgent(Agent):
             )
             agent.act("Kick off a release build on the build server")
         ```
+
+    Example (optional tools for `act()`):
+        Register tools from `askui.tools.store` (or your own `Tool` implementations)
+        either on the agent so they apply to all `act()` calls, or only for one call.
+
+        ```python
+        from askui import ComputerAgent
+        from askui.tools.store.computer import ComputerSaveScreenshotTool
+
+        with ComputerAgent(
+            act_tools=[ComputerSaveScreenshotTool(base_dir="/path/to/screenshots")]
+        ) as agent:
+            agent.act("Take a screenshot and save it as demo/demo.png")
+
+        with ComputerAgent() as agent:
+            agent.act(
+                "Take a screenshot and save it as demo/demo.png",
+                tools=[ComputerSaveScreenshotTool(base_dir="/path/to/screenshots")],
+            )
+        ```
     """
 
     @telemetry.record_call(
@@ -548,8 +570,8 @@ def cli(
 
             with ComputerAgent() as agent:
                 # Use for Windows
-                agent.cli(r'start "" "C:\Program Files\VideoLAN\VLC\vlc.exe"') # Start in VLC non-blocking
-                agent.cli(r'"C:\Program Files\VideoLAN\VLC\vlc.exe"') # Start in VLC blocking
+                agent.cli(r'start "" "C:\\Program Files\\VideoLAN\\VLC\\vlc.exe"') # Start in VLC non-blocking
+                agent.cli(r'"C:\\Program Files\\VideoLAN\\VLC\\vlc.exe"') # Start in VLC blocking
 
                 # Mac
                 agent.cli("open -a chrome")  # Open Chrome non-blocking for mac

src/askui/computer_agent.py

@@ -371,18 +371,18 @@ def _add_message(self, message: MessageParam) -> None:
         Args:
             message: Message to add
         """
+        # Report to reporter
+        self._reporter.add_message(
+            self.current_speaker.name, message.model_dump(mode="json")
+        )
+
         if not self._truncation_strategy:
             logger.error("No truncation strategy, cannot add message")
             return
 
         # Add to truncation strategy
         self._truncation_strategy.append_message(message)
 
-        # Report to reporter
-        self._reporter.add_message(
-            self.current_speaker.name, message.model_dump(mode="json")
-        )
-
     @tracer.start_as_current_span("_handle_continue_conversation")
     def _handle_continue_conversation(self, result: SpeakerResult) -> bool:
         """Handle speaker result status and determine if loop should continue.

src/askui/models/shared/conversation.py

@@ -21,11 +21,11 @@
 from askui.models.shared.token_counter import SimpleTokenCounter
 from askui.models.shared.tools import ToolCollection
 from askui.prompts.truncation import SUMMARIZE_INSTRUCTION_PROMPT
+from askui.reporting import Reporter
 
 if TYPE_CHECKING:
     from askui.callbacks.conversation_callback import ConversationCallback
     from askui.models.shared.conversation import Conversation
-    from askui.reporting import Reporter
 
 logger = logging.getLogger(__name__)
 
@@ -81,6 +81,7 @@ def _summarize_message_history(
     system: SystemPrompt | None = None,
     tools: ToolCollection | None = None,
     provider_options: dict[str, Any] | None = None,
+    reporter: Reporter | None = None,
 ) -> MessageParam:
     """Ask the VLM to summarize the conversation history.
 
@@ -99,6 +100,7 @@ def _summarize_message_history(
             Required for cache hits on the prefix.
         provider_options: Provider-specific options (e.g. ``betas``)
             used by the regular conversation calls.
+        reporter: Reporter to log errors during summarization to
 
     Returns:
         The raw VLM response message.
@@ -121,13 +123,21 @@ def _summarize_message_history(
         )
     )
 
-    return vlm_provider.create_message(
-        messages=messages_to_summarize,
-        max_tokens=2048,
-        system=system,
-        tools=tools,
-        provider_options=provider_options,
-    )
+    try:
+        return vlm_provider.create_message(
+            messages=messages_to_summarize,
+            max_tokens=2048,
+            system=system,
+            tools=tools,
+            provider_options=provider_options,
+        )
+    except Exception as e:
+        # catch e.g. BadRequestError
+        error_msg = f"Truncation Failed with error: {e}"
+        logger.exception(error_msg)
+        if reporter:
+            reporter.add_message("TruncationStrategy", error_msg)
+        raise
 
 
 def _extract_summary_text(response: MessageParam) -> str:
@@ -405,6 +415,7 @@ def truncate(self) -> None:
             system=system,
             tools=tools,
             provider_options=provider_options,
+            reporter=self.reporter,
         )
         if self.reporter:
             self.reporter.add_message(
@@ -731,6 +742,7 @@ def truncate(self) -> None:
             system=system,
             tools=tools,
             provider_options=provider_options,
+            reporter=self.reporter,
         )
         if self.reporter:
             self.reporter.add_message(

src/askui/models/shared/truncation_strategies.py

@@ -741,3 +741,46 @@ def temporary_select(self, computer_id: str) -> AbstractContextManager[Self]:
             ```
         """
         raise NotImplementedError
+
+    def get_file_names(self, absolute_directory_path: str) -> list[str]:
+        """
+        List file names in an absolute directory on the automation target
+        (desktop Agent OS).
+
+        Args:
+            absolute_directory_path (str): Absolute directory path on the target system.
+
+        Returns:
+            list[str]: Names of files in that directory.
+
+        Raises:
+            NotImplementedError: If the implementation does not support this operation.
+        """
+        raise NotImplementedError
+
+    def get_file(self, path: str) -> Image.Image | str:
+        """
+        Read a file from the automation target (desktop Agent OS).
+
+        Binary image payloads are returned as `PIL.Image.Image` when recognized;
+        otherwise UTF-8 text when decodable.
+
+        Args:
+            path (str): File path on the target system.
+
+        Returns:
+            Image.Image | str: Decoded file contents.
+
+        Raises:
+            NotImplementedError: If the implementation does not support this operation.
+        """
+        raise NotImplementedError
+
+    def remove_virtual_displays(self) -> None:
+        """
+        Remove virtual displays from the controller, leaving real displays only.
+
+        Raises:
+            NotImplementedError: If the implementation does not support this operation.
+        """
+        raise NotImplementedError