Merge pull request #221 from askui/integrate-new-tools-from-core

Integrate new tools from core

Author: mlikasam-askui

pyproject.toml

@@ -74,7 +74,7 @@ typecheck = "mypy"
     "lint:fix",
 ] }
 "grpc:gen" = "bash scripts/grpc-gen.sh"
-"json:gen" = "datamodel-codegen --output-model-type pydantic_v2.BaseModel --input src/askui/tools/askui/askui_ui_controller_grpc/json_schema/ --input-file-type jsonschema --output src/askui/tools/askui/askui_ui_controller_grpc/generated/"
+"json:gen" = "datamodel-codegen --output-model-type pydantic_v2.BaseModel --input src/askui/tools/askui/askui_ui_controller_grpc/json_schema/ --input-file-type jsonschema --output src/askui/tools/askui/askui_ui_controller_grpc/generated/ --use-title-as-name "
 
 [dependency-groups]
 dev = [

src/askui/agent.py

@@ -13,6 +13,7 @@
 )
 from askui.tools.computer import (
     ComputerGetMousePositionTool,
+    ComputerGetSystemInfoTool,
     ComputerKeyboardPressedTool,
     ComputerKeyboardReleaseTool,
     ComputerKeyboardTapTool,
@@ -92,6 +93,7 @@ def __init__(
             models=models,
             tools=[
                 ExceptionTool(),
+                ComputerGetSystemInfoTool(),
                 ComputerGetMousePositionTool(),
                 ComputerKeyboardPressedTool(),
                 ComputerKeyboardReleaseTool(),

src/askui/agent_base.py

@@ -106,6 +106,7 @@ def __init__(
         )
 
         self.act_tool_collection = ToolCollection(tools=tools)
+        self.act_tool_collection.add_agent_os(agent_os)
 
         self.act_settings = ActSettings()
         self.caching_settings = CachingSettings()

src/askui/models/anthropic/messages_api.py

@@ -74,7 +74,7 @@ def create_message(
 
         response = self._client.beta.messages.create(  # type: ignore[misc]
             messages=_messages,
-            max_tokens=max_tokens or 4096,
+            max_tokens=max_tokens or 8192,
             model=model,
             tools=tools.to_params() if not isinstance(tools, Omit) else omit,
             betas=betas,

src/askui/models/shared/android_base_tool.py

@@ -3,6 +3,7 @@
 from askui.models.shared.tool_tags import ToolTags
 from askui.models.shared.tools import ToolWithAgentOS
 from askui.tools import AgentOs
+from askui.tools.agent_os_type_error import AgentOsTypeError
 from askui.tools.android.agent_os import AndroidAgentOs
 
 
@@ -33,12 +34,10 @@ def agent_os(self) -> AndroidAgentOs:
         """
         agent_os = super().agent_os
         if not isinstance(agent_os, AndroidAgentOs):
-            msg = (
-                "Agent OS is not an AndroidAgentOs. "
-                "Call `agent_os = ...` or initialize the tool with an "
-                "AndroidAgentOs."
+            raise AgentOsTypeError(
+                expected_type=AndroidAgentOs,
+                actual_type=type(agent_os),
             )
-            raise TypeError(msg)
         return agent_os
 
     @agent_os.setter
@@ -52,9 +51,8 @@ def agent_os(self, agent_os: AgentOs | AndroidAgentOs) -> None:
             TypeError: If the agent OS is not an AndroidAgentOs instance.
         """
         if not isinstance(agent_os, AndroidAgentOs):
-            msg = (
-                "Agent OS must be an AndroidAgentOs instance. "
-                f"Got {type(agent_os).__name__} instead."
+            raise AgentOsTypeError(
+                expected_type=AndroidAgentOs,
+                actual_type=type(agent_os),
             )
-            raise TypeError(msg)
         self._agent_os = agent_os

src/askui/models/shared/computer_base_tool.py

@@ -3,6 +3,7 @@
 from askui.models.shared.tool_tags import ToolTags
 from askui.models.shared.tools import ToolWithAgentOS
 from askui.tools.agent_os import AgentOs
+from askui.tools.agent_os_type_error import AgentOsTypeError
 from askui.tools.android.agent_os import AndroidAgentOs
 
 
@@ -30,12 +31,10 @@ def agent_os(self) -> AgentOs:
         """
         agent_os = super().agent_os
         if not isinstance(agent_os, AgentOs):
-            msg = (
-                "Agent OS is not a ComputerAgentOs. "
-                "Call `agent_os = ...` or initialize the tool with a "
-                "ComputerAgentOs."
+            raise AgentOsTypeError(
+                expected_type=AgentOs,
+                actual_type=type(agent_os),
             )
-            raise TypeError(msg)
         return agent_os
 
     @agent_os.setter
@@ -49,9 +48,8 @@ def agent_os(self, agent_os: AgentOs | AndroidAgentOs) -> None:
             TypeError: If the agent OS is not an AgentOs instance.
         """
         if not isinstance(agent_os, AgentOs):
-            msg = (
-                "Agent OS must be an AgentOs instance. "
-                f"Got {type(agent_os).__name__} instead."
+            raise AgentOsTypeError(
+                expected_type=AgentOs,
+                actual_type=type(agent_os),
             )
-            raise TypeError(msg)
         self._agent_os = agent_os

src/askui/models/shared/settings.py

@@ -19,7 +19,7 @@ class MessageSettings(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     betas: list[AnthropicBetaParam] | Omit = omit
-    max_tokens: int = 4096
+    max_tokens: int = 8192
     system: ActSystemPrompt | None = None
     thinking: BetaThinkingConfigParam | Omit = omit
     tool_choice: BetaToolChoiceParam | Omit = omit

src/askui/models/shared/tools.py

@@ -473,12 +473,13 @@ def _run_regular_tool(
         except AgentException:
             raise
         except Exception as e:  # noqa: BLE001
+            error_message = getattr(e, "message", str(e))
             logger.warning(
                 "Tool failed",
-                extra={"tool_name": tool_use_block_param.name, "error": str(e)},
+                extra={"tool_name": tool_use_block_param.name, "error": error_message},
             )
             return ToolResultBlockParam(
-                content=str(e),
+                content=f"Tool raised an unexpected error: {error_message}",
                 is_error=True,
                 tool_use_id=tool_use_block_param.id,
             )

src/askui/tools/__init__.py

@@ -1,5 +1,5 @@
 from .agent_os import AgentOs, Coordinate, ModifierKey, PcKey
-from .askui.command_helpers import create_style
+from .askui.askui_controller import RenderObjectStyle
 from .computer_agent_os_facade import ComputerAgentOsFacade
 from .toolbox import AgentToolbox
 
@@ -9,6 +9,6 @@
     "ModifierKey",
     "PcKey",
     "Coordinate",
-    "create_style",
+    "RenderObjectStyle",
     "ComputerAgentOsFacade",
 ]

src/askui/tools/agent_os.py

@@ -7,9 +7,17 @@
 from askui.models.shared.tool_tags import ToolTags
 
 if TYPE_CHECKING:
+    from askui.tools.askui.askui_ui_controller_grpc.generated import (
+        Controller_V1_pb2 as controller_v1_pbs,
+    )
     from askui.tools.askui.askui_ui_controller_grpc.generated.AgentOS_Send_Request_2501 import (  # noqa: E501
         RenderObjectStyle,
     )
+    from askui.tools.askui.askui_ui_controller_grpc.generated.AgentOS_Send_Response_2501 import (  # noqa: E501
+        GetActiveProcessResponseModel,
+        GetActiveWindowResponseModel,
+        GetSystemInfoResponseModel,
+    )
 
 MouseButton = Literal["left", "middle", "right"]
 
@@ -175,12 +183,16 @@ class Display(BaseModel):
     )
 
     id: int = Field(validation_alias="displayID")
+    name: str = Field(validation_alias="name")
     size: DisplaySize = Field(validation_alias="sizeInPixels")
 
 
 class DisplaysListResponse(BaseModel):
     data: list[Display] = Field(validation_alias="displays")
 
+    def __str__(self) -> str:
+        return ",".join([str(display) for display in self.data])
+
 
 InputEvent = ClickEvent
 
@@ -529,3 +541,121 @@ def clear_render_objects(self) -> None:
             Response confirming the clearing.
         """
         raise NotImplementedError
+
+    def get_process_list(
+        self, get_extended_info: bool = False
+    ) -> "controller_v1_pbs.Response_GetProcessList":
+        """
+        Get a list of running processes.
+
+        Args:
+            get_extended_info (bool, optional): Whether to include
+                extended process information.
+                Defaults to `False`.
+
+        Returns:
+            controller_v1_pbs.Response_GetProcessList: Process list response containing:
+                - processes: List of ProcessInfo objects
+        """
+        raise NotImplementedError
+
+    def get_window_list(
+        self, process_id: int
+    ) -> "controller_v1_pbs.Response_GetWindowList":
+        """
+        Get a list of windows for a specific process.
+
+        Args:
+            process_id (int): The ID of the process to get windows for.
+
+        Returns:
+            controller_v1_pbs.Response_GetWindowList: Window list response containing:
+                - windows: List of WindowInfo objects with ID and name
+        """
+        raise NotImplementedError
+
+    def set_mouse_delay(self, delay_ms: int) -> None:
+        """
+        Configure mouse action delay.
+
+        Args:
+            delay_ms (int): The delay in milliseconds to set for mouse actions.
+        """
+        raise NotImplementedError
+
+    def set_keyboard_delay(self, delay_ms: int) -> None:
+        """
+        Configure keyboard action delay.
+
+        Args:
+            delay_ms (int): The delay in milliseconds to set for keyboard actions.
+        """
+        raise NotImplementedError
+
+    def set_active_window(self, process_id: int, window_id: int) -> int:
+        """
+        Set the active window for automation.
+        Adds the window as a virtual display and returns the display ID.
+        It raises an error if display length is not increased after adding the window.
+
+        Args:
+            process_id (int): The ID of the process that owns the window.
+            window_id (int): The ID of the window to set as active.
+
+        Returns:
+            int: The new Display ID.
+
+        Raises:
+            AskUiControllerError:
+            If display length is not increased after adding the window.
+        """
+        raise NotImplementedError
+
+    def get_system_info(self) -> "GetSystemInfoResponseModel":
+        """
+        Get the system information.
+
+        Returns:
+            GetSystemInfoResponseModel: The system information.
+        """
+        raise NotImplementedError
+
+    def get_active_process(self) -> "GetActiveProcessResponseModel":
+        """
+        Get the active process.
+
+        Returns:
+            GetActiveProcessResponseModel: The active process.
+        """
+        raise NotImplementedError
+
+    def set_active_process(self, process_id: int) -> None:
+        """
+        Set the active process.
+
+        Args:
+            process_id (int): The ID of the process to set as active.
+        """
+        raise NotImplementedError
+
+    def get_active_window(self) -> "GetActiveWindowResponseModel":
+        """
+        Gets the window id and name in addition to the process id
+             and name of the currently active window (in focus).
+
+        Returns:
+            GetActiveWindowResponseModel: The active window.
+        """
+        raise NotImplementedError
+
+    def set_window_in_focus(self, process_id: int, window_id: int) -> None:
+        """
+        Sets the window with the specified windowId of the process
+            with the specified processId active,
+            which brings it to the front and gives it focus.
+
+        Args:
+            process_id (int): The ID of the process that owns the window.
+            window_id (int): The ID of the window to set as active.
+        """
+        raise NotImplementedError