Added API layer for tool calls (#89)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Direct tool execution without LLM orchestration, returning structured
results with execution time and metadata.
* Public endpoints to list and run available tools, plus actions: click,
type, scroll, navigate, hover.
* Accessibility-tree retrieval and OpenAI-compatible execution result
formatting.
  * Execution tracing/logging surfaced for diagnostics.

* **Chores**
* Configurable upstream branch for browser operator builds and updated
bundled agent asset.

<sub>✏️ Tip: You can customize this high-level summary in your review
settings.</sub>
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: Oleh Luchkiv <olesho@gmail.com>
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

Author: 3 people

agent-server/nodejs/src/api-server.js

@@ -1418,6 +1418,55 @@ export class BrowserAgentServer extends EventEmitter {
     }
   }
 
+  /**
+   * Execute a tool directly on a connected DevTools client
+   * This bypasses LLM orchestration and calls the tool directly
+   * @param {Object} connection - DevTools WebSocket connection
+   * @param {string} tool - Tool name (e.g., 'perform_action', 'navigate_url')
+   * @param {Object} args - Tool-specific arguments
+   * @param {number} timeout - Execution timeout in milliseconds
+   * @returns {Promise<Object>} Tool execution result
+   */
+  async executeToolDirect(connection, tool, args, timeout = 30000) {
+    const rpcId = `tool-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
+
+    logger.info('Executing tool directly', {
+      clientId: connection.clientId,
+      tool,
+      timeout
+    });
+
+    try {
+      // Prepare RPC request for execute_tool method
+      const response = await connection.rpcClient.callMethod(
+        connection.ws,
+        'execute_tool',
+        {
+          tool,
+          args,
+          timeout
+        },
+        timeout + 5000 // Add buffer for network overhead
+      );
+
+      logger.info('Tool execution completed', {
+        clientId: connection.clientId,
+        tool,
+        success: response?.result?.success
+      });
+
+      return response;
+
+    } catch (error) {
+      logger.error('Tool execution failed', {
+        clientId: connection.clientId,
+        tool,
+        error: error.message
+      });
+      throw error;
+    }
+  }
+
   /**
    * Execute JavaScript in a browser tab
    * @param {string} tabId - Tab ID (target ID)

agent-server/nodejs/src/lib/BrowserAgentServer.js

@@ -867,7 +867,7 @@ grd_files_bundled_sources = [
   "front_end/panels/ai_chat/agent_framework/implementation/agents/ScrollActionAgent.js",
   "front_end/panels/ai_chat/agent_framework/implementation/agents/WebTaskAgent.js",
   "front_end/panels/ai_chat/agent_framework/implementation/agents/SearchAgent.js",
-  "front_end/panels/ai_chat/agent_framework/implementation/agents/ActionAgentV0.js",
+  "front_end/panels/ai_chat/agent_framework/implementation/agents/ActionAgentV1.js",
   "front_end/panels/ai_chat/agent_framework/implementation/agents/ActionAgentV2.js",
   "front_end/panels/ai_chat/common/MarkdownViewerUtil.js",
   "front_end/panels/ai_chat/evaluation/runner/VisionAgentEvaluationRunner.js",

config/gni/devtools_grd_files.gni

@@ -43,22 +43,27 @@ RUN /workspace/depot_tools/ensure_bootstrap
 RUN npm run build
 
 # Add Browser Operator fork and switch to it
+# Branch is configurable via build arg (default: main)
+ARG BROWSER_OPERATOR_BRANCH=main
 RUN git remote add upstream https://github.com/BrowserOperator/browser-operator-core.git
 RUN git fetch upstream
-RUN git checkout upstream/main
+RUN git checkout upstream/${BROWSER_OPERATOR_BRANCH}
 
 # Copy local LLM changes on top of the upstream code
-# This allows iterative development without breaking BUILD.gn compatibility
+# This allows iterative development of LLM providers without rebuilding everything
 COPY front_end/panels/ai_chat/LLM /workspace/devtools/devtools-frontend/front_end/panels/ai_chat/LLM
 
+# Regenerate GN build files
+RUN gn gen out/Default
+
 # AUTOMATED_MODE: Default true for API mode (Type 2/3). Override with --build-arg AUTOMATED_MODE=false for Type 1.
 ARG AUTOMATED_MODE=true
 RUN if [ "$AUTOMATED_MODE" = "true" ]; then \
       sed -i 's/AUTOMATED_MODE: false/AUTOMATED_MODE: true/' \
       front_end/panels/ai_chat/core/BuildConfig.ts; \
     fi
 
-# Build Browser Operator version with local changes
+# Build Browser Operator version with local changes (fresh GN generation)
 RUN npm run build
 
 # ==============================================================================

docker/Dockerfile

@@ -20,17 +20,23 @@ import {
   EvaluationRequest,
   EvaluationSuccessResponse,
   EvaluationErrorResponse,
+  ToolExecutionRequest,
+  ToolExecutionSuccessResponse,
+  ToolExecutionErrorResponse,
   ErrorCodes,
   isWelcomeMessage,
   isRegistrationAckMessage,
   isEvaluationRequest,
+  isToolExecutionRequest,
   isPongMessage,
   createRegisterMessage,
   createReadyMessage,
   createAuthVerifyMessage,
   createStatusMessage,
   createSuccessResponse,
-  createErrorResponse
+  createErrorResponse,
+  createToolExecutionSuccessResponse,
+  createToolExecutionErrorResponse
 } from './EvaluationProtocol.js';
 
 const logger = createLogger('EvaluationAgent');
@@ -171,6 +177,9 @@ export class EvaluationAgent {
       else if (isEvaluationRequest(message)) {
         await this.handleEvaluationRequest(message);
       }
+      else if (isToolExecutionRequest(message)) {
+        await this.handleToolExecutionRequest(message);
+      }
       else if (isPongMessage(message)) {
         logger.debug('Received pong');
       }
@@ -599,6 +608,91 @@ export class EvaluationAgent {
     }
   }
 
+  /**
+   * Handle direct tool execution request (no LLM orchestration)
+   * This allows calling browser automation tools directly via API
+   */
+  private async handleToolExecutionRequest(request: ToolExecutionRequest): Promise<void> {
+    const { params, id } = request;
+    const startTime = Date.now();
+
+    logger.info('Received tool execution request', {
+      tool: params.tool,
+      hasArgs: !!params.args,
+      timeout: params.timeout
+    });
+
+    try {
+      // Get the tool from registry
+      const tool = ToolRegistry.getRegisteredTool(params.tool);
+      if (!tool) {
+        const errorResponse = createToolExecutionErrorResponse(
+          id,
+          ErrorCodes.INVALID_TOOL,
+          `Tool not found: ${params.tool}`,
+          params.tool,
+          `Tool '${params.tool}' is not registered in the ToolRegistry`
+        );
+        if (this.client) {
+          this.client.send(errorResponse);
+        }
+        return;
+      }
+
+      // Execute the tool directly (no LLM, no navigation, no retries)
+      const timeout = params.timeout || 30000;
+      const result = await this.executeToolWithTimeout(
+        tool,
+        params.args,
+        timeout,
+        undefined, // No tracing context for direct tool calls
+        params.tool
+      );
+
+      const executionTime = Date.now() - startTime;
+
+      // Send success response
+      const successResponse = createToolExecutionSuccessResponse(
+        id,
+        params.tool,
+        result,
+        executionTime
+      );
+
+      if (this.client) {
+        this.client.send(successResponse);
+      }
+
+      logger.info('Tool execution completed', {
+        tool: params.tool,
+        executionTime,
+        success: true
+      });
+
+    } catch (error) {
+      const executionTime = Date.now() - startTime;
+      const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+
+      logger.error(`Tool execution failed: ${errorMessage}`, {
+        tool: params.tool,
+        executionTime
+      });
+
+      // Send error response
+      const errorResponse = createToolExecutionErrorResponse(
+        id,
+        ErrorCodes.TOOL_EXECUTION_ERROR,
+        'Tool execution failed',
+        params.tool,
+        errorMessage
+      );
+
+      if (this.client) {
+        this.client.send(errorResponse);
+      }
+    }
+  }
+
   private async executeToolWithTimeout(
     tool: any,
     input: any,

front_end/panels/ai_chat/evaluation/EvaluationAgent.ts

@@ -79,6 +79,44 @@ export interface EvaluationRequest {
   id: string;
 }
 
+// Direct tool execution request (no LLM orchestration)
+export interface ToolExecutionRequest {
+  jsonrpc: '2.0';
+  method: 'execute_tool';
+  params: ToolExecutionParams;
+  id: string;
+}
+
+export interface ToolExecutionParams {
+  tool: string;           // Tool name (e.g., 'perform_action', 'navigate_url')
+  args: any;              // Tool-specific arguments
+  timeout?: number;       // Optional timeout (default 30000ms)
+}
+
+export interface ToolExecutionSuccessResponse {
+  jsonrpc: '2.0';
+  result: {
+    success: true;
+    output: any;
+    executionTime: number;
+    tool: string;
+  };
+  id: string;
+}
+
+export interface ToolExecutionErrorResponse {
+  jsonrpc: '2.0';
+  error: {
+    code: number;
+    message: string;
+    data?: {
+      tool: string;
+      error: string;
+    };
+  };
+  id: string;
+}
+
 export interface EvaluationParams {
   evaluationId: string;
   name: string;
@@ -170,6 +208,10 @@ export function isEvaluationRequest(msg: any): msg is EvaluationRequest {
   return msg?.jsonrpc === '2.0' && msg?.method === 'evaluate';
 }
 
+export function isToolExecutionRequest(msg: any): msg is ToolExecutionRequest {
+  return msg?.jsonrpc === '2.0' && msg?.method === 'execute_tool';
+}
+
 export function isPongMessage(msg: any): msg is PongMessage {
   return msg?.type === 'pong';
 }
@@ -254,4 +296,43 @@ export function createErrorResponse(
     },
     id
   };
+}
+
+export function createToolExecutionSuccessResponse(
+  id: string,
+  tool: string,
+  output: any,
+  executionTime: number
+): ToolExecutionSuccessResponse {
+  return {
+    jsonrpc: '2.0',
+    result: {
+      success: true,
+      output,
+      executionTime,
+      tool
+    },
+    id
+  };
+}
+
+export function createToolExecutionErrorResponse(
+  id: string,
+  code: number,
+  message: string,
+  tool: string,
+  error: string
+): ToolExecutionErrorResponse {
+  return {
+    jsonrpc: '2.0',
+    error: {
+      code,
+      message,
+      data: {
+        tool,
+        error
+      }
+    },
+    id
+  };
 }