Skip to content

Commit 8fc16d2

Browse files
miguelg719chromiebotclaude
authored
[fix] Anthropic CUA triple_click mapping (#2104) (#2107)
# Why `AnthropicCUAClient.convertToolUseToAction()` handled actions like `click`, `double_click`, and `left_click`, but did not explicitly support `triple_click`. Two issues: 1. The action type remained in snake_case (`triple_click`) instead of being normalized to `tripleClick` — the format expected by `v3CuaAgentHandler.executeAction()`. 2. Coordinates provided via Anthropic’s coordinate: [x, y] format were not converted into the separate x / y fields required by the handler. As a result, Anthropic triple-click actions were silently ignored by the handler switch. # What Changed - AnthropicCUAClient.ts - Added explicit handling for triple_click / tripleClick in convertToolUseToAction() - Mirrors the existing double_click implementation - Normalizes action type to tripleClick - Extracts coordinates from both: - coordinate: [x, y] - direct x / y fields - v3CuaAgentHandler.ts - Added "triple_click" as a switch-case alias in executeAction() # Test Plan Added `anthropic-cua-triple-click.test.ts` (Vitest) covering: - triple_click with coordinate: [x, y] - triple_click with direct x / y fields All existing unit tests continue to pass. <!-- This is an auto-generated description by cubic. --> --- ## Summary by cubic Fixes Anthropic CUA `triple_click` so it runs as `tripleClick` with x/y parsed from `coordinate: [x, y]` or explicit fields. Also adds a CI job to run evals unit tests. - **Bug Fixes** - Normalize `triple_click`/`tripleClick` to `tripleClick` in `AnthropicCUAClient.convertToolUseToAction()`. - Parse `x`/`y` from `coordinate: [x, y]` or direct fields. - Add `"triple_click"` alias in `v3CuaAgentHandler.executeAction()`. - **Dependencies** - Changeset to publish a patch for `@browserbasehq/stagehand`. <sup>Written for commit a230bfc. Summary will update on new commits.</sup> <!-- End of auto-generated description by cubic. --> --------- Co-authored-by: Chromie <miguel@browserbase.com> Co-authored-by: Claude <noreply@anthropic.com>
1 parent ae32f73 commit 8fc16d2

5 files changed

Lines changed: 139 additions & 0 deletions

File tree

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@browserbasehq/stagehand": patch
3+
---
4+
5+
Fix Anthropic CUA `triple_click` action mapping.

.github/workflows/ci.yml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -335,6 +335,24 @@ jobs:
335335
- name: Run CLI Tests
336336
run: pnpm exec turbo run test:cli --filter=@browserbasehq/browse-cli
337337

338+
run-evals-unit-tests:
339+
name: Evals Unit Tests
340+
runs-on: ubuntu-latest
341+
needs: [run-build, determine-changes]
342+
if: needs.determine-changes.outputs.evals == 'true'
343+
steps:
344+
- uses: actions/checkout@v4
345+
with:
346+
fetch-depth: 1
347+
348+
- uses: ./.github/actions/setup-node-pnpm-turbo
349+
with:
350+
use-prebuilt-artifacts: "true"
351+
restore-turbo-cache: "false"
352+
353+
- name: Run Evals Unit Tests
354+
run: pnpm --filter @browserbasehq/stagehand-evals run test:unit
355+
338356
discover-core-tests:
339357
runs-on: ubuntu-latest
340358
needs: [determine-changes]

packages/core/lib/v3/agent/AnthropicCUAClient.ts

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -901,6 +901,17 @@ export class AnthropicCUAClient extends AgentClient {
901901
(input.coordinate ? (input.coordinate as number[])[1] : 0),
902902
...input,
903903
};
904+
} else if (action === "triple_click" || action === "tripleClick") {
905+
return {
906+
type: "tripleClick",
907+
x:
908+
(input.x as number) ||
909+
(input.coordinate ? (input.coordinate as number[])[0] : 0),
910+
y:
911+
(input.y as number) ||
912+
(input.coordinate ? (input.coordinate as number[])[1] : 0),
913+
...input,
914+
};
904915
} else if (action === "scroll") {
905916
// Convert Anthropic's coordinate, scroll_amount and scroll_direction into scroll_x and scroll_y
906917
const x =

packages/core/lib/v3/handlers/v3CuaAgentHandler.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -324,6 +324,7 @@ export class V3CuaAgentHandler {
324324
}
325325
return { success: true };
326326
}
327+
case "triple_click":
327328
case "tripleClick": {
328329
const { x, y } = action;
329330
if (recording) {
Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
import { describe, expect, it, vi, beforeEach } from "vitest";
2+
import { AnthropicCUAClient } from "../../lib/v3/agent/AnthropicCUAClient.js";
3+
import Anthropic from "@anthropic-ai/sdk";
4+
5+
vi.mock("@anthropic-ai/sdk", () => {
6+
const mockCreate = vi.fn();
7+
8+
return {
9+
default: class MockAnthropic {
10+
beta = {
11+
messages: {
12+
create: mockCreate,
13+
},
14+
};
15+
},
16+
};
17+
});
18+
19+
describe("AnthropicCUAClient triple_click handling", () => {
20+
let mockCreate: ReturnType<typeof vi.fn>;
21+
let client: AnthropicCUAClient;
22+
let executedActions: Array<Record<string, unknown>>;
23+
24+
beforeEach(() => {
25+
vi.clearAllMocks();
26+
const anthropic = new Anthropic({ apiKey: "test" });
27+
mockCreate = anthropic.beta.messages.create as ReturnType<typeof vi.fn>;
28+
29+
client = new AnthropicCUAClient(
30+
"anthropic",
31+
"claude-sonnet-4-5-20250929",
32+
undefined,
33+
{
34+
apiKey: "test-key",
35+
},
36+
);
37+
client.setViewport(1280, 720);
38+
client.setScreenshotProvider(async () => "fake-base64-screenshot");
39+
40+
executedActions = [];
41+
client.setActionHandler(async (action) => {
42+
executedActions.push({ ...action });
43+
});
44+
});
45+
46+
it("should convert triple_click with coordinate array to tripleClick action", async () => {
47+
mockCreate.mockResolvedValue({
48+
id: "test-id",
49+
content: [
50+
{
51+
type: "tool_use",
52+
id: "tool-1",
53+
name: "computer",
54+
input: {
55+
action: "triple_click",
56+
coordinate: [640, 360],
57+
},
58+
},
59+
],
60+
usage: { input_tokens: 10, output_tokens: 20 },
61+
});
62+
63+
const logger = vi.fn();
64+
await client.executeStep(
65+
[{ role: "user", content: "triple click the paragraph" }],
66+
logger,
67+
);
68+
69+
expect(executedActions).toHaveLength(1);
70+
expect(executedActions[0].type).toBe("tripleClick");
71+
expect(executedActions[0].x).toBe(640);
72+
expect(executedActions[0].y).toBe(360);
73+
});
74+
75+
it("should convert triple_click with x/y fields to tripleClick action", async () => {
76+
mockCreate.mockResolvedValue({
77+
id: "test-id",
78+
content: [
79+
{
80+
type: "tool_use",
81+
id: "tool-2",
82+
name: "computer",
83+
input: {
84+
action: "triple_click",
85+
x: 100,
86+
y: 200,
87+
},
88+
},
89+
],
90+
usage: { input_tokens: 10, output_tokens: 20 },
91+
});
92+
93+
const logger = vi.fn();
94+
await client.executeStep(
95+
[{ role: "user", content: "triple click the line" }],
96+
logger,
97+
);
98+
99+
expect(executedActions).toHaveLength(1);
100+
expect(executedActions[0].type).toBe("tripleClick");
101+
expect(executedActions[0].x).toBe(100);
102+
expect(executedActions[0].y).toBe(200);
103+
});
104+
});

0 commit comments

Comments
 (0)