Skip to content

Commit bc5726e

Browse files
thymikeethymikee
authored
test: cover web provider scenario (#827)
1 parent 833479e commit bc5726e

4 files changed

Lines changed: 367 additions & 0 deletions

File tree

‎scripts/integration-progress-model.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -371,6 +371,11 @@ function summarizeProviderPressure(files) {
371371
pattern:
372372
/\bLinuxToolProvider\b|\blinuxToolProvider\b|\brunCommand\b|\bwhichCommand\b|\bxdotool\b|\bydotool\b|\bxclip\b|\bscrot\b|\bgrim\b|\bwmctrl\b|\bpkill\b/g,
373373
},
374+
{
375+
name: 'Web semantic provider',
376+
pattern:
377+
/\bWebProvider\b|\bwebProvider\b|\bwithWebProvider\b|\bresolveWebProvider\b|\['web'/g,
378+
},
374379
{
375380
name: 'Recording provider',
376381
pattern: /\bRecordingProvider\b|\brecordingProvider\b|\bstartRecording\b/g,

‎test/integration/provider-scenarios/fixtures.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,15 @@ export const PROVIDER_SCENARIO_LINUX: DeviceInfo = {
6666
booted: true,
6767
};
6868

69+
export const PROVIDER_SCENARIO_WEB: DeviceInfo = {
70+
platform: 'web',
71+
id: 'agent-browser-chrome',
72+
name: 'Agent Browser Chrome',
73+
kind: 'device',
74+
target: 'desktop',
75+
booted: true,
76+
};
77+
6978
export function createDemoIosApp(prefix: string): { tempRoot: string; appPath: string } {
7079
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
7180
const appPath = path.join(tempRoot, 'Demo.app');

‎test/integration/provider-scenarios/web-desktop.test.ts‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
import assert from 'node:assert/strict';
2+
import fs from 'node:fs';
3+
import { test } from 'vitest';
4+
import { assertFlatToolCall, assertPngFile } from './assertions.ts';
5+
import { PROVIDER_SCENARIO_WEB } from './fixtures.ts';
6+
import { createProviderScenarioTempPath, withProviderScenarioResource } from './harness.ts';
7+
import { runProviderScenario } from './scenario.ts';
8+
import { createWebDesktopWorld } from './web-world.ts';
9+
10+
const WEB_URL = 'https://example.test/dashboard';
11+
12+
test('Provider-backed integration web desktop flow uses semantic web provider calls', async () => {
13+
await withProviderScenarioResource(createWebDesktopWorld, async ({ daemon, semanticCalls }) => {
14+
const screenshotPath = createProviderScenarioTempPath(
15+
'agent-device-provider-scenario-web',
16+
'png',
17+
);
18+
19+
try {
20+
const devices = await daemon.client().devices.list({ platform: 'web' });
21+
assert.equal(devices.length, 1);
22+
assert.equal(devices[0]?.platform, 'web');
23+
assert.equal(devices[0]?.id, PROVIDER_SCENARIO_WEB.id);
24+
assert.equal(devices[0]?.target, 'desktop');
25+
26+
await runProviderScenario(daemon, [
27+
{
28+
name: 'open web URL',
29+
command: 'open',
30+
positionals: [WEB_URL],
31+
flags: { platform: 'web' },
32+
},
33+
{
34+
name: 'capture interactive web snapshot',
35+
command: 'snapshot',
36+
flags: { snapshotInteractiveOnly: true },
37+
assert: (snapshot) => {
38+
const labels = snapshot.json?.result?.data?.nodes?.map(
39+
(node: { label?: string }) => node.label,
40+
);
41+
assert.deepEqual(labels, [
42+
WEB_URL,
43+
'Ready',
44+
'Email',
45+
'Submit order',
46+
'Ready',
47+
'Below the fold',
48+
]);
49+
},
50+
},
51+
{
52+
name: 'read snapshot ref text',
53+
command: 'get',
54+
positionals: ['text', '@e2'],
55+
expectData: { text: 'Ready' },
56+
},
57+
{
58+
name: 'find visible text',
59+
command: 'find',
60+
positionals: ['text', 'Submit order', 'exists'],
61+
expectData: { found: true },
62+
},
63+
{
64+
name: 'assert visible text',
65+
command: 'is',
66+
positionals: ['visible', 'label="Submit order"'],
67+
expectData: { pass: true },
68+
},
69+
{
70+
name: 'wait for text',
71+
command: 'wait',
72+
positionals: ['text', 'Ready', '100'],
73+
expectData: { text: 'Ready' },
74+
},
75+
{
76+
name: 'click submit ref',
77+
command: 'click',
78+
positionals: ['@e4'],
79+
expectData: { x: 84, y: 166 },
80+
},
81+
{
82+
name: 'fill email ref',
83+
command: 'fill',
84+
positionals: ['@e3', 'qa@example.test'],
85+
flags: { delayMs: 1 },
86+
expectData: { text: 'qa@example.test' },
87+
},
88+
{
89+
name: 'type suffix',
90+
command: 'type',
91+
positionals: [' ok'],
92+
expectData: { text: ' ok' },
93+
},
94+
{
95+
name: 'scroll by pixels',
96+
command: 'scroll',
97+
positionals: ['down'],
98+
flags: { pixels: 240 },
99+
expectData: { pixels: 240 },
100+
},
101+
{
102+
name: 'capture web screenshot artifact',
103+
command: 'screenshot',
104+
positionals: [screenshotPath],
105+
flags: {
106+
screenshotFullscreen: true,
107+
screenshotNoStabilize: true,
108+
},
109+
expectData: { path: screenshotPath },
110+
assert: () => {
111+
assertPngFile(screenshotPath);
112+
},
113+
},
114+
]);
115+
116+
const actions = daemon.session()?.actions ?? [];
117+
assert.ok(
118+
actions.some(
119+
(action) => action.command === 'click' && action.positionals.join(' ') === '@e4',
120+
),
121+
'Expected ref click action to be recorded on the session',
122+
);
123+
assert.ok(
124+
actions.some(
125+
(action) =>
126+
action.command === 'fill' &&
127+
action.positionals.join(' ') === '@e3 qa@example.test' &&
128+
action.flags.delayMs === 1,
129+
),
130+
'Expected ref fill action to be recorded on the session',
131+
);
132+
assert.ok(
133+
actions.some(
134+
(action) => action.command === 'type' && action.positionals.join(' ') === ' ok',
135+
),
136+
'Expected type action to be recorded on the session',
137+
);
138+
139+
const close = await daemon.callCommand('close', [WEB_URL]);
140+
assert.equal(close.statusCode, 200, JSON.stringify(close.json));
141+
142+
assertFlatToolCall(semanticCalls, ['web', 'open', WEB_URL, '']);
143+
assertFlatToolCall(semanticCalls, ['web', 'snapshot', 'true', '']);
144+
assertFlatToolCall(semanticCalls, ['web', 'click', '84', '166']);
145+
assertFlatToolCall(semanticCalls, ['web', 'fill', '144', '114', 'qa@example.test', '1']);
146+
assertFlatToolCall(semanticCalls, ['web', 'type', ' ok', '0']);
147+
assertFlatToolCall(semanticCalls, ['web', 'scroll', 'down', '', '240']);
148+
assertFlatToolCall(semanticCalls, [
149+
'web',
150+
'screenshot',
151+
screenshotPath,
152+
'true',
153+
'false',
154+
'app',
155+
]);
156+
assertFlatToolCall(semanticCalls, ['web', 'close', WEB_URL]);
157+
} finally {
158+
fs.rmSync(screenshotPath, { force: true });
159+
}
160+
});
161+
}, 10_000);

‎test/integration/provider-scenarios/web-world.ts‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
import fs from 'node:fs';
2+
import type { WebProvider } from '../../../src/platforms/web/provider.ts';
3+
import type { RawSnapshotNode } from '../../../src/utils/snapshot.ts';
4+
import { validPng } from './assertions.ts';
5+
import { PROVIDER_SCENARIO_WEB } from './fixtures.ts';
6+
import { createProviderScenarioHarness, type ProviderScenarioHarness } from './harness.ts';
7+
import type { FlatToolCall } from './providers.ts';
8+
9+
const INPUT_RECT = { x: 24, y: 96, width: 240, height: 36 };
10+
const BUTTON_RECT = { x: 24, y: 148, width: 120, height: 36 };
11+
12+
type WebPageState = {
13+
openedTarget: string;
14+
inputValue: string;
15+
statusText: string;
16+
scrolled: boolean;
17+
};
18+
19+
export type WebDesktopWorld = {
20+
daemon: ProviderScenarioHarness;
21+
semanticCalls: FlatToolCall[];
22+
close: () => Promise<void>;
23+
};
24+
25+
export async function createWebDesktopWorld(): Promise<WebDesktopWorld> {
26+
const semanticCalls: FlatToolCall[] = [];
27+
const state: WebPageState = {
28+
openedTarget: 'about:blank',
29+
inputValue: '',
30+
statusText: 'Ready',
31+
scrolled: false,
32+
};
33+
34+
const provider: WebProvider = {
35+
open: async (target, options) => {
36+
semanticCalls.push(['web', 'open', target, options?.url ?? '']);
37+
state.openedTarget = target;
38+
state.statusText = 'Ready';
39+
},
40+
close: async (target) => {
41+
semanticCalls.push(['web', 'close', target ?? '']);
42+
},
43+
snapshot: async (options) => {
44+
semanticCalls.push([
45+
'web',
46+
'snapshot',
47+
String(options?.interactiveOnly ?? ''),
48+
String(options?.surface ?? ''),
49+
]);
50+
return { nodes: webSnapshotNodes(state), truncated: false };
51+
},
52+
screenshot: async (outPath, options) => {
53+
semanticCalls.push([
54+
'web',
55+
'screenshot',
56+
outPath,
57+
String(options?.fullscreen ?? ''),
58+
String(options?.stabilize ?? ''),
59+
String(options?.surface ?? ''),
60+
]);
61+
fs.writeFileSync(outPath, validPng());
62+
},
63+
click: async (x, y) => {
64+
semanticCalls.push(['web', 'click', String(x), String(y)]);
65+
if (pointInRect(x, y, BUTTON_RECT)) {
66+
state.statusText = 'Submitted';
67+
}
68+
},
69+
fill: async (x, y, text, options) => {
70+
semanticCalls.push([
71+
'web',
72+
'fill',
73+
String(x),
74+
String(y),
75+
text,
76+
String(options?.delayMs ?? 0),
77+
]);
78+
if (pointInRect(x, y, INPUT_RECT)) {
79+
state.inputValue = text;
80+
}
81+
},
82+
typeText: async (text, options) => {
83+
semanticCalls.push(['web', 'type', text, String(options?.delayMs ?? 0)]);
84+
state.inputValue += text;
85+
},
86+
scroll: async (direction, options) => {
87+
semanticCalls.push([
88+
'web',
89+
'scroll',
90+
direction,
91+
String(options?.amount ?? ''),
92+
String(options?.pixels ?? ''),
93+
]);
94+
state.scrolled = true;
95+
},
96+
};
97+
98+
const daemon = await createProviderScenarioHarness({
99+
webProvider: () => provider,
100+
deviceInventoryProvider: async () => [PROVIDER_SCENARIO_WEB],
101+
});
102+
103+
let closed = false;
104+
return {
105+
daemon,
106+
semanticCalls,
107+
close: async () => {
108+
if (closed) return;
109+
closed = true;
110+
await daemon.close();
111+
},
112+
};
113+
}
114+
115+
function webSnapshotNodes(state: WebPageState): RawSnapshotNode[] {
116+
return [
117+
{
118+
index: 0,
119+
role: 'document',
120+
label: state.openedTarget,
121+
rect: { x: 0, y: 0, width: 390, height: 720 },
122+
enabled: true,
123+
hittable: true,
124+
visibleToUser: true,
125+
depth: 0,
126+
},
127+
{
128+
index: 1,
129+
role: 'static text',
130+
label: 'Ready',
131+
rect: { x: 24, y: 32, width: 160, height: 28 },
132+
enabled: true,
133+
hittable: true,
134+
visibleToUser: true,
135+
depth: 1,
136+
parentIndex: 0,
137+
},
138+
{
139+
index: 2,
140+
role: 'text field',
141+
label: 'Email',
142+
value: state.inputValue,
143+
rect: INPUT_RECT,
144+
enabled: true,
145+
hittable: true,
146+
visibleToUser: true,
147+
depth: 1,
148+
parentIndex: 0,
149+
},
150+
{
151+
index: 3,
152+
role: 'button',
153+
label: 'Submit order',
154+
rect: BUTTON_RECT,
155+
enabled: true,
156+
hittable: true,
157+
visibleToUser: true,
158+
depth: 1,
159+
parentIndex: 0,
160+
},
161+
{
162+
index: 4,
163+
role: 'static text',
164+
label: state.statusText,
165+
rect: { x: 24, y: 204, width: 180, height: 28 },
166+
enabled: true,
167+
hittable: true,
168+
visibleToUser: true,
169+
depth: 1,
170+
parentIndex: 0,
171+
},
172+
{
173+
index: 5,
174+
role: 'static text',
175+
label: state.scrolled ? 'Scrolled section' : 'Below the fold',
176+
rect: { x: 24, y: 620, width: 180, height: 28 },
177+
enabled: true,
178+
hittable: true,
179+
visibleToUser: true,
180+
depth: 1,
181+
parentIndex: 0,
182+
},
183+
];
184+
}
185+
186+
function pointInRect(
187+
x: number,
188+
y: number,
189+
rect: { x: number; y: number; width: number; height: number },
190+
): boolean {
191+
return x >= rect.x && x <= rect.x + rect.width && y >= rect.y && y <= rect.y + rect.height;
192+
}

0 commit comments

Comments
 (0)