Skip to content

Commit dae6733

Browse files
kycutlerkycutler
committed
Support deferred results from playwright code tool
1 parent 3f2c82d commit dae6733

5 files changed

Lines changed: 201 additions & 47 deletions

File tree

‎src/vs/platform/browserView/common/playwrightService.ts‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,13 @@ import { createDecorator } from '../../instantiation/common/instantiation.js';
88

99
export const IPlaywrightService = createDecorator<IPlaywrightService>('playwrightService');
1010

11+
export interface IInvokeFunctionResult {
12+
result: unknown;
13+
summary: string;
14+
/** When present the function did not complete within the timeout. Pass this ID to {@link IPlaywrightService.waitForDeferredResult} to keep waiting. */
15+
deferredResultId?: string;
16+
}
17+
1118
/**
1219
* A service for using Playwright to connect to and automate the integrated browser.
1320
*
@@ -74,12 +81,30 @@ export interface IPlaywrightService {
7481
/**
7582
* Run a function with access to a Playwright page and return a result for tool output, including error handling.
7683
* The first function argument is always the Playwright `page` object, and additional arguments can be passed after.
84+
*
85+
* When {@link timeoutMs} is provided, the call races against that timeout.
86+
* If the timeout fires before the function completes, or the function is otherwise interrupted,
87+
* the in-flight promise is stored as a *deferred result* and the returned object includes a
88+
* {@link deferredResultId} that can be passed to {@link waitForDeferredResult} to resume waiting.
89+
* When {@link timeoutMs} is omitted the function runs to completion with no deferral.
90+
*
7791
* @param pageId The browser view ID identifying the page to operate on.
7892
* @param fnDef The function code to execute. Should contain the function definition but not its invocation, e.g. `async (page, arg1, arg2) => { ... }`.
7993
* @param args Additional arguments to pass to the function after the `page` object.
80-
* @returns The result of the function execution, including a page summary.
94+
* @param timeoutMs Maximum time (in ms) to wait for the function to complete before deferring. When omitted the call awaits indefinitely.
95+
* @returns The result of the function execution, including a page summary and optionally a deferredResultId if the call did not complete.
96+
*/
97+
invokeFunction(pageId: string, fnDef: string, args?: unknown[], timeoutMs?: number): Promise<IInvokeFunctionResult>;
98+
99+
/**
100+
* Continue waiting for a previously deferred function invocation.
101+
*
102+
* @param deferredResultId The ID returned from a timed-out {@link invokeFunction} call.
103+
* @param timeoutMs Maximum time (in ms) to wait before returning a deferred result again.
104+
* @returns The same shape as {@link invokeFunction}. If the result is still not
105+
* available after the timeout, {@link deferredResultId} is returned again.
81106
*/
82-
invokeFunction(pageId: string, fnDef: string, ...args: unknown[]): Promise<{ result: unknown; summary: string }>;
107+
waitForDeferredResult(deferredResultId: string, timeoutMs: number): Promise<IInvokeFunctionResult>;
83108

84109
/**
85110
* Responds to a file chooser dialog on the given page.

‎src/vs/platform/browserView/node/playwrightService.ts‎

Lines changed: 87 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,16 @@
33
* Licensed under the MIT License. See License.txt in the project root for license information.
44
*--------------------------------------------------------------------------------------------*/
55

6-
import { Disposable, DisposableStore } from '../../../base/common/lifecycle.js';
7-
import { DeferredPromise } from '../../../base/common/async.js';
6+
import { Disposable, DisposableMap, DisposableStore, IDisposable } from '../../../base/common/lifecycle.js';
7+
import { DeferredPromise, disposableTimeout, raceTimeout } from '../../../base/common/async.js';
88
import { Emitter, Event } from '../../../base/common/event.js';
99
import { ILogService } from '../../log/common/log.js';
10-
import { IPlaywrightService } from '../common/playwrightService.js';
10+
import { IInvokeFunctionResult, IPlaywrightService } from '../common/playwrightService.js';
1111
import { IBrowserViewGroupRemoteService } from '../node/browserViewGroupRemoteService.js';
1212
import { IBrowserViewGroup } from '../common/browserViewGroup.js';
13-
import { PlaywrightTab } from './playwrightTab.js';
13+
import { PlaywrightTab, DialogInterruptedError } from './playwrightTab.js';
1414
import { CDPEvent, CDPRequest, CDPResponse } from '../common/cdp/types.js';
15+
import { generateUuid } from '../../../base/common/uuid.js';
1516

1617
// eslint-disable-next-line local/code-import-patterns
1718
import type { Browser, BrowserContext, Page } from 'playwright-core';
@@ -29,6 +30,8 @@ declare module 'playwright-core' {
2930
}
3031
}
3132

33+
const DEFERRED_RESULT_CLEANUP_MS = 5 * 60_000; // 5 minutes
34+
3235
/**
3336
* Shared-process implementation of {@link IPlaywrightService}.
3437
*
@@ -45,6 +48,12 @@ export class PlaywrightService extends Disposable implements IPlaywrightService
4548
private _browser: Browser | undefined;
4649
private _initPromise: Promise<void> | undefined;
4750

51+
/** In-flight deferred results keyed by their generated ID. */
52+
private readonly _deferredResults = this._register(new DisposableMap<string, {
53+
pageId: string;
54+
promise: Promise<unknown>;
55+
} & IDisposable>());
56+
4857
constructor(
4958
private readonly windowId: number,
5059
private readonly browserViewGroupRemoteService: IBrowserViewGroupRemoteService,
@@ -157,10 +166,23 @@ export class PlaywrightService extends Disposable implements IPlaywrightService
157166
return this._pages.runAgainstPage(pageId, (page) => fn(page, args));
158167
}
159168

160-
async invokeFunction(pageId: string, fnDef: string, ...args: unknown[]): Promise<{ result: unknown; summary: string }> {
169+
private async invokeFunctionWithDeferral<T>(pageId: string, fnDef: string, args: unknown[], timeoutMs: number): Promise<IInvokeFunctionResult> {
170+
await this.initialize();
171+
172+
const vm = await import('vm');
173+
const fn = vm.compileFunction(`return (${fnDef})(page, ...args)`, ['page', 'args'], { parsingContext: vm.createContext() });
174+
175+
return this._runWithDeferral(pageId, (page) => fn(page, args ?? []), timeoutMs);
176+
}
177+
178+
async invokeFunction(pageId: string, fnDef: string, args: unknown[] = [], timeoutMs?: number): Promise<IInvokeFunctionResult> {
161179
this.logService.info(`[PlaywrightService] Invoking function on view ${pageId}`);
162180

163181
try {
182+
if (timeoutMs !== undefined) {
183+
return this.invokeFunctionWithDeferral(pageId, fnDef, args, timeoutMs);
184+
}
185+
164186
let result;
165187
try {
166188
result = await this.invokeFunctionRaw(pageId, fnDef, ...args);
@@ -182,6 +204,66 @@ export class PlaywrightService extends Disposable implements IPlaywrightService
182204
}
183205
}
184206

207+
async waitForDeferredResult(deferredResultId: string, timeoutMs: number): Promise<IInvokeFunctionResult> {
208+
const entry = this._deferredResults.get(deferredResultId);
209+
if (!entry) {
210+
throw new Error(`No deferred result found with ID "${deferredResultId}". It may have been cleaned up or already consumed.`);
211+
}
212+
213+
const { pageId, promise } = entry;
214+
// Remove eagerly — _runWithDeferral will re-insert if interrupted again.
215+
this._deferredResults.deleteAndDispose(deferredResultId);
216+
217+
// The callback ignores the page param since execution is already in-flight.
218+
return this._runWithDeferral(pageId, () => promise, timeoutMs, deferredResultId);
219+
}
220+
221+
/**
222+
* Run a callback against a page with deferred result support.
223+
*/
224+
private async _runWithDeferral(pageId: string, callback: (page: Page) => Promise<unknown>, timeoutMs: number, existingDeferredId?: string): Promise<IInvokeFunctionResult> {
225+
const effectiveTimeout = timeoutMs;
226+
227+
// Start execution via safeRunAgainstPage, but capture the raw promise
228+
// independently so it can be deferred if a dialog or timeout interrupts.
229+
let rawPromise: Promise<unknown> | undefined;
230+
const wrappedPromise = this._pages.runAgainstPage(pageId, async (page) => {
231+
rawPromise = callback(page);
232+
return rawPromise;
233+
});
234+
235+
let result: unknown;
236+
let interrupted = false;
237+
238+
try {
239+
result = await raceTimeout(wrappedPromise, effectiveTimeout, () => { interrupted = true; });
240+
} catch (err: unknown) {
241+
if (err instanceof DialogInterruptedError) {
242+
interrupted = true;
243+
}
244+
result = err instanceof Error ? err.message : String(err);
245+
}
246+
247+
let deferredResultId: string | undefined;
248+
if (interrupted) {
249+
deferredResultId = existingDeferredId ?? generateUuid();
250+
// Prevent unhandled rejection if the deferred result is abandoned
251+
rawPromise!.catch(() => { });
252+
const cleanup = disposableTimeout(() => this._deferredResults.deleteAndDispose(deferredResultId!), DEFERRED_RESULT_CLEANUP_MS);
253+
this._deferredResults.set(deferredResultId, { pageId, promise: rawPromise!, dispose: () => cleanup.dispose() });
254+
255+
this.logService.info(`[PlaywrightService] Execution interrupted, deferred as ${deferredResultId}`);
256+
}
257+
258+
let summary: string;
259+
try {
260+
summary = await this._pages.getSummary(pageId);
261+
} catch (err: unknown) {
262+
summary = err instanceof Error ? err.message : String(err);
263+
}
264+
return { result, summary, deferredResultId };
265+
}
266+
185267
async replyToFileChooser(pageId: string, files: string[]): Promise<{ summary: string }> {
186268
await this.initialize();
187269
const summary = await this._pages.replyToFileChooser(pageId, files);

‎src/vs/platform/browserView/node/playwrightTab.ts‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,18 @@ declare module 'playwright-core' {
1616
}
1717
}
1818

19+
/**
20+
* Thrown when a dialog (alert, confirm, prompt) opens while a page action is
21+
* running. The caller should defer the underlying promise and let the agent
22+
* handle the dialog before retrying.
23+
*/
24+
export class DialogInterruptedError extends Error {
25+
constructor() {
26+
super('Action was interrupted by a dialog');
27+
this.name = 'DialogInterruptedError';
28+
}
29+
}
30+
1931
/**
2032
* Wrapper around a Playwright page that tracks additional state like active dialogs and recent console messages,
2133
* and can produce a summary of the page's current state for use in tools.
@@ -152,7 +164,7 @@ export class PlaywrightTab {
152164
return raceCancellablePromises([dialogOpened, actionCompleted]).then(() => {
153165
if (!actionDidComplete) {
154166
// A dialog was opened before the action completed. Note we don't cancel the action, just ignore its result.
155-
throw new Error('Action was interrupted by a dialog');
167+
throw new DialogInterruptedError();
156168
}
157169
return result!;
158170
});
@@ -185,7 +197,7 @@ export class PlaywrightTab {
185197
`Recent events:`,
186198
...logs.map(log => `- [${new Date(log.time).toISOString()}] (${log.type}) ${log.description}`)
187199
] : []),
188-
...(snapshot ? ['Snapshot:', snapshot] : [])
200+
`Snapshot: ${snapshotFromPage ? snapshot ? `\n${snapshot}` : '<unchanged>' : '<unavailable>'}`,
189201
].join('\n');
190202
}
191203

‎src/vs/workbench/contrib/browserView/electron-browser/tools/browserToolHelpers.ts‎

Lines changed: 34 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import { MarkdownString } from '../../../../../base/common/htmlContent.js';
77
import { URI } from '../../../../../base/common/uri.js';
88
import { localize } from '../../../../../nls.js';
99
import { BrowserViewUri } from '../../../../../platform/browserView/common/browserViewUri.js';
10-
import { IPlaywrightService } from '../../../../../platform/browserView/common/playwrightService.js';
10+
import { IInvokeFunctionResult, IPlaywrightService } from '../../../../../platform/browserView/common/playwrightService.js';
1111
import { IEditorService } from '../../../../services/editor/common/editorService.js';
1212
import { IToolResult } from '../../../chat/common/tools/languageModelToolsService.js';
1313
import { BrowserEditorInput } from '../../common/browserEditorInput.js';
@@ -42,6 +42,9 @@ export async function playwrightInvokeRaw<TArgs extends unknown[], TReturn>(
4242
/**
4343
* Shared helper for running a Playwright function against a page and returning
4444
* a tool result. Handles success/error formatting.
45+
*
46+
* Calls {@link IPlaywrightService.invokeFunction} without a timeout so the
47+
* action runs to completion — no deferred results are ever produced.
4548
*/
4649
export async function playwrightInvoke<TArgs extends unknown[], TReturn>(
4750
playwrightService: IPlaywrightService,
@@ -50,18 +53,41 @@ export async function playwrightInvoke<TArgs extends unknown[], TReturn>(
5053
...args: TArgs
5154
): Promise<IToolResult> {
5255
try {
53-
const result = await playwrightService.invokeFunction(pageId, fn.toString(), ...args);
54-
return {
55-
content: [
56-
{ kind: 'text', value: result.result ? JSON.stringify(result.result) : 'Script executed successfully' },
57-
{ kind: 'text', value: result.summary }
58-
]
59-
};
56+
const result = await playwrightService.invokeFunction(pageId, fn.toString(), args);
57+
return invokeFunctionResultToToolResult(result);
6058
} catch (e) {
6159
return errorResult(e instanceof Error ? e.message : String(e));
6260
}
6361
}
6462

63+
/**
64+
* Convert an {@link IInvokeFunctionResult} to an {@link IToolResult},
65+
* including any {@link IInvokeFunctionResult.deferredResultId}.
66+
*/
67+
export function invokeFunctionResultToToolResult(result: IInvokeFunctionResult, code?: string): IToolResult {
68+
const content: IToolResult['content'] = [];
69+
if (result.result) {
70+
content.push({ kind: 'text', value: JSON.stringify(result.result) });
71+
}
72+
if (result.deferredResultId) {
73+
content.push({ kind: 'text', value: `[deferredResultId=${result.deferredResultId}] The code has not finished executing yet. Call run_playwright_code again with this deferredResultId and the same pageId (no code) to continue waiting.` });
74+
}
75+
content.push({ kind: 'text', value: result.summary });
76+
return {
77+
content,
78+
...(code ? {
79+
toolResultDetails: {
80+
input: code,
81+
inputLanguage: 'javascript',
82+
output: result.result
83+
? [{ type: 'embed' as const, isText: true, value: JSON.stringify(result.result, null, 2) }]
84+
: [],
85+
isError: false,
86+
},
87+
} : {}),
88+
};
89+
}
90+
6591
export function errorResult(message: string): IToolResult {
6692
return {
6793
content: [{ kind: 'text', value: message }],

‎src/vs/workbench/contrib/browserView/electron-browser/tools/runPlaywrightCodeTool.ts‎

Lines changed: 39 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import { MarkdownString } from '../../../../../base/common/htmlContent.js';
99
import { localize } from '../../../../../nls.js';
1010
import { IPlaywrightService } from '../../../../../platform/browserView/common/playwrightService.js';
1111
import { ToolDataSource, type CountTokensCallback, type IPreparedToolInvocation, type IToolData, type IToolImpl, type IToolInvocation, type IToolInvocationPreparationContext, type IToolResult, type ToolProgress } from '../../../chat/common/tools/languageModelToolsService.js';
12-
import { errorResult } from './browserToolHelpers.js';
12+
import { errorResult, invokeFunctionResultToToolResult } from './browserToolHelpers.js';
1313
import { OpenPageToolId } from './openBrowserTool.js';
1414

1515
export const RunPlaywrightCodeToolData: IToolData = {
@@ -29,16 +29,30 @@ export const RunPlaywrightCodeToolData: IToolData = {
2929
},
3030
code: {
3131
type: 'string',
32-
description: `The Playwright code to execute. The code must be concise, serve one clear purpose, and be self-contained. You **must not** directly access \`document\` or \`window\` using this tool. You must access it via the provided \`page\` object, e.g. "return page.evaluate(() => document.title)".`
32+
description: `The Playwright code to execute. The code must be concise, serve one clear purpose, and be self-contained. You **must not** directly access \`document\` or \`window\` using this tool. You must access it via the provided \`page\` object, e.g. "return page.evaluate(() => document.title)". Omit this when resuming a deferred execution via deferredResultId.`
33+
},
34+
deferredResultId: {
35+
type: 'string',
36+
description: `If a previous call returned a deferredResultId, pass it here to continue waiting for that execution to complete.`
37+
},
38+
timeoutMs: {
39+
type: 'number',
40+
description: `Maximum time in milliseconds to wait for the code to complete. Defaults to 5000 (5 seconds).`
3341
},
3442
},
35-
required: ['pageId', 'code'],
43+
required: ['pageId'],
44+
oneOf: [
45+
{ required: ['code'] },
46+
{ required: ['deferredResultId'] },
47+
]
3648
},
3749
};
3850

3951
interface IRunPlaywrightCodeToolParams {
4052
pageId: string;
41-
code: string;
53+
code?: string;
54+
deferredResultId?: string;
55+
timeoutMs?: number;
4256
}
4357

4458
export class RunPlaywrightCodeTool implements IToolImpl {
@@ -48,6 +62,14 @@ export class RunPlaywrightCodeTool implements IToolImpl {
4862

4963
async prepareToolInvocation(context: IToolInvocationPreparationContext, _token: CancellationToken): Promise<IPreparedToolInvocation | undefined> {
5064
const params = context.parameters as IRunPlaywrightCodeToolParams;
65+
66+
if (params.deferredResultId) {
67+
return {
68+
invocationMessage: new MarkdownString(localize('browser.runCode.waitInvocation', "Waiting for Playwright code to complete...")),
69+
pastTenseMessage: new MarkdownString(localize('browser.runCode.waitPast', "Waited for Playwright code")),
70+
};
71+
}
72+
5173
const code = params.code ?? '';
5274
return {
5375
invocationMessage: new MarkdownString(localize('browser.runCode.invocation', "Running Playwright code...")),
@@ -68,41 +90,28 @@ export class RunPlaywrightCodeTool implements IToolImpl {
6890
return errorResult(`No page ID provided. Use '${OpenPageToolId}' first.`);
6991
}
7092

93+
// Resume waiting for a deferred execution
94+
if (params.deferredResultId) {
95+
try {
96+
const result = await this.playwrightService.waitForDeferredResult(params.deferredResultId, params.timeoutMs ?? 5_000);
97+
return invokeFunctionResultToToolResult(result);
98+
} catch (e) {
99+
return errorResult(e instanceof Error ? e.message : String(e));
100+
}
101+
}
102+
71103
if (!params.code) {
72-
return errorResult('The "code" parameter is required.');
104+
return errorResult('Either "code" or "deferredResultId" must be provided.');
73105
}
74106

75107
let result;
76108
try {
77-
result = await this.playwrightService.invokeFunction(params.pageId, `async (page) => { ${params.code} }`);
109+
result = await this.playwrightService.invokeFunction(params.pageId, `async (page) => { ${params.code} }`, undefined, params.timeoutMs ?? 5_000);
78110
} catch (e) {
79111
const message = e instanceof Error ? e.message : String(e);
80112
return errorResult(`Code execution failed: ${message}`);
81113
}
82114

83-
const json = JSON.stringify(result.result || null);
84-
85-
let outputMessage;
86-
if (result.result) {
87-
outputMessage = new MarkdownString();
88-
outputMessage.appendMarkdown(localize('browser.runCode.outputLabel', 'Output:'));
89-
outputMessage.appendText('\n');
90-
outputMessage.appendCodeblock('json', json);
91-
}
92-
93-
return {
94-
content: [
95-
{ kind: 'text', value: result.result ? json : 'Code executed successfully' },
96-
{ kind: 'text', value: result.summary }
97-
],
98-
toolResultDetails: {
99-
input: params.code.trim(),
100-
inputLanguage: 'javascript',
101-
output: result.result
102-
? [{ type: 'embed', isText: true, value: JSON.stringify(result.result, null, 2) }]
103-
: [],
104-
isError: false,
105-
},
106-
};
115+
return invokeFunctionResultToToolResult(result, params.code.trim());
107116
}
108117
}

0 commit comments

Comments
 (0)