ui: add opt-in run_javascript frontend tool (ggml-org#24244)

ServeurpersoCom · web-flow · commit 483609509d58 · 2026-06-09T18:02:31.000+02:00
* ui: add opt-in run_javascript frontend tool

Expose a run_javascript tool to the model, executed entirely in the
browser through the existing agentic loop. Code runs in a Web Worker
inside a sandboxed iframe with an opaque origin, isolated from the
WebUI and its API. Console output, errors and the return value are
fed back as the tool result. The parent enforces a hard timeout by
removing the iframe, which terminates the worker.

Disabled by default, toggle in Settings &gt; Developer.

* ui: address review feedback from allozaur

Use the JsonSchemaType enum for the tool definition parameter types
instead of raw string literals, extending it with STRING and NUMBER.

Move the worker shim and the iframe harness html into their own files
so the service no longer carries inline source blobs.

Replace the remaining magic strings with constants: SANDBOX_EMPTY_OUTPUT
and SANDBOX_TRUNCATION_NOTICE, and reuse NEWLINE_SEPARATOR for joins.

* ui: move sandbox worker shim to a raw imported file

Replace the inline worker template string with a real sandbox-worker.js
imported as raw text, and build the iframe harness from it in
sandbox-harness.ts. The raw worker ships as a string, not a module, so
it is excluded from eslint and the typecheck program.
diff --git a/tools/ui/eslint.config.js b/tools/ui/eslint.config.js
@@ -46,7 +46,14 @@ export default ts.config(
 	},
 	{
 		// Exclude generated build output and Storybook files from ESLint
-		ignores: ['dist/**', 'build/**', '.svelte-kit/**', 'test-results/**', '.storybook/**/*']
+		ignores: [
+			'dist/**',
+			'build/**',
+			'.svelte-kit/**',
+			'test-results/**',
+			'.storybook/**/*',
+			'src/lib/services/sandbox-worker.js'
+		]
 	},
 	storybook.configs['flat/recommended']
 );
diff --git a/tools/ui/src/lib/constants/index.ts b/tools/ui/src/lib/constants/index.ts
@@ -37,6 +37,7 @@ export * from './model-id';
 export * from './precision';
 export * from './processing-info';
 export * from './routes';
+export * from './sandbox';
 export * from './settings-keys';
 export * from './settings-registry';
 export * from './supported-file-types';
diff --git a/tools/ui/src/lib/constants/sandbox.ts b/tools/ui/src/lib/constants/sandbox.ts
@@ -0,0 +1,39 @@
+import { JsonSchemaType, ToolCallType } from '$lib/enums';
+import type { OpenAIToolDefinition } from '$lib/types';
+
+export const SANDBOX_TOOL_NAME = 'run_javascript';
+
+export const SANDBOX_TIMEOUT_MS_DEFAULT = 10000;
+
+export const SANDBOX_TIMEOUT_MS_MAX = 30000;
+
+export const SANDBOX_OUTPUT_MAX_CHARS = 8192;
+
+export const SANDBOX_EMPTY_OUTPUT = '(no output)';
+
+export const SANDBOX_TRUNCATION_NOTICE = '[output truncated]';
+
+export const SANDBOX_TOOL_DEFINITION: OpenAIToolDefinition = {
+	type: ToolCallType.FUNCTION,
+	function: {
+		name: SANDBOX_TOOL_NAME,
+		description:
+			'Execute JavaScript in a sandboxed browser worker (no DOM, no page access). ' +
+			'Top level await is supported. Use console.log to print intermediate values; ' +
+			'a top level return statement is captured as the result.',
+		parameters: {
+			type: JsonSchemaType.OBJECT,
+			properties: {
+				code: {
+					type: JsonSchemaType.STRING,
+					description: 'JavaScript source to execute'
+				},
+				timeout_ms: {
+					type: JsonSchemaType.NUMBER,
+					description: `Execution timeout in milliseconds, default ${SANDBOX_TIMEOUT_MS_DEFAULT}, max ${SANDBOX_TIMEOUT_MS_MAX}`
+				}
+			},
+			required: ['code']
+		}
+	}
+};
diff --git a/tools/ui/src/lib/constants/settings-keys.ts b/tools/ui/src/lib/constants/settings-keys.ts
@@ -69,6 +69,7 @@ export const SETTINGS_KEYS = {
 	ENABLE_THINKING: 'enableThinking',
 	SHOW_RAW_OUTPUT_SWITCH: 'showRawOutputSwitch',
 	// PY_INTERPRETER_ENABLED: 'pyInterpreterEnabled',
+	JS_SANDBOX_ENABLED: 'jsSandboxEnabled',
 	CUSTOM_JSON: 'customJson',
 	CUSTOM_CSS: 'customCss'
 } as const;
diff --git a/tools/ui/src/lib/constants/settings-registry.ts b/tools/ui/src/lib/constants/settings-registry.ts
@@ -690,6 +690,14 @@ const SETTINGS_REGISTRY: Record<string, SettingsSectionEntry> = {
 					paramType: SyncableParameterType.BOOLEAN
 				}
 			},
+			{
+				key: SETTINGS_KEYS.JS_SANDBOX_ENABLED,
+				label: 'JavaScript sandbox tool',
+				help: 'Expose a run_javascript tool to the model. Code runs in a Web Worker inside a sandboxed iframe with an opaque origin, isolated from the WebUI and its API, with a hard timeout.',
+				defaultValue: false,
+				type: SettingsFieldType.CHECKBOX,
+				section: SETTINGS_SECTION_SLUGS.DEVELOPER
+			},
 			{
 				key: SETTINGS_KEYS.CUSTOM_JSON,
 				label: 'Custom JSON',
diff --git a/tools/ui/src/lib/constants/tools.ts b/tools/ui/src/lib/constants/tools.ts
@@ -2,10 +2,12 @@ import { ToolSource } from '$lib/enums/tools.enums';
 
 export const TOOL_GROUP_LABELS = {
 	[ToolSource.BUILTIN]: 'Built-in',
-	[ToolSource.CUSTOM]: 'JSON Schema'
+	[ToolSource.CUSTOM]: 'JSON Schema',
+	[ToolSource.FRONTEND]: 'Browser'
 } as const;
 
 export const TOOL_SERVER_LABELS = {
 	[ToolSource.BUILTIN]: 'Built-in Tools',
-	[ToolSource.CUSTOM]: 'Custom Tools'
+	[ToolSource.CUSTOM]: 'Custom Tools',
+	[ToolSource.FRONTEND]: 'Browser Tools'
 } as const;
diff --git a/tools/ui/src/lib/enums/mcp.enums.ts b/tools/ui/src/lib/enums/mcp.enums.ts
@@ -54,7 +54,9 @@ export enum MCPContentType {
  * JSON Schema types used in MCP tool definitions
  */
 export enum JsonSchemaType {
-	OBJECT = 'object'
+	OBJECT = 'object',
+	STRING = 'string',
+	NUMBER = 'number'
 }
 
 /**
diff --git a/tools/ui/src/lib/enums/tools.enums.ts b/tools/ui/src/lib/enums/tools.enums.ts
@@ -1,7 +1,8 @@
 export enum ToolSource {
 	BUILTIN = 'builtin',
 	MCP = 'mcp',
-	CUSTOM = 'custom'
+	CUSTOM = 'custom',
+	FRONTEND = 'frontend'
 }
 
 export enum ToolPermissionDecision {
diff --git a/tools/ui/src/lib/services/index.ts b/tools/ui/src/lib/services/index.ts
@@ -261,6 +261,26 @@ export { ParameterSyncService } from './parameter-sync.service';
  */
 export { MCPService } from './mcp.service';
 
+/**
+ * **SandboxService** - Frontend JavaScript execution in a browser sandbox
+ *
+ * Stateless executor for the run_javascript frontend tool. Model generated
+ * code runs in a Web Worker spawned inside a sandboxed iframe with an opaque
+ * origin: no access to the app origin, its storage or its API, and outgoing
+ * requests carry a null origin. The code never touches a main thread, so the
+ * parent enforces the timeout by removing the iframe, which terminates the
+ * worker at the browser level.
+ *
+ * **Architecture & Relationships:**
+ * - **SandboxService** (this class): Stateless sandbox execution
+ * - **toolsStore**: Exposes the tool definition when the sandbox is enabled
+ * - **agenticStore**: Dispatches ToolSource.FRONTEND calls here
+ *
+ * @see SANDBOX_TOOL_DEFINITION in constants/sandbox.ts - tool schema sent to the LLM
+ * @see agenticStore in stores/agentic.svelte.ts - tool dispatch
+ */
+export { SandboxService } from './sandbox.service';
+
 /**
  * **RouterService** — Dynamic route URL construction utility
  *
diff --git a/tools/ui/src/lib/services/sandbox-harness.ts b/tools/ui/src/lib/services/sandbox-harness.ts
@@ -0,0 +1,25 @@
+import WORKER_SHIM from './sandbox-worker.js?raw';
+
+/**
+ * Harness loaded as srcdoc into a sandboxed iframe (allow-scripts only).
+ * The opaque origin is the security boundary: no access to the app origin,
+ * its storage or its API. The harness spawns a worker so model code never
+ * runs on a main thread, which makes the parent timeout enforceable by
+ * removing the iframe.
+ */
+export const SANDBOX_HARNESS_HTML = `<!doctype html><script>
+const SHIM = ${JSON.stringify(WORKER_SHIM)};
+addEventListener('message', (event) => {
+	const respond = (payload) => parent.postMessage(payload, '*');
+	let worker;
+	try {
+		worker = new Worker(URL.createObjectURL(new Blob([SHIM], { type: 'text/javascript' })));
+	} catch (err) {
+		respond({ logs: [], result: null, error: 'Worker creation failed: ' + err });
+		return;
+	}
+	worker.onmessage = (msg) => respond(msg.data);
+	worker.onerror = (err) => respond({ logs: [], result: null, error: String(err.message || err) });
+	worker.postMessage({ code: event.data.code });
+});
+</script>`;
diff --git a/tools/ui/src/lib/services/sandbox-worker.js b/tools/ui/src/lib/services/sandbox-worker.js
@@ -0,0 +1,30 @@
+const logs = [];
+const fmt = (value) => {
+	if (typeof value === 'string') return value;
+	try {
+		return JSON.stringify(value);
+	} catch {
+		return String(value);
+	}
+};
+const capture =
+	(level, prefix) =>
+	(...args) => {
+		logs.push(prefix + args.map(fmt).join(' '));
+	};
+console.log = capture('log', '');
+console.info = capture('info', '');
+console.debug = capture('debug', '');
+console.warn = capture('warn', 'warn: ');
+console.error = capture('error', 'error: ');
+self.onmessage = async (event) => {
+	const reply = { logs, result: null, error: null };
+	try {
+		const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor;
+		const value = await new AsyncFunction(event.data.code)();
+		if (value !== undefined) reply.result = fmt(value);
+	} catch (err) {
+		reply.error = err instanceof Error ? err.stack || err.message : String(err);
+	}
+	self.postMessage(reply);
+};
diff --git a/tools/ui/src/lib/services/sandbox.service.ts b/tools/ui/src/lib/services/sandbox.service.ts
@@ -0,0 +1,112 @@
+import {
+	NEWLINE_SEPARATOR,
+	SANDBOX_EMPTY_OUTPUT,
+	SANDBOX_OUTPUT_MAX_CHARS,
+	SANDBOX_TIMEOUT_MS_DEFAULT,
+	SANDBOX_TIMEOUT_MS_MAX,
+	SANDBOX_TOOL_NAME,
+	SANDBOX_TRUNCATION_NOTICE
+} from '$lib/constants';
+import { SANDBOX_HARNESS_HTML } from './sandbox-harness';
+import type { ToolExecutionResult } from '$lib/types';
+
+interface SandboxReply {
+	logs?: unknown;
+	result?: unknown;
+	error?: unknown;
+}
+
+function formatReply(reply: SandboxReply): ToolExecutionResult {
+	const lines: string[] = [];
+
+	if (Array.isArray(reply.logs)) {
+		for (const line of reply.logs) lines.push(String(line));
+	}
+
+	if (reply.error != null) {
+		lines.push(`Error: ${String(reply.error)}`);
+	} else if (reply.result != null) {
+		lines.push(`=> ${String(reply.result)}`);
+	}
+
+	let content = lines.join(NEWLINE_SEPARATOR);
+	if (!content) content = SANDBOX_EMPTY_OUTPUT;
+	if (content.length > SANDBOX_OUTPUT_MAX_CHARS) {
+		content = `${content.slice(0, SANDBOX_OUTPUT_MAX_CHARS)}${NEWLINE_SEPARATOR}${SANDBOX_TRUNCATION_NOTICE}`;
+	}
+
+	return { content, isError: reply.error != null };
+}
+
+export class SandboxService {
+	/**
+	 * Execute a frontend sandbox tool call and return its output.
+	 * One disposable iframe per execution, removed on completion,
+	 * timeout or abort. Removing the iframe terminates the worker
+	 * at the browser level, so runaway code cannot outlive it.
+	 */
+	static executeTool(
+		toolName: string,
+		params: Record<string, unknown>,
+		signal?: AbortSignal
+	): Promise<ToolExecutionResult> {
+		if (toolName !== SANDBOX_TOOL_NAME) {
+			return Promise.resolve({ content: `Unknown frontend tool: ${toolName}`, isError: true });
+		}
+
+		const code = typeof params.code === 'string' ? params.code : '';
+		if (!code) {
+			return Promise.resolve({ content: 'Missing required parameter: code', isError: true });
+		}
+
+		const requested = Number(params.timeout_ms);
+		const timeoutMs =
+			Number.isFinite(requested) && requested > 0
+				? Math.min(requested, SANDBOX_TIMEOUT_MS_MAX)
+				: SANDBOX_TIMEOUT_MS_DEFAULT;
+
+		return new Promise<ToolExecutionResult>((resolve, reject) => {
+			const iframe = document.createElement('iframe');
+			iframe.setAttribute('sandbox', 'allow-scripts');
+			iframe.style.display = 'none';
+			iframe.srcdoc = SANDBOX_HARNESS_HTML;
+
+			let settled = false;
+
+			const cleanup = () => {
+				settled = true;
+				clearTimeout(timer);
+				window.removeEventListener('message', onMessage);
+				signal?.removeEventListener('abort', onAbort);
+				iframe.remove();
+			};
+
+			const finish = (result: ToolExecutionResult) => {
+				if (settled) return;
+				cleanup();
+				resolve(result);
+			};
+
+			const onAbort = () => {
+				if (settled) return;
+				cleanup();
+				reject(new DOMException('Sandbox execution aborted', 'AbortError'));
+			};
+
+			const onMessage = (event: MessageEvent) => {
+				if (event.source !== iframe.contentWindow) return;
+				finish(formatReply((event.data ?? {}) as SandboxReply));
+			};
+
+			const timer = setTimeout(
+				() => finish({ content: `Execution timed out after ${timeoutMs} ms`, isError: true }),
+				timeoutMs
+			);
+
+			window.addEventListener('message', onMessage);
+			signal?.addEventListener('abort', onAbort);
+			iframe.onload = () => iframe.contentWindow?.postMessage({ code }, '*');
+			document.body.appendChild(iframe);
+		});
+	}
+}
diff --git a/tools/ui/src/lib/stores/agentic.svelte.ts b/tools/ui/src/lib/stores/agentic.svelte.ts
@@ -29,6 +29,7 @@ import { permissionsStore } from '$lib/stores/permissions.svelte';
 import { ToolSource, ToolPermissionDecision } from '$lib/enums';
 import { SvelteMap } from 'svelte/reactivity';
 import { ToolsService } from '$lib/services/tools.service';
+import { SandboxService } from '$lib/services/sandbox.service';
 import { isAbortError } from '$lib/utils';
 import { DEFAULT_AGENTIC_CONFIG, NEWLINE_SEPARATOR } from '$lib/constants';
 import {
@@ -784,6 +785,13 @@ class AgenticStore {
 
 							result = executionResult.content;
 
+							if (executionResult.isError) toolSuccess = false;
+						} else if (toolSource === ToolSource.FRONTEND) {
+							const args = this.parseToolArguments(toolCall.function.arguments);
+							const executionResult = await SandboxService.executeTool(toolName, args, signal);
+
+							result = executionResult.content;
+
 							if (executionResult.isError) toolSuccess = false;
 						} else {
 							const mcpCall: MCPToolCall = {
diff --git a/tools/ui/src/lib/stores/tools.svelte.ts b/tools/ui/src/lib/stores/tools.svelte.ts
diff --git a/tools/ui/tsconfig.json b/tools/ui/tsconfig.json

Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,8 @@`
`1`	`1`	`export enum ToolSource {`
`2`	`2`	`BUILTIN = 'builtin',`
`3`	`3`	`MCP = 'mcp',`
`4`		`- CUSTOM = 'custom'`
	`4`	`+ CUSTOM = 'custom',`
	`5`	`+ FRONTEND = 'frontend'`
`5`	`6`	`}`
`6`	`7`
`7`	`8`	`export enum ToolPermissionDecision {`