Skip to content

Commit 1412812

Browse files
esokulluesokullu
committed
Restore API-driven cloud browser runs
1 parent 557f6ab commit 1412812

13 files changed

Lines changed: 1038 additions & 11 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ This changelog was generated from the repository Git history and release tags. V
1010
- Added a selection shortcut for Chrome and Firefox with Summarize, Explain, Quiz me, Proofread, Translate, and custom WebBrain prompts.
1111
- Expanded the native selection context menu with matching preset actions, translation languages, and direct side-panel access.
1212
- Added a persistent setting to hide or restore the floating selection shortcut.
13+
- Added the managed cloud-browser bridge for API-driven run, status, abort, active-tab control, and validated structured results.
1314

1415
### Changed
1516
- Simplified the selection shortcut to a compact purple question-mark icon.

‎src/chrome/src/agent/agent.js‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { AGENT_TOOLS, AGENT_TOOL_NAMES, RESERVED_AGENT_TOOL_NAMES, getToolsForMode, SYSTEM_PROMPT_ASK, SYSTEM_PROMPT_ACT, SYSTEM_PROMPT_ACT_COMPACT, SYSTEM_PROMPT_ACT_MID, SYSTEM_PROMPT_DEV_APPENDIX } from './tools.js';
2+
import { handleDoneJson } from './cloud-output.js';
23
import { URL_FAMILY_TOOLS, resourceBucket, bucketArgsKey } from './loop-bucket.js';
34
import { isCredentialField, CREDENTIAL_NOTE_STRICT } from './credential-fields.js';
45
import { detectProgressAction, formatLedgerRow, formatLedgerSummary, isBlockedLedgerDowngrade, isTerminalLedgerStatus, isValidLedgerStatus, ledgerDoneBlock, normalizeLedgerStatus, progressCounts, selectLedgerRows, unresolvedLedgerRows, upsertLedgerItems } from './progress-ledger.js';
@@ -224,6 +225,7 @@ export class Agent {
224225
// abort() and clearConversation() cancel all pending clarifications so
225226
// the agent loop doesn't deadlock.
226227
this._pendingClarifications = new Map();
228+
this.cloudRunContexts = new Map(); // tabId -> { outputSchema, schemaRepairUsed }
227229
// Deterministic capability × origin permission gate. "Always" grants are
228230
// persisted in extension storage; "once" grants live for the current turn.
229231
this.permissions = new PermissionManager({
@@ -8071,6 +8073,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
80718073
}
80728074

80738075
async executeTool(tabId, name, args, onUpdate = null) {
8076+
if (name === 'done_json') {
8077+
return handleDoneJson(this.cloudRunContexts.get(tabId), args);
8078+
}
80748079
if (name === 'get_window_info') {
80758080
return await this._getWindowInfo(tabId);
80768081
}
@@ -11595,10 +11600,18 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
1159511600
}
1159611601
this._clearLoopState(tabId);
1159711602
this._runningTabs.add(tabId);
11603+
const previousCloudContext = this.cloudRunContexts.get(tabId);
11604+
if (runOptions.cloudRun) {
11605+
this.cloudRunContexts.set(tabId, { outputSchema: runOptions.outputSchema || null, schemaRepairUsed: false });
11606+
}
1159811607
try {
1159911608
return await this._processMessageInner(tabId, userMessage, onUpdate, mode, attachments, runOptions);
1160011609
} finally {
1160111610
this.currentCostState.delete(tabId);
11611+
if (runOptions.cloudRun) {
11612+
if (previousCloudContext) this.cloudRunContexts.set(tabId, previousCloudContext);
11613+
else this.cloudRunContexts.delete(tabId);
11614+
}
1160211615
this._runningTabs.delete(tabId);
1160311616
this._clearLoopState(tabId);
1160411617
}
@@ -11746,7 +11759,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
1174611759
}
1174711760
const tier = provider.promptTier;
1174811761
const skillTools = this._skillToolDefinitions(mode, tier);
11749-
const tools = getToolsForMode(mode, { strictSecretMode: this.strictSecretMode, tier, skillTools });
11762+
const cloudRunContext = this.cloudRunContexts.get(tabId) || null;
11763+
const tools = getToolsForMode(mode, {
11764+
strictSecretMode: this.strictSecretMode,
11765+
tier,
11766+
skillTools,
11767+
cloudRun: !!cloudRunContext,
11768+
outputSchema: cloudRunContext?.outputSchema || null,
11769+
});
1175011770
const allowedToolNames = new Set(tools.map(t => t.function.name));
1175111771
const plannerTemperature = this._isActionMode(mode) ? 0.15 : 0.3;
1175211772
let steps = 0;
@@ -12058,10 +12078,18 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
1205812078
}
1205912079
this._clearLoopState(tabId);
1206012080
this._runningTabs.add(tabId);
12081+
const previousCloudContext = this.cloudRunContexts.get(tabId);
12082+
if (runOptions.cloudRun) {
12083+
this.cloudRunContexts.set(tabId, { outputSchema: runOptions.outputSchema || null, schemaRepairUsed: false });
12084+
}
1206112085
try {
1206212086
return await this._processMessageStreamInner(tabId, userMessage, onUpdate, mode, runOptions);
1206312087
} finally {
1206412088
this.currentCostState.delete(tabId);
12089+
if (runOptions.cloudRun) {
12090+
if (previousCloudContext) this.cloudRunContexts.set(tabId, previousCloudContext);
12091+
else this.cloudRunContexts.delete(tabId);
12092+
}
1206512093
this._runningTabs.delete(tabId);
1206612094
this._clearLoopState(tabId);
1206712095
}
@@ -12129,7 +12157,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
1212912157
}
1213012158
const tier = provider.promptTier;
1213112159
const skillTools = this._skillToolDefinitions(mode, tier);
12132-
const tools = getToolsForMode(mode, { strictSecretMode: this.strictSecretMode, tier, skillTools });
12160+
const cloudRunContext = this.cloudRunContexts.get(tabId) || null;
12161+
const tools = getToolsForMode(mode, {
12162+
strictSecretMode: this.strictSecretMode,
12163+
tier,
12164+
skillTools,
12165+
cloudRun: !!cloudRunContext,
12166+
outputSchema: cloudRunContext?.outputSchema || null,
12167+
});
1213312168
const allowedToolNames = new Set(tools.map(t => t.function.name));
1213412169
const plannerTemperature = this._isActionMode(mode) ? 0.15 : 0.3;
1213512170
let steps = 0;

‎src/chrome/src/agent/cloud-output.js‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
export function validateCloudOutput(value, schema) {
2+
const errors = [];
3+
const push = (path, message) => errors.push(`${path}: ${message}`);
4+
const isObject = item => !!item && typeof item === 'object' && !Array.isArray(item);
5+
const keywords = new Set(['type', 'properties', 'required', 'items', 'enum', 'description', 'additionalProperties']);
6+
7+
const validate = (item, spec, path = '$') => {
8+
if (typeof spec === 'string') {
9+
let shorthand = spec.trim();
10+
const optional = shorthand.endsWith('?');
11+
if (optional) shorthand = shorthand.slice(0, -1).trim();
12+
if ((item === undefined || item === null) && optional) return;
13+
if (shorthand.endsWith('[]')) {
14+
if (!Array.isArray(item)) {
15+
push(path, `expected array of ${shorthand.slice(0, -2)}`);
16+
return;
17+
}
18+
item.forEach((child, index) => validate(child, shorthand.slice(0, -2) || 'any', `${path}[${index}]`));
19+
return;
20+
}
21+
const valid = shorthand === 'any'
22+
|| (shorthand === 'string' && typeof item === 'string')
23+
|| (shorthand === 'number' && typeof item === 'number' && !Number.isNaN(item))
24+
|| (shorthand === 'integer' && Number.isInteger(item))
25+
|| (shorthand === 'boolean' && typeof item === 'boolean')
26+
|| (shorthand === 'object' && isObject(item))
27+
|| (shorthand === 'array' && Array.isArray(item));
28+
if (!valid) push(path, shorthand === 'any' ? 'unsupported value' : `expected ${shorthand}`);
29+
return;
30+
}
31+
32+
if (Array.isArray(spec)) {
33+
if (!Array.isArray(item)) {
34+
push(path, 'expected array');
35+
return;
36+
}
37+
item.forEach((child, index) => validate(child, spec[0] || 'any', `${path}[${index}]`));
38+
return;
39+
}
40+
41+
if (!isObject(spec)) return;
42+
const jsonSchema = Object.keys(spec).some(key => keywords.has(key));
43+
if (!jsonSchema) {
44+
if (!isObject(item)) {
45+
push(path, 'expected object');
46+
return;
47+
}
48+
for (const [key, childSpec] of Object.entries(spec)) {
49+
const optional = typeof childSpec === 'string' && childSpec.trim().endsWith('?');
50+
if (!(key in item)) {
51+
if (!optional) push(`${path}.${key}`, 'missing required property');
52+
continue;
53+
}
54+
validate(item[key], childSpec, `${path}.${key}`);
55+
}
56+
return;
57+
}
58+
59+
if (Array.isArray(spec.enum) && !spec.enum.includes(item)) {
60+
push(path, `expected one of ${JSON.stringify(spec.enum)}`);
61+
}
62+
const types = Array.isArray(spec.type) ? spec.type : (spec.type ? [spec.type] : []);
63+
if (types.length) {
64+
const typeOk = types.some(type => {
65+
if (type === 'array') return Array.isArray(item);
66+
if (type === 'object') return isObject(item);
67+
if (type === 'integer') return Number.isInteger(item);
68+
if (type === 'number') return typeof item === 'number' && !Number.isNaN(item);
69+
if (type === 'null') return item === null;
70+
return typeof item === type;
71+
});
72+
if (!typeOk) push(path, `expected ${types.join(' or ')}`);
73+
}
74+
if (spec.properties || spec.required) {
75+
if (!isObject(item)) {
76+
push(path, 'expected object with properties');
77+
return;
78+
}
79+
for (const key of Array.isArray(spec.required) ? spec.required : []) {
80+
if (!(key in item)) push(`${path}.${key}`, 'missing required property');
81+
}
82+
for (const [key, childSpec] of Object.entries(spec.properties || {})) {
83+
if (key in item) validate(item[key], childSpec, `${path}.${key}`);
84+
}
85+
if (spec.additionalProperties === false) {
86+
const allowed = new Set(Object.keys(spec.properties || {}));
87+
for (const key of Object.keys(item)) {
88+
if (!allowed.has(key)) push(`${path}.${key}`, 'additional property is not allowed');
89+
}
90+
}
91+
}
92+
if (spec.items && Array.isArray(item)) {
93+
item.forEach((child, index) => validate(child, spec.items, `${path}[${index}]`));
94+
}
95+
};
96+
97+
validate(value, schema);
98+
return { ok: errors.length === 0, errors };
99+
}
100+
export function handleDoneJson(context, args = {}) {
101+
if (!context?.outputSchema) {
102+
return {
103+
success: false,
104+
error: 'done_json is only available during a cloud run with an output schema.',
105+
};
106+
}
107+
const result = Object.prototype.hasOwnProperty.call(args, 'result') ? args.result : undefined;
108+
const summary = String(args.summary || '').trim() || 'Task completed.';
109+
const validation = validateCloudOutput(result, context.outputSchema);
110+
if (validation.ok) {
111+
return { done: true, doneJson: true, summary, result, cloudResult: result };
112+
}
113+
const message = `done_json result did not match outputSchema: ${validation.errors.slice(0, 8).join('; ')}`;
114+
if (!context.schemaRepairUsed) {
115+
context.schemaRepairUsed = true;
116+
return {
117+
success: false,
118+
schemaValidationError: true,
119+
error: `${message}. Call done_json exactly one more time with a corrected result.`,
120+
expectedSchema: context.outputSchema,
121+
};
122+
}
123+
return {
124+
done: true,
125+
doneJson: true,
126+
cloudFailed: true,
127+
schemaValidationError: true,
128+
summary: 'Structured cloud run failed schema validation.',
129+
error: message,
130+
expectedSchema: context.outputSchema,
131+
invalidResult: result,
132+
};
133+
}

‎src/chrome/src/agent/tools.js‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -884,7 +884,7 @@ export const ASK_ONLY_TOOLS = [
884884
*/
885885
export const AGENT_TOOL_NAMES = new Set(AGENT_TOOLS.map(t => t.function.name));
886886
export const RETIRED_AGENT_TOOL_NAMES = new Set(['screenshot', 'full_page_screenshot', 'record_tab', 'stop_recording']);
887-
export const RESERVED_AGENT_TOOL_NAMES = new Set([...AGENT_TOOL_NAMES, ...RETIRED_AGENT_TOOL_NAMES]);
887+
export const RESERVED_AGENT_TOOL_NAMES = new Set([...AGENT_TOOL_NAMES, ...RETIRED_AGENT_TOOL_NAMES, 'done_json']);
888888
export const DEV_ONLY_TOOL_NAMES = new Set(['read_page_source', 'inspect_element_styles']);
889889
export const DEV_EXTENDED_TOOL_NAMES = new Set([
890890
...DEV_ONLY_TOOL_NAMES,
@@ -956,6 +956,22 @@ const DONE_TOOL_STRICT_WITH_OUTCOME = {
956956
},
957957
};
958958

959+
const DONE_JSON_TOOL = {
960+
type: 'function',
961+
function: {
962+
name: 'done_json',
963+
description: 'Complete a structured cloud run. Call this only when the task is finished and result exactly matches the requested output schema. If validation fails, repair the result and call done_json once more.',
964+
parameters: {
965+
type: 'object',
966+
properties: {
967+
result: { type: 'object', description: 'Machine-readable result matching the requested output schema.' },
968+
summary: { type: 'string', description: 'Short human-readable completion summary.' },
969+
},
970+
required: ['result', 'summary'],
971+
},
972+
},
973+
};
974+
959975
/**
960976
* Get tools filtered by mode.
961977
*
@@ -996,6 +1012,8 @@ export function getToolsForMode(mode, opts = {}) {
9961012
});
9971013
base = [...base, ...extras];
9981014
}
1015+
const useDoneJson = normalizedMode === 'act' && tier === 'full' && opts.cloudRun === true && !!opts.outputSchema;
1016+
if (useDoneJson) return base.map(tool => (tool.function.name === 'done' ? DONE_JSON_TOOL : tool));
9991017
const useOutcomeDone = normalizedMode !== 'ask' && tier !== 'compact';
10001018
if (!opts.strictSecretMode && !useOutcomeDone) return base;
10011019
const replacement = opts.strictSecretMode

‎src/chrome/src/background.js‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,14 @@ import {
1818
getClaudeOAuthStatus,
1919
} from './providers/oauth-claude.js';
2020
import { getBalance as capsolverGetBalance } from './agent/captcha-solver.js';
21+
import { createCloudRunController } from './cloud-runs.js';
22+
import { ensureOffscreen } from './offscreen/ensure.js';
2123
import {
2224
SELECTION_TRANSLATION_LANGUAGES,
2325
buildContextMenuPrompt,
2426
buildSelectionPrompt,
2527
createContextMenuStorage,
2628
} from './context-menu-storage.js';
27-
// (ensureOffscreen + transcribeAudio used to be imported here; both are
28-
// now consumed inside src/recorder/host.js, which background.js calls into.)
2929
import {
3030
prepareRecordingHost,
3131
startTabRecording,
@@ -91,6 +91,14 @@ scheduler.start();
9191
// happen AFTER providerManager is constructed.
9292
setRecorderProviderManager(providerManager);
9393

94+
const cloudRunController = createCloudRunController({
95+
chromeApi: chrome,
96+
agent,
97+
ensureOffscreen,
98+
sendIndicator: (tabId, type) => sendIndicatorMessage(tabId, type),
99+
});
100+
cloudRunController.syncBridge().catch(() => {});
101+
94102
const MAX_AGENT_STEPS_DEFAULT = 130;
95103
const MAX_AGENT_STEPS_UNLIMITED_SENTINEL = 200;
96104
const CONTEXT_MENU_ASK_SELECTION_ID = 'webbrain-ask-selection';
@@ -679,6 +687,7 @@ chrome.runtime.onInstalled.addListener(async () => {
679687
await providerManager.load();
680688
await loadMaxSteps();
681689
await syncAgentUserMemoryFromStorage().catch(() => {});
690+
await cloudRunController.syncBridge().catch(() => {});
682691
scheduleUserMemoryExtractionDrain(5000);
683692
console.log('[WebBrain] Extension installed, providers loaded.');
684693
});
@@ -689,13 +698,17 @@ chrome.runtime.onStartup?.addListener(async () => {
689698
await providerManager.load();
690699
await loadMaxSteps();
691700
await syncAgentUserMemoryFromStorage().catch(() => {});
701+
await cloudRunController.syncBridge().catch(() => {});
692702
scheduleUserMemoryExtractionDrain(5000);
693703
});
694704

695705
// Listen for setting changes
696706
chrome.storage.onChanged.addListener((changes) => {
697707
if (PROFILE_SYNC_DATA_KEYS.some((key) => changes[key])) profileSync.noteChanges(changes).catch(() => {});
698708
if (changes.providers || changes.activeProvider) providerManager.load().catch(() => {});
709+
if (changes.webbrainCloudBridgeEnabled || changes.webbrainCloudBridgeUrl) {
710+
cloudRunController.syncBridge().catch(() => {});
711+
}
699712
if (changes.maxAgentSteps) {
700713
agent.maxSteps = normalizeMaxAgentSteps(changes.maxAgentSteps.newValue);
701714
}
@@ -1501,6 +1514,18 @@ async function handleMessage(msg, sender) {
15011514
}
15021515

15031516
switch (msg.action) {
1517+
case 'cloud_run':
1518+
return await cloudRunController.startRun(msg);
1519+
case 'cloud_status':
1520+
return await cloudRunController.status(msg);
1521+
case 'cloud_abort':
1522+
return await cloudRunController.abort(msg);
1523+
case 'cloud_bridge_start':
1524+
return await cloudRunController.startBridge(msg.url);
1525+
case 'cloud_bridge_stop':
1526+
return await cloudRunController.stopBridge();
1527+
case 'cloud_bridge_status':
1528+
return await cloudRunController.bridgeStatus();
15041529
case 'prepare_recording_host':
15051530
return await prepareRecordingHost();
15061531
case 'start_tab_recording': {

0 commit comments

Comments
 (0)