Skip to content

Commit 5e2bab3

Browse files
esokulluesokullu
committed
verify CAPTCHA gates across frames
1 parent 2b2f6ee commit 5e2bab3

3 files changed

Lines changed: 241 additions & 34 deletions

File tree

‎src/chrome/src/agent/agent.js‎

Lines changed: 88 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,7 @@ export class Agent extends LoopDetector {
251251
// asking the user. The agent reads the key from chrome.storage.local
252252
// at call time so rotating the key doesn't require a restart.
253253
this.captchaSolverEnabled = false;
254-
this._captchaGateStates = new Map(); // tabId -> { key, status, publicGate }
254+
this._captchaGateStates = new Map(); // tabId -> { key, status, publicGate, challengeFrameId? }
255255
// Pre-execution planner (Settings → Plan before Act). Default "try";
256256
// attempts a read-only planning LLM call and degrades the current turn to
257257
// Ask/read-only if structured planning itself fails. "strict" fails closed.
@@ -2760,13 +2760,29 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
27602760
frameContexts,
27612761
navigationFrames,
27622762
);
2763-
return visibleCandidates.find(candidate => candidate.visible === true)?.challenge || null;
2763+
const candidate = visibleCandidates.find(entry => entry.visible === true);
2764+
return candidate
2765+
? {
2766+
...candidate.challenge,
2767+
frameId: candidate.frameId,
2768+
frameUrl: candidate.frameUrl || '',
2769+
}
2770+
: null;
27642771
}
27652772

2766-
async _detectChallengeDialogBeforeMutation(tabId) {
2773+
async _detectChallengeDialogBeforeMutation(tabId, options = {}) {
2774+
const includeStatus = options?.includeStatus === true;
2775+
const expectedFrameId = Number.isInteger(options?.expectedFrameId)
2776+
? options.expectedFrameId
2777+
: null;
27672778
let navigationFrames = [];
2779+
let navigationInspectionComplete = false;
27682780
try {
2769-
navigationFrames = await chrome.webNavigation?.getAllFrames?.({ tabId }) || [];
2781+
const discoveredFrames = await chrome.webNavigation?.getAllFrames?.({ tabId });
2782+
if (Array.isArray(discoveredFrames)) {
2783+
navigationFrames = discoveredFrames;
2784+
navigationInspectionComplete = true;
2785+
}
27702786
} catch {}
27712787
if (!navigationFrames.length) {
27722788
navigationFrames = [{ frameId: 0, parentFrameId: -1, url: '' }];
@@ -2776,27 +2792,37 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
27762792
func: detectChallengeDialogInPage,
27772793
args: [{ includeFrameContext: true }],
27782794
});
2795+
const inspected = (results) => {
2796+
const entries = (results || []).map(entry => ({
2797+
frameId: Number.isInteger(entry?.frameId) ? entry.frameId : 0,
2798+
payload: entry?.result,
2799+
}));
2800+
const inspectedFrameIds = new Set(entries
2801+
.filter(entry => entry.payload && typeof entry.payload === 'object')
2802+
.map(entry => entry.frameId));
2803+
const expectedFrameStillExists = expectedFrameId !== null
2804+
&& navigationFrames.some(frame => frame?.frameId === expectedFrameId);
2805+
const challenge = this._visibleChallengeDialogFromFrames(entries, navigationFrames);
2806+
return includeStatus
2807+
? {
2808+
challenge,
2809+
inspectionComplete: expectedFrameId === null
2810+
|| inspectedFrameIds.has(expectedFrameId)
2811+
|| (navigationInspectionComplete && !expectedFrameStillExists),
2812+
}
2813+
: challenge;
2814+
};
27792815
try {
27802816
const results = await execute({ tabId, allFrames: true });
2781-
return this._visibleChallengeDialogFromFrames(
2782-
(results || []).map(entry => ({
2783-
frameId: Number.isInteger(entry?.frameId) ? entry.frameId : 0,
2784-
payload: entry?.result,
2785-
})),
2786-
navigationFrames,
2787-
);
2817+
return inspected(results);
27882818
} catch {
27892819
try {
27902820
const results = await execute({ tabId });
2791-
return this._visibleChallengeDialogFromFrames(
2792-
(results || []).map(entry => ({
2793-
frameId: Number.isInteger(entry?.frameId) ? entry.frameId : 0,
2794-
payload: entry?.result,
2795-
})),
2796-
navigationFrames,
2797-
);
2821+
return inspected(results);
27982822
} catch {
2799-
return null;
2823+
return includeStatus
2824+
? { challenge: null, inspectionComplete: false }
2825+
: null;
28002826
}
28012827
}
28022828
}
@@ -2806,7 +2832,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
28062832
const gatedAction = this._isBrowserMutationTool(toolName)
28072833
|| gatedCompletion
28082834
|| isNetworkMutation(toolName, toolArgs);
2809-
if (!this.captchaSolverEnabled || !gatedAction || toolName === 'solve_captcha') return null;
2835+
if (!gatedAction || toolName === 'solve_captcha') return null;
28102836
const activeGate = this._captchaGateStates.get(tabId);
28112837
if (activeGate && !this._shouldRetryCaptchaManualGate(activeGate)) return null;
28122838
const challenge = await this._detectChallengeDialogBeforeMutation(tabId);
@@ -2819,6 +2845,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
28192845
{
28202846
pageContent: `dialog ${JSON.stringify(String(challenge.label).slice(0, 200))}`,
28212847
pageUrl,
2848+
captchaChallengeFrameId: Number.isInteger(challenge.frameId)
2849+
? challenge.frameId
2850+
: null,
2851+
captchaChallengeFrameUrl: challenge.frameUrl || '',
28222852
},
28232853
{},
28242854
);
@@ -2868,6 +2898,33 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
28682898
&& toolResult.truncated !== true
28692899
&& toolResult.hasMore !== true
28702900
&& toolResult.autoDegraded !== true;
2901+
if (
2902+
!challenge
2903+
&& activeGate
2904+
&& authoritativeRootRead
2905+
&& Number.isInteger(activeGate.challengeFrameId)
2906+
) {
2907+
const frameInspection = await this._detectChallengeDialogBeforeMutation(tabId, {
2908+
includeStatus: true,
2909+
expectedFrameId: activeGate.challengeFrameId,
2910+
});
2911+
if (frameInspection.challenge?.label) {
2912+
challenge = detectChallengeDialog(
2913+
`dialog ${JSON.stringify(String(frameInspection.challenge.label).slice(0, 200))}`
2914+
);
2915+
} else if (!frameInspection.inspectionComplete) {
2916+
const guardedGate = {
2917+
...activeGate.publicGate,
2918+
verificationFrameReadRequired: true,
2919+
};
2920+
this._captchaGateStates.set(tabId, {
2921+
...activeGate,
2922+
publicGate: guardedGate,
2923+
});
2924+
toolResult.captchaGate = guardedGate;
2925+
return { gate: guardedGate, loopCheck: { kind: 'none' } };
2926+
}
2927+
}
28712928
const loopCheck = challenge || authoritativeRootRead
28722929
? this._checkVerificationChallengeLoop(tabId, {
28732930
pageUrl,
@@ -3019,7 +3076,17 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
30193076
...(detection?.error ? { selectionFailed: true } : {}),
30203077
...(detection?.selected && !selectedCorrelated ? { candidateNotCorrelated: true } : {}),
30213078
};
3022-
this._captchaGateStates.set(tabId, { key, status: publicGate.status, publicGate });
3079+
this._captchaGateStates.set(tabId, {
3080+
key,
3081+
status: publicGate.status,
3082+
publicGate,
3083+
...(Number.isInteger(toolResult.captchaChallengeFrameId)
3084+
? {
3085+
challengeFrameId: toolResult.captchaChallengeFrameId,
3086+
challengeFrameUrl: String(toolResult.captchaChallengeFrameUrl || ''),
3087+
}
3088+
: {}),
3089+
});
30233090
toolResult.captchaGate = publicGate;
30243091
return { gate: publicGate, loopCheck };
30253092
}

‎src/firefox/src/agent/agent.js‎

Lines changed: 78 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,7 @@ export class Agent extends LoopDetector {
211211
// model to try `solve_captcha` once before falling back to asking
212212
// the user. The API key is read at call time from browser.storage.
213213
this.captchaSolverEnabled = false;
214-
this._captchaGateStates = new Map(); // tabId -> { key, status, publicGate }
214+
this._captchaGateStates = new Map(); // tabId -> { key, status, publicGate, challengeFrameId? }
215215
// Pre-execution planner (Settings → Plan before Act). Default "try";
216216
// attempts a read-only planning LLM call and degrades the current turn to
217217
// Ask/read-only if structured planning itself fails. "strict" fails closed.
@@ -2430,13 +2430,29 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
24302430
frameContexts,
24312431
navigationFrames,
24322432
);
2433-
return visibleCandidates.find(candidate => candidate.visible === true)?.challenge || null;
2433+
const candidate = visibleCandidates.find(entry => entry.visible === true);
2434+
return candidate
2435+
? {
2436+
...candidate.challenge,
2437+
frameId: candidate.frameId,
2438+
frameUrl: candidate.frameUrl || '',
2439+
}
2440+
: null;
24342441
}
24352442

2436-
async _detectChallengeDialogBeforeMutation(tabId) {
2443+
async _detectChallengeDialogBeforeMutation(tabId, options = {}) {
2444+
const includeStatus = options?.includeStatus === true;
2445+
const expectedFrameId = Number.isInteger(options?.expectedFrameId)
2446+
? options.expectedFrameId
2447+
: null;
24372448
let navigationFrames = [];
2449+
let navigationInspectionComplete = false;
24382450
try {
2439-
navigationFrames = await browser.webNavigation?.getAllFrames?.({ tabId }) || [];
2451+
const discoveredFrames = await browser.webNavigation?.getAllFrames?.({ tabId });
2452+
if (Array.isArray(discoveredFrames)) {
2453+
navigationFrames = discoveredFrames;
2454+
navigationInspectionComplete = true;
2455+
}
24402456
} catch {}
24412457
if (!navigationFrames.length) {
24422458
navigationFrames = [{ frameId: 0, parentFrameId: -1, url: '' }];
@@ -2457,18 +2473,31 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
24572473
return null;
24582474
}
24592475
}));
2460-
return this._visibleChallengeDialogFromFrames(
2461-
frameEntries.filter(Boolean),
2476+
const successfulEntries = frameEntries.filter(entry =>
2477+
entry?.payload && typeof entry.payload === 'object'
2478+
);
2479+
const challenge = this._visibleChallengeDialogFromFrames(
2480+
successfulEntries,
24622481
navigationFrames,
24632482
);
2483+
if (!includeStatus) return challenge;
2484+
const inspectedFrameIds = new Set(successfulEntries.map(entry => entry.frameId));
2485+
const expectedFrameStillExists = expectedFrameId !== null
2486+
&& navigationFrames.some(frame => frame?.frameId === expectedFrameId);
2487+
return {
2488+
challenge,
2489+
inspectionComplete: expectedFrameId === null
2490+
|| inspectedFrameIds.has(expectedFrameId)
2491+
|| (navigationInspectionComplete && !expectedFrameStillExists),
2492+
};
24642493
}
24652494

24662495
async _captchaMutationPreflight(tabId, toolName, toolArgs = {}) {
24672496
const gatedCompletion = toolName === 'done' || toolName === 'done_json';
24682497
const gatedAction = this._isBrowserMutationTool(toolName)
24692498
|| gatedCompletion
24702499
|| isNetworkMutation(toolName, toolArgs);
2471-
if (!this.captchaSolverEnabled || !gatedAction || toolName === 'solve_captcha') return null;
2500+
if (!gatedAction || toolName === 'solve_captcha') return null;
24722501
const activeGate = this._captchaGateStates.get(tabId);
24732502
if (activeGate && !this._shouldRetryCaptchaManualGate(activeGate)) return null;
24742503
const challenge = await this._detectChallengeDialogBeforeMutation(tabId);
@@ -2481,6 +2510,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
24812510
{
24822511
pageContent: `dialog ${JSON.stringify(String(challenge.label).slice(0, 200))}`,
24832512
pageUrl,
2513+
captchaChallengeFrameId: Number.isInteger(challenge.frameId)
2514+
? challenge.frameId
2515+
: null,
2516+
captchaChallengeFrameUrl: challenge.frameUrl || '',
24842517
},
24852518
{},
24862519
);
@@ -2530,6 +2563,33 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
25302563
&& toolResult.truncated !== true
25312564
&& toolResult.hasMore !== true
25322565
&& toolResult.autoDegraded !== true;
2566+
if (
2567+
!challenge
2568+
&& activeGate
2569+
&& authoritativeRootRead
2570+
&& Number.isInteger(activeGate.challengeFrameId)
2571+
) {
2572+
const frameInspection = await this._detectChallengeDialogBeforeMutation(tabId, {
2573+
includeStatus: true,
2574+
expectedFrameId: activeGate.challengeFrameId,
2575+
});
2576+
if (frameInspection.challenge?.label) {
2577+
challenge = detectChallengeDialog(
2578+
`dialog ${JSON.stringify(String(frameInspection.challenge.label).slice(0, 200))}`
2579+
);
2580+
} else if (!frameInspection.inspectionComplete) {
2581+
const guardedGate = {
2582+
...activeGate.publicGate,
2583+
verificationFrameReadRequired: true,
2584+
};
2585+
this._captchaGateStates.set(tabId, {
2586+
...activeGate,
2587+
publicGate: guardedGate,
2588+
});
2589+
toolResult.captchaGate = guardedGate;
2590+
return { gate: guardedGate, loopCheck: { kind: 'none' } };
2591+
}
2592+
}
25332593
const loopCheck = challenge || authoritativeRootRead
25342594
? this._checkVerificationChallengeLoop(tabId, {
25352595
pageUrl,
@@ -2681,7 +2741,17 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
26812741
...(detection?.error ? { selectionFailed: true } : {}),
26822742
...(detection?.selected && !selectedCorrelated ? { candidateNotCorrelated: true } : {}),
26832743
};
2684-
this._captchaGateStates.set(tabId, { key, status: publicGate.status, publicGate });
2744+
this._captchaGateStates.set(tabId, {
2745+
key,
2746+
status: publicGate.status,
2747+
publicGate,
2748+
...(Number.isInteger(toolResult.captchaChallengeFrameId)
2749+
? {
2750+
challengeFrameId: toolResult.captchaChallengeFrameId,
2751+
challengeFrameUrl: String(toolResult.captchaChallengeFrameUrl || ''),
2752+
}
2753+
: {}),
2754+
});
26852755
toolResult.captchaGate = publicGate;
26862756
return { gate: publicGate, loopCheck };
26872757
}

0 commit comments

Comments
 (0)