Skip to content

Commit 6756fa8

Browse files
diegosouzapwdiegosouzapw
authored
refactor(executors): extract challenge solver from duckduckgo-web (#6020)
Extract the DuckDuckGo anti-abuse challenge solver + FE signals (CHALLENGE_STUBS, countHtmlElements, buildHtmlLookup, sha256Base64, solveDuckDuckGoChallenge, makeDuckDuckGoFeSignals) verbatim into the leaf duckduckgo-web/challenge.ts. The vm sandbox + 5s timeout (SECURITY note) are preserved. Host imports back the two it uses. Host 924 -> 788 LOC. Byte-identical bodies (verbatim 132/132), leaf does not import the host (no cycle). The now-dead createHash/parse5 host imports are removed; vm stays (still used in host). Auth/cookie/warm/seed/executor untouched. Adds a split-guard; consumer tests stay green (duckduckgo-web-executor 15, duckduckgo-domain-4037 8).
1 parent 2ce64e8 commit 6756fa8

3 files changed

Lines changed: 188 additions & 144 deletions

File tree

‎open-sse/executors/duckduckgo-web.ts‎

Lines changed: 2 additions & 144 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
import { createHash, generateKeyPairSync, randomUUID } from "node:crypto";
1+
import { generateKeyPairSync, randomUUID } from "node:crypto";
22
import vm from "node:vm";
3-
import { parseFragment, serialize } from "parse5";
3+
import { solveDuckDuckGoChallenge, makeDuckDuckGoFeSignals } from "./duckduckgo-web/challenge.ts";
44
import { BaseExecutor, type ExecuteInput } from "./base.ts";
55
import { FETCH_TIMEOUT_MS } from "../config/constants.ts";
66
import { prepareToolMessages, buildToolAwareResult } from "../translator/webTools.ts";
@@ -82,71 +82,6 @@ type DuckDuckGoChallengeResult = {
8282

8383
let durablePublicKey: JsonWebKey | null = null;
8484

85-
const CHALLENGE_STUBS = String.raw`
86-
var __ua = __DDG_REAL_UA__;
87-
var __HTML_LOOKUP = __DDG_HTML_LOOKUP__;
88-
function __makeHtmlElement(tag) {
89-
var state = { _innerHTML: '', _qsaCount: 0, _cssText: '' };
90-
var el = {
91-
tagName: String(tag).toUpperCase(), nodeName: String(tag).toUpperCase(), nodeType: 1,
92-
children: [], childNodes: [], classList: [], dataset: {},
93-
offsetWidth: 1, offsetHeight: 1, clientWidth: 1, clientHeight: 1, scrollHeight: 1, scrollWidth: 1,
94-
getBoundingClientRect: function(){ return { x: 0, y: 0, top: 0, left: 0, right: 1, bottom: 1, width: 1, height: 1, toJSON: function(){ return {}; } }; },
95-
setAttribute: function(){}, removeAttribute: function(){},
96-
getAttribute: function(a){ if(a==='srcdoc') return state._srcdoc||''; return null; },
97-
hasAttribute: function(){ return false; }, appendChild: function(c){ return c; }, removeChild: function(c){ return c; },
98-
addEventListener: function(){}, removeEventListener: function(){}, querySelector: function(){ return null; },
99-
querySelectorAll: function(s){ if (s === '*') { var arr = []; arr.length = state._qsaCount; return arr; } return []; },
100-
cloneNode: function(){ return __makeHtmlElement(tag); }
101-
};
102-
Object.defineProperty(el, 'style', { value: new Proxy({}, { set: function(t, k, v){ t[k] = v; if (k === 'cssText') state._cssText = String(v); return true; }, get: function(t, k){ if (k === 'cssText') return state._cssText; return t[k] || ''; } }), enumerable: true, configurable: true });
103-
Object.defineProperty(el, 'innerHTML', { get: function(){ return state._innerHTML; }, set: function(v){ var key = String(v); var entry = __HTML_LOOKUP && __HTML_LOOKUP[key]; if (entry) { state._innerHTML = String(entry.html); state._qsaCount = entry.count|0; } else { state._innerHTML = key; state._qsaCount = 0; } }, enumerable: true, configurable: true });
104-
Object.defineProperty(el, 'outerHTML', { get: function(){ return '<' + tag + '>' + state._innerHTML + '</' + tag + '>'; }, enumerable: true });
105-
Object.defineProperty(el, 'srcdoc', { get: function(){ return state._srcdoc||''; }, set: function(v){ state._srcdoc = String(v); }, enumerable: true });
106-
Object.defineProperty(el, 'contentWindow', { get: function(){ var w = {}; w.document = __ifDoc; w.Proxy = Proxy; w.self = w; w.top = w; w.parent = w; w.window = w; return w; }, enumerable: true });
107-
Object.defineProperty(el, 'contentDocument', { get: function(){ return __ifDoc; }, enumerable: true });
108-
return el;
109-
}
110-
function __mkObj(name, base) {
111-
base = base || {};
112-
return new Proxy(base, {
113-
get: function(t, k) {
114-
if (k in t) return t[k];
115-
if (k === Symbol.toPrimitive) return function(){ return ''; };
116-
if (k === Symbol.iterator) return undefined;
117-
if (k === 'then' || k === 'catch' || k === 'finally') return undefined;
118-
if (k === 'constructor') return Object;
119-
if (k === 'toString' || k === 'valueOf') return function(){ return '[object ' + name + ']'; };
120-
if (k === 'length') return 0;
121-
if (k === 'nodeType') return 1;
122-
if (k === 'tagName' || k === 'nodeName') return 'DIV';
123-
if (k === 'innerHTML' || k === 'outerHTML' || k === 'textContent' || k === 'innerText' || k === 'value') return '';
124-
if (k === 'children' || k === 'childNodes' || k === 'classList') return [];
125-
// Real numeric layout values for the DDG challenge DOM probes.
126-
if (k === 'offsetWidth' || k === 'offsetHeight' || k === 'clientWidth' || k === 'clientHeight' || k === 'scrollHeight' || k === 'scrollWidth') return 1;
127-
if (k === 'getBoundingClientRect') return function(){ return { x: 0, y: 0, top: 0, left: 0, right: 1, bottom: 1, width: 1, height: 1, toJSON: function(){ return {}; } }; };
128-
if (typeof k === 'string' && (k.indexOf('get') === 0 || k.indexOf('query') === 0 || k.indexOf('find') === 0)) return function(){ return k === 'querySelectorAll' || k === 'getElementsByTagName' || k === 'getElementsByClassName' ? [] : null; };
129-
return function(){ return __mkObj(name + '.' + String(k)); };
130-
},
131-
has: function(t, k){ return k in t; }, set: function(t, k, v){ t[k] = v; return true; }
132-
});
133-
}
134-
function __parseCssDisplay(cssText){ if(!cssText) return ''; var m = String(cssText).match(/(?:^|;)\\s*display\\s*:\\s*([^;]+)/i); return m ? String(m[1]).trim() : ''; }
135-
function __getComputedStyle(el){ var cssText = el && el.style && el.style.cssText || ''; var display = __parseCssDisplay(cssText); return { getPropertyValue: function(name){ if(String(name).toLowerCase()==='display') return display; return ''; }, cssText: cssText, display: display }; }
136-
var __ifMeta = __mkObj('meta', { getAttribute: function(a){ return a==='content' ? "default-src 'none'; script-src 'unsafe-inline';" : null; }, hasAttribute: function(a){ return a==='content'; }, tagName: 'META', nodeName: 'META' });
137-
var __ifDoc = __mkObj('iframeDoc', { querySelector: function(s){ if (s && s.indexOf('Content-Security-Policy') !== -1) return __ifMeta; if (s === 'meta') return __ifMeta; return null; }, querySelectorAll: function(s){ if (s && s.indexOf('Content-Security-Policy') !== -1) return [__ifMeta]; if (s === 'meta') return [__ifMeta]; return []; }, getElementsByTagName: function(t){ return t && t.toLowerCase()==='meta' ? [__ifMeta] : []; }, body: __mkObj('iframeBody'), head: __mkObj('iframeHead'), documentElement: __mkObj('iframeRoot'), createElement: function(){ return __mkObj('elem', {setAttribute:function(){}, appendChild:function(){}, removeChild:function(){}, getAttribute:function(){return null;}, hasAttribute:function(){return false;}}); }, cookie: '', readyState: 'complete' });
138-
var __iframeEl = __mkObj('iframe', { contentDocument: __ifDoc, contentWindow: __mkObj('iframeWin', { document: __ifDoc, top: undefined, parent: undefined }), document: __ifDoc, getAttribute: function(a){ if (a==='sandbox') return 'allow-scripts allow-same-origin'; if (a==='srcdoc') return ''; if (a==='id') return 'jsa'; return null; }, hasAttribute: function(a){ return a==='sandbox'||a==='id'; }, tagName: 'IFRAME', nodeName: 'IFRAME', id: 'jsa' });
139-
var document = __mkObj('document', { querySelector: function(s){ if (s === '#jsa') return __iframeEl; if (s && s.indexOf('Content-Security-Policy') !== -1) return __ifMeta; return null; }, querySelectorAll: function(s){ if (s === '#jsa') return [__iframeEl]; if (s && s.indexOf('Content-Security-Policy') !== -1) return [__ifMeta]; return []; }, getElementById: function(id){ return id==='jsa' ? __iframeEl : null; }, getElementsByTagName: function(t){ if(t&&t.toLowerCase()==='iframe') return [__iframeEl]; return []; }, getElementsByClassName: function(){ return []; }, body: __mkObj('body', {appendChild:function(){}, removeChild:function(){}, querySelector:function(s){return s==='#jsa'?__iframeEl:null;}, querySelectorAll:function(s){return s==='#jsa'?[__iframeEl]:[];}}), head: __mkObj('head'), documentElement: __mkObj('root'), createElement: function(tag){ return __makeHtmlElement(tag||'div'); }, createTextNode: function(t){ return {nodeType:3, nodeValue:String(t||''), textContent:String(t||'')}; }, cookie: '', readyState: 'complete', title: '', addEventListener: function(){}, removeEventListener: function(){} });
140-
var window = __mkObj('window', { document: document, __DDG_BE_VERSION__: 1, __DDG_FE_CHAT_HASH__: 1, navigator: __mkObj('navigator', { userAgent: __ua, webdriver: false, language: 'en-US', languages: ['en-US','en'], platform: 'Linux x86_64', vendor: 'Google Inc.', appVersion: '5.0 (X11)', cookieEnabled: true, onLine: true, hardwareConcurrency: 8, deviceMemory: 8 }), innerWidth: 1280, innerHeight: 800, outerWidth: 1280, outerHeight: 800, devicePixelRatio: 1, screen: __mkObj('screen', { width:1920, height:1080, availWidth:1920, availHeight:1080, colorDepth:24, pixelDepth:24 }), location: __mkObj('location', { href:'https://duck.ai/', origin:'https://duck.ai', host:'duck.ai', hostname:'duck.ai', protocol:'https:', pathname:'/' }), performance: __mkObj('perf', { now: function(){ return 0; }, timeOrigin: 0 }), history: __mkObj('history', { length: 1, state: null }), addEventListener: function(){}, removeEventListener: function(){}, dispatchEvent: function(){return true;}, setTimeout: function(fn){ try{fn();}catch(e){} return 0; }, clearTimeout: function(){}, hasOwnProperty: function(k){ if (k==='__DDG_BE_VERSION__'||k==='__DDG_FE_CHAT_HASH__') return true; return Object.prototype.hasOwnProperty.call(this,k); } });
141-
window.top = window; window.self = window; window.window = window; window.parent = window; window.globalThis = window;
142-
var top = window, self = window, parent = window, navigator = window.navigator, location = window.location, screen = window.screen, performance = window.performance, history = window.history;
143-
var __R = null, __E = null;
144-
function __HTMLClass(name){ var c = function(){}; c.prototype = __mkObj(name+'.proto'); return c; }
145-
var HTMLElement = __HTMLClass('HTMLElement'), HTMLDivElement = __HTMLClass('HTMLDivElement'), HTMLIFrameElement = __HTMLClass('HTMLIFrameElement'), HTMLDocument = __HTMLClass('HTMLDocument'), Document = __HTMLClass('Document'), Element = __HTMLClass('Element'), Node = __HTMLClass('Node'), Window = __HTMLClass('Window'), Event = __HTMLClass('Event'), MouseEvent = __HTMLClass('MouseEvent'), KeyboardEvent = __HTMLClass('KeyboardEvent'), TouchEvent = __HTMLClass('TouchEvent'), XMLHttpRequest = __HTMLClass('XMLHttpRequest'), WebSocket = __HTMLClass('WebSocket'), Image = __HTMLClass('Image'), FormData = __HTMLClass('FormData'), Blob = __HTMLClass('Blob'), File = __HTMLClass('File'), FileReader = __HTMLClass('FileReader'), URL = __HTMLClass('URL'), URLSearchParams = __HTMLClass('URLSearchParams'), Headers = __HTMLClass('Headers'), Request = __HTMLClass('Request'), Response = __HTMLClass('Response');
146-
var fetch = function(){ return Promise.resolve(__mkObj('resp', {ok:true, status:200, json:function(){return Promise.resolve({});}, text:function(){return Promise.resolve('');}})); };
147-
var getComputedStyle = __getComputedStyle;
148-
`;
149-
15085
function extractDuckDuckGoContent(data: unknown): string {
15186
if (!data || typeof data !== "object") return "";
15287
const record = data as Record<string, unknown>;
@@ -247,83 +182,6 @@ function getDuckDuckGoModelCapabilities(model: string): DuckDuckGoModelCapabilit
247182
return { reasoningEffort: null };
248183
}
249184

250-
function countHtmlElements(node: unknown): number {
251-
if (!node || typeof node !== "object") return 0;
252-
const record = node as { nodeName?: string; childNodes?: unknown[] };
253-
const own = record.nodeName && record.nodeName !== "#document-fragment" ? 1 : 0;
254-
let childCount = 0;
255-
for (const child of record.childNodes ?? []) {
256-
childCount += countHtmlElements(child);
257-
}
258-
return own + childCount;
259-
}
260-
261-
function buildHtmlLookup(js: string): Record<string, { html: string; count: number }> {
262-
const lookup: Record<string, { html: string; count: number }> = {};
263-
const seen = new Set<string>();
264-
const pattern = /(['"])(<[^'"]{1,400}?)\1/g;
265-
for (const match of js.matchAll(pattern)) {
266-
const html = match[2];
267-
if (seen.has(html)) continue;
268-
seen.add(html);
269-
const fragment = parseFragment(html);
270-
lookup[html] = {
271-
html: serialize(fragment),
272-
count: Math.max(0, countHtmlElements(fragment) - 1),
273-
};
274-
}
275-
return lookup;
276-
}
277-
278-
function sha256Base64(value: string): string {
279-
return createHash("sha256").update(value, "utf8").digest("base64");
280-
}
281-
282-
async function solveDuckDuckGoChallenge(challenge: string, userAgent: string): Promise<string> {
283-
// SECURITY NOTE: This function executes base64-decoded JavaScript from duck.ai via vm.runInContext.
284-
// The challenge code is upstream-supplied (supply-chain surface). It is sandboxed with a 5s timeout
285-
// to limit DoS risk. This is intentional for the DDG challenge solver to work.
286-
const js = Buffer.from(challenge, "base64").toString("utf8");
287-
const stubs = CHALLENGE_STUBS.replace("__DDG_REAL_UA__", JSON.stringify(userAgent)).replace(
288-
"__DDG_HTML_LOOKUP__",
289-
JSON.stringify(buildHtmlLookup(js))
290-
);
291-
const context = vm.createContext({});
292-
vm.runInContext(stubs, context, { timeout: 5000 });
293-
const result = (await vm.runInContext(js, context, {
294-
timeout: 5000,
295-
})) as DuckDuckGoChallengeResult;
296-
const clientHashes = Array.isArray(result.client_hashes) ? result.client_hashes : [];
297-
if (clientHashes.length === 0)
298-
throw new Error("DuckDuckGo challenge returned empty client_hashes");
299-
clientHashes[0] = userAgent;
300-
result.client_hashes = clientHashes.map((hash) => sha256Base64(String(hash)));
301-
return Buffer.from(JSON.stringify(result), "utf8").toString("base64");
302-
}
303-
304-
function makeDuckDuckGoFeSignals(): string {
305-
const start = Date.now() - 3000;
306-
let delta = 80 + Math.floor(Math.random() * 101);
307-
const events: Array<Record<string, unknown>> = [{ name: "onboarding_impression_1", delta }];
308-
delta += 120 + Math.floor(Math.random() * 141);
309-
events.push({ name: "onboarding_impression_2", delta });
310-
delta += 200 + Math.floor(Math.random() * 301);
311-
events.push({ name: "startNewChat", delta });
312-
const keyEvents = 6 + Math.floor(Math.random() * 13);
313-
for (let i = 0; i < keyEvents; i++) {
314-
delta += 40 + Math.floor(Math.random() * 141);
315-
events.push({ name: "user_input", delta });
316-
}
317-
delta += 120 + Math.floor(Math.random() * 231);
318-
events.push({ name: "user_submit", delta });
319-
const payload = {
320-
start,
321-
events,
322-
end: Math.max(delta + 20 + Math.floor(Math.random() * 71), 3000),
323-
};
324-
return Buffer.from(JSON.stringify(payload), "utf8").toString("base64");
325-
}
326-
327185
function extractDuckDuckGoFeVersion(html: string): string | null {
328186
return html.match(FE_VERSION_PATTERN)?.[0] ?? null;
329187
}

0 commit comments

Comments
 (0)