diff --git a/package.json b/package.json index d29f54fc7..14b6ba9ea 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "iso-639-1": "^3.1.5", "js-levenshtein": "^1.1.6", "js-yaml": "^4.1.0", + "jsdom": "^29.0.0", "normalize-url": "^9.0.0", "p-queue": "^7.3.4", "pixelmatch": "^5.3.0", @@ -50,6 +51,7 @@ "@types/get-folder-size": "^3.0.4", "@types/js-levenshtein": "^1.1.3", "@types/js-yaml": "^4.0.8", + "@types/jsdom": "^28.0.0", "@types/node": "^20.8.7", "@types/pixelmatch": "^5.2.6", "@types/pngjs": "^6.0.4", diff --git a/src/crawler.ts b/src/crawler.ts index 8ca4f13ea..24847c696 100644 --- a/src/crawler.ts +++ b/src/crawler.ts @@ -19,7 +19,10 @@ import yaml from "js-yaml"; import { WACZ, WACZInitOpts, mergeCDXJ } from "./util/wacz.js"; import { HealthChecker } from "./util/healthcheck.js"; -import { TextExtractViaSnapshot } from "./util/textextract.js"; +import { + TextExtractViaResponse, + TextExtractViaSnapshot, +} from "./util/textextract.js"; import { initStorage, getFileSize, @@ -1178,11 +1181,55 @@ self.__bx_behaviors.selectMainBehavior(); recorder.pageSeedDepth = depth; } + // Enable Network domain to track document request for raw text extraction + let documentRequestId: string | null = null; + let trackDocumentRequest: + | ((params: Protocol.Network.ResponseReceivedEvent) => void) + | null = null; + + if (this.params.text.includes("to-warc-from-raw")) { + await cdp.send("Network.enable"); + trackDocumentRequest = ( + params: Protocol.Network.ResponseReceivedEvent, + ) => { + if (params.type === "Document" && !documentRequestId) { + documentRequestId = params.requestId; + logger.debug( + "Captured document request ID", + { requestId: documentRequestId, url: params.response.url }, + "text", + ); + } + }; + cdp.on("Network.responseReceived", trackDocumentRequest); + } + // run custom driver here, if any - if (this.driver) { - await this.driver({ page, data, crawler: this, seed }); - } else { - await this.loadPage(page, data, seed); + try { + if (this.driver) { + await this.driver({ page, data, crawler: this, seed }); + } else { + await this.loadPage(page, data, seed); + } + } finally { + // Store document request ID and clean up listener + if (documentRequestId) { + data.documentRequestId = documentRequestId; + logger.info( + "Stored document request ID for raw text extraction", + { url: data.url, requestId: documentRequestId }, + "text", + ); + } else if (trackDocumentRequest) { + logger.warn( + "No document request ID captured", + { url: data.url }, + "text", + ); + } + if (trackDocumentRequest) { + cdp.off("Network.responseReceived", trackDocumentRequest); + } } data.title = await timedRun( @@ -1235,23 +1282,61 @@ self.__bx_behaviors.selectMainBehavior(); } } - let textextract = null; + let textExtract = null; if (this.textWriter) { - textextract = new TextExtractViaSnapshot(cdp, { + textExtract = new TextExtractViaSnapshot(cdp, { writer: this.textWriter, url, skipDocs: this.skipTextDocs, }); - const { text } = await textextract.extractAndStoreText( + const { text } = await textExtract.extractAndStoreText( "text", false, this.params.text.includes("to-warc"), + data.originalWarcRecordId, ); if (text !== null && (this.textInPages || saveOutput)) { data.text = text; } + + if (this.params.text.includes("to-warc-from-raw")) { + logger.info( + "Extracting text from raw response", + { + url, + hasRequestId: !!data.documentRequestId, + refersTo: String(data.originalWarcRecordId), + }, + "text", + ); + textExtract = new TextExtractViaResponse(cdp, { + writer: this.textWriter, + url, + skipDocs: this.skipTextDocs, + requestId: data.documentRequestId, + }); + const { text: textFromResponse } = + await textExtract.extractAndStoreText( + "text-from-response", + false, + this.params.text.includes("to-warc-from-raw"), + data.originalWarcRecordId, + ); + logger.info( + "Extracted text from raw response result", + { + url, + hasText: textFromResponse !== null, + refersTo: String(data.originalWarcRecordId), + }, + "text", + ); + if (textFromResponse !== null && (this.textInPages || saveOutput)) { + data.textFromResponse = textFromResponse; + } + } } data.loadState = LoadState.EXTRACTION_DONE; @@ -1280,8 +1365,13 @@ self.__bx_behaviors.selectMainBehavior(); data.loadState = LoadState.BEHAVIORS_DONE; } - if (textextract && this.params.text.includes("final-to-warc")) { - await textextract.extractAndStoreText("textFinal", true, true); + if (textExtract && this.params.text.includes("final-to-warc")) { + await textExtract.extractAndStoreText( + "textFinal", + true, + true, + data.originalWarcRecordId, + ); } if ( @@ -2640,6 +2730,7 @@ self.__bx_behaviors.selectMainBehavior(); logDetails: LogDetails = {}, ts = 0, pageid?: string, + originalWarcRecordId?: string, ) { if (this.limitHit) { return false; @@ -2658,7 +2749,7 @@ self.__bx_behaviors.selectMainBehavior(); } const result = await this.crawlState.addToQueue( - { url, seedId, depth, extraHops, ts, pageid }, + { url, seedId, depth, extraHops, ts, pageid, originalWarcRecordId }, this.pageLimit, ); diff --git a/src/replaycrawler.ts b/src/replaycrawler.ts index 091c2e4c4..763ac2ec6 100644 --- a/src/replaycrawler.ts +++ b/src/replaycrawler.ts @@ -21,6 +21,8 @@ import { WARCWriter } from "./util/warcwriter.js"; import { parseRx } from "./util/seeds.js"; import { getFileOrUrlJson } from "./util/file_reader.js"; import { WACZLoader } from "./util/wacz.js"; +import { CSR_CLUES } from "./util/clientSideRenderingClues.js"; +import { extractTextFromDOM } from "./util/textextract.js"; // RWP Replay Prefix const REPLAY_PREFIX = "http://localhost:9990/replay/w/replay/"; @@ -43,12 +45,17 @@ type ComparisonData = { comparison: { screenshotMatch?: number; textMatch?: number; + rawTextMatch?: number; resourceCounts: { crawlGood?: number; crawlBad?: number; replayGood?: number; replayBad?: number; }; + csrClues?: { + clues: Record; + categories: Record; + }; }; }; @@ -65,6 +72,7 @@ export class ReplayCrawler extends Crawler { pageInfos: Map; infoWriter: WARCWriter | null; + htmlWriter: WARCWriter | null; reloadTimeouts: WeakMap; @@ -95,13 +103,14 @@ export class ReplayCrawler extends Crawler { this.skipTextDocs = SKIP_FRAMES; this.params.screenshot = ["view"]; - this.params.text = ["to-warc"]; + this.params.text = ["to-warc", "to-warc-from-raw"]; this.params.serviceWorker = "enabled"; this.reloadTimeouts = new WeakMap(); this.infoWriter = null; + this.htmlWriter = null; this.includeRx = parseRx(this.params.scopeIncludeRx); this.excludeRx = parseRx(this.params.scopeExcludeRx); @@ -111,6 +120,7 @@ export class ReplayCrawler extends Crawler { await super.bootstrap(); this.infoWriter = this.createExtraResourceWarcWriter("info"); + this.htmlWriter = this.createExtraResourceWarcWriter("html"); } protected parseArgs() { @@ -226,6 +236,7 @@ export class ReplayCrawler extends Crawler { } async _addPageIfInScope({ url, ts, id, mime }: ReplayPage, depth: number) { + logger.debug("Adding page to queue", { url, id, hasId: !!id }, "replay"); if (mime && mime !== "text/html") { logger.info("Skipping non-HTML page", { url, mime }, "replay"); return; @@ -252,7 +263,7 @@ export class ReplayCrawler extends Crawler { } } - await this.queueUrl(0, url, depth, 0, {}, ts, id); + await this.queueUrl(0, url, depth, 0, {}, ts, id, id); } async loadPagesDirect(pages: ReplayPage[]) { @@ -384,7 +395,7 @@ export class ReplayCrawler extends Crawler { async crawlPage(opts: WorkerState): Promise { await this.writeStats(); - const { page, data, workerid } = opts; + const { page, data, workerid, cdp } = opts; const { url, ts, pageid } = data; if (!ts) { @@ -409,7 +420,7 @@ export class ReplayCrawler extends Crawler { comparison: { resourceCounts: {} }, counts: { jsErrors: 0 }, tsStatus: 999, - }; + } satisfies ReplayPageInfoRecord; this.pageInfos.set(page, pageInfo); let replayFrame; @@ -422,22 +433,121 @@ export class ReplayCrawler extends Crawler { replayFrame = page.frames()[SKIP_FRAMES]; } + // Capture document request ID for raw text extraction if enabled + let documentRequestId: string | null = null; + let trackDocumentRequest: + | ((params: Protocol.Network.ResponseReceivedEvent) => void) + | null = null; + + if (this.params.text.includes("to-warc-from-raw")) { + trackDocumentRequest = ( + params: Protocol.Network.ResponseReceivedEvent, + ) => { + if (params.type === "Document" && !documentRequestId) { + documentRequestId = params.requestId; + logger.debug( + "Captured document request ID", + { requestId: documentRequestId, url: params.response.url }, + "text", + ); + } + }; + cdp.on("Network.responseReceived", trackDocumentRequest); + } + + let navigationSuccess = false; try { - await replayFrame.goto( + const response = await replayFrame.goto( `${REPLAY_PREFIX}${timestamp}mp_/${url}`, this.gotoOpts, ); + navigationSuccess = true; + if (response) { + const status = response.status(); + logger.info( + "Replay navigation completed", + { url, status, replayUrl: `${REPLAY_PREFIX}${timestamp}mp_/${url}` }, + "replay", + ); + if (status >= 400) { + logger.warn( + "Replay returned error status", + { url, status }, + "replay", + ); + } + } else { + logger.warn( + "Replay navigation returned null response", + { url }, + "replay", + ); + } } catch (e) { logger.warn( - "Loading replay timed out", + "Loading replay timed out or failed", { ...logDetails, ...formatErr(e) }, "replay", ); + } finally { + // Store document request ID and clean up listener + if (documentRequestId) { + data.documentRequestId = documentRequestId; + logger.info( + "Stored document request ID for raw text extraction", + { + url: data.url, + requestId: documentRequestId, + originalWarcRecordId: String(data.originalWarcRecordId), + }, + "text", + ); + } else if (trackDocumentRequest) { + logger.warn( + "No document request ID captured", + { + url: data.url, + originalWarcRecordId: String(data.originalWarcRecordId), + }, + "text", + ); + } + if (trackDocumentRequest) { + cdp.off("Network.responseReceived", trackDocumentRequest); + } } - while (replayFrame.url().indexOf("about:blank") >= 0) { - logger.debug("Waiting for replay frame to update"); - await sleep(2); + // Only wait for frame to update if navigation succeeded + if (navigationSuccess) { + let frameWaitIterations = 0; + const maxFrameWaitIterations = 150; // 150 * 2 seconds = 5 min timeout + while (replayFrame.url().indexOf("about:blank") >= 0) { + if (frameWaitIterations >= maxFrameWaitIterations) { + logger.warn( + "Replay frame failed to update from about:blank", + { + url, + frameUrl: replayFrame.url(), + iterations: frameWaitIterations, + }, + "replay", + ); + break; + } + logger.debug( + "Waiting for replay frame to update", + { + url, + frameUrl: replayFrame.url(), + iterations: frameWaitIterations, + }, + "replay", + ); + await sleep(2); + frameWaitIterations++; + } + } else { + logger.warn("Skipping frame wait - navigation failed", { url }, "replay"); } // optionally reload (todo: reevaluate if this is needed) @@ -467,6 +577,15 @@ export class ReplayCrawler extends Crawler { await this.compareResources(page, data, url, date); + if (data.doRawTextCompare) { + await this.compareRawText(page, data, url, date); + + await this.writeHtmlToWarc(page, url, date, data.originalWarcRecordId); + } + + if (this.params.qaDetectClientSideRendering) + await this.processCSRClues(page, data); + await this.processPageInfo(page, data); } @@ -561,6 +680,7 @@ export class ReplayCrawler extends Crawler { url, date.toISOString().replace(/[^\d]/g, ""), ); + state.originalText = origText; const replayText = state.text; if (origText === undefined || replayText === undefined) { @@ -591,6 +711,182 @@ export class ReplayCrawler extends Crawler { } } + async compareRawText(page: Page, state: PageState, url: string, date: Date) { + // In the future, we could use the existing `text-from-response` data in the + // original archived data if it exists, but for now we're running on data we + // know doesn't include it + logger.info( + "Extracting raw text from archived response", + { url }, + "replay", + ); + const rawText = await this.extractRawTextFromArchive(page, url, date); + + // Write raw text to WARC if textWriter is available + if (rawText && this.textWriter) { + this.textWriter.writeNewResourceRecord( + { + buffer: new TextEncoder().encode(rawText), + resourceType: "text-from-response", + contentType: "text/plain", + url: url, + date: date, + refersTo: state.originalWarcRecordId, + }, + { type: "text-from-response", url: url }, + "replay", + ); + logger.info( + "Wrote raw text to WARC", + { url, length: rawText.length, refersTo: state.originalWarcRecordId }, + "replay", + ); + } + + // Get the crawl-time rendered text (extracted from browser after JS runs) + const origText = state.originalText; + + if (rawText === undefined || origText === undefined) { + logger.warn( + "Text missing for raw vs rendered comparison", + { + url, + rawTextLen: rawText?.length, + renderedTextLen: origText?.length, + }, + "replay", + ); + return; + } + + const dist = levenshtein(rawText, origText); + const maxLen = Math.max(rawText.length, origText.length); + + let matchPercent = 1.0; + if (maxLen > 0) { + matchPercent = (maxLen - dist) / maxLen; + } + logger.info("Raw vs Rendered Text Levenshtein Dist", { + url, + dist, + matchPercent, + maxLen, + rawTextLen: rawText.length, + renderedTextLen: origText.length, + }); + + const pageInfo = this.pageInfos.get(page); + if (pageInfo) { + pageInfo.comparison.rawTextMatch = matchPercent; + } + } + + async extractRawTextFromArchive( + page: Page, + url: string, + date: Date, + ): Promise { + const timestamp = date.toISOString().slice(0, 19).replace(/[T:-]/g, ""); + // `id_` suffix ensures wabac.js serves the unaltered source + const replayUrl = REPLAY_PREFIX + `${timestamp}id_/${url}`; + + const frame = page.frames()[1]; + if (!frame) { + logger.warn( + "Replay frame missing for raw text extraction", + { url }, + "replay", + ); + return undefined; + } + + const htmlContent = await frame.evaluate(async (url) => { + const response = await fetch(url, { + method: "GET", + credentials: "include", + }); + if (response.status !== 200) { + return undefined; + } + return await response.text(); + }, replayUrl); + + if (!htmlContent) { + logger.warn( + "Could not fetch HTML content for raw text extraction", + { url }, + "replay", + ); + return undefined; + } + + // Extract text from the raw HTML using the shared extraction logic + const { JSDOM } = await import("jsdom"); + const dom = new JSDOM(htmlContent); + try { + return extractTextFromDOM(dom.window.document); + } finally { + dom.window.close(); + } + } + + async writeHtmlToWarc( + page: Page, + url: string, + date: Date, + originalWarcRecordId?: string, + ) { + const timestamp = date.toISOString().slice(0, 19).replace(/[T:-]/g, ""); + // `id_` suffix ensures wabac.js serves the unaltered source + const replayUrl = REPLAY_PREFIX + `${timestamp}id_/${url}`; + + const frame = page.frames()[1]; + if (!frame) { + logger.warn( + "Replay frame missing for HTML extraction", + { url }, + "replay", + ); + return; + } + + const htmlContent = await frame.evaluate(async (url) => { + const response = await fetch(url, { + method: "GET", + credentials: "include", + }); + if (response.status !== 200) { + return undefined; + } + return await response.text(); + }, replayUrl); + + if (!htmlContent) { + logger.warn("Could not fetch HTML content for WARC", { url }, "replay"); + return; + } + + if (this.htmlWriter) { + this.htmlWriter.writeNewResourceRecord( + { + buffer: new TextEncoder().encode(htmlContent), + resourceType: "html-response", + contentType: "text/html", + url: url, + date: date, + refersTo: originalWarcRecordId, + }, + { type: "html-response", url: url }, + "replay", + ); + logger.info( + "Wrote HTML to WARC", + { url, length: htmlContent.length, refersTo: originalWarcRecordId }, + "replay", + ); + } + } + async compareResources( page: Page, state: PageState, @@ -628,6 +924,14 @@ export class ReplayCrawler extends Crawler { pageInfo.ts = origResData.ts; } + const status = origResData.urls[origResData.url]?.status; + + logger.info("Page Resource Status", { status }); + + if (origResData.url && status === 200) { + state.doRawTextCompare = true; + } + const { resourceCounts } = pageInfo.comparison; const { good: crawlGood, bad: crawlBad } = this.countResources(origResData); @@ -739,6 +1043,9 @@ export class ReplayCrawler extends Crawler { if (this.infoWriter) { await this.infoWriter.flush(); } + if (this.htmlWriter) { + await this.htmlWriter.flush(); + } } async processPageInfo(page: Page, state?: PageState) { @@ -755,7 +1062,7 @@ export class ReplayCrawler extends Crawler { if (state) { const { comparison } = pageInfo; - // add comparison to page state + // add comparison & csr clues to page state (state as ComparisonPageState).comparison = comparison; } @@ -765,20 +1072,132 @@ export class ReplayCrawler extends Crawler { resourceType: "pageinfo", contentType: "application/json", url: pageInfo.url, + refersTo: state?.originalWarcRecordId, }, { type: "pageinfo", url: pageInfo.url }, "replay", ); + logger.debug( + "Wrote pageinfo to WARC", + { url: pageInfo.url, refersTo: state?.originalWarcRecordId }, + "replay", + ); this.pageInfos.delete(page); } } + async processCSRClues(page: Page, state?: PageState) { + const pageInfo = this.pageInfos.get(page); + if (!pageInfo) { + logger.warn("No pageInfo found for CSR clues processing", {}, "replay"); + return; + } + + if (!state?.isHTMLPage) { + logger.info( + "Skipping CSR clues - not an HTML page", + { url: pageInfo.url }, + "replay", + ); + return; + } + + if (!pageInfo.ts) { + logger.warn("Missing page timestamp, skipping", { pageInfo }); + return; + } + + const timestamp = new Date(pageInfo.ts) + .toISOString() + .slice(0, 19) + .replace(/[T:-]/g, ""); + // `id_` suffix to timestamp ensures wabac.js serves the unaltered source + const replayUrl = REPLAY_PREFIX + `${timestamp}id_/${pageInfo.url}`; + + const frame = page.frames()[1]; + if (!frame) { + logger.warn( + "Replay frame missing for CSR clues", + { url: pageInfo.url }, + "replay", + ); + return; + } + + const htmlContent = await frame.evaluate(async (url) => { + const response = await fetch(url, { + method: "GET", + credentials: "include", + }); + if (response.status !== 200) { + console.log("Got non-200 status code, proceeding", { url }); + } + return await response.text(); + }, replayUrl); + + if (!htmlContent) { + logger.warn( + "Could not fetch HTML content for CSR clues", + { url: pageInfo.url }, + "replay", + ); + return; + } + + const clueCounts: Record = {}; + const categoryCounts: Record = {}; + + for (const clue of CSR_CLUES) { + const regex = new RegExp(clue.pattern, "gi"); + const matches = htmlContent.match(regex); + + if (matches && matches.length > 0) { + // count by clue name + clueCounts[clue.name] = matches.length; + + // count by category (sum of all patterns in category) + categoryCounts[clue.category] = + (categoryCounts[clue.category] || 0) + matches.length; + + logger.debug( + "CSR clue detected", + { + pattern: clue.name, + category: clue.category, + url: pageInfo.url, + count: matches.length, + }, + "replay", + ); + } + } + + // add clue counts to pageInfo + pageInfo.comparison["csrClues"] = { + clues: clueCounts, + categories: categoryCounts, + }; + if (Object.keys(clueCounts).length > 0) { + logger.info( + "CSR clues detected", + { + url: pageInfo.url, + categories: categoryCounts, + clues: clueCounts, + }, + "replay", + ); + } else { + logger.info("No CSR clues detected", { url: pageInfo.url }, "replay"); + } + } + protected pageEntryForRedis( entry: Record, - state: PageState, + state: ComparisonPageState, ) { - entry.comparison = (state as ComparisonPageState).comparison; + entry.comparison = state.comparison; return entry; } diff --git a/src/util/argParser.ts b/src/util/argParser.ts index f1f41b082..7afe345d5 100644 --- a/src/util/argParser.ts +++ b/src/util/argParser.ts @@ -19,6 +19,7 @@ import { DEFAULT_MAX_RETRIES, BxFunctionBindings, DEFAULT_CRAWL_ID_TEMPLATE, + type ExtractTextType, } from "./constants.js"; import { interpolateFilename } from "./storage.js"; import { screenshotTypes } from "./screenshots.js"; @@ -38,7 +39,7 @@ export type CrawlerArgs = ReturnType & { logLevel: LogLevel[]; logContext: LogContext[]; logExcludeContext: LogContext[]; - text: string[]; + text: ExtractTextType[]; customBehaviors: string[]; @@ -704,6 +705,12 @@ class ArgParser { type: "boolean", }, + qaDetectClientSideRendering: { + describe: + "use heuristics to detect when a page might use client-side rendering (CSR)", + type: "boolean", + }, + sshProxyPrivateKeyFile: { describe: "path to SSH private key for SOCKS5 over SSH proxy connection", diff --git a/src/util/clientSideRenderingClues.ts b/src/util/clientSideRenderingClues.ts new file mode 100644 index 000000000..5c0efbadd --- /dev/null +++ b/src/util/clientSideRenderingClues.ts @@ -0,0 +1,213 @@ +export type CSRClue = { + name: string; + category: string; + pattern: string; + description: string; +}; + +/** + * Various clues to identify client-side rendering (CSR) in source HTML. + */ +export const CSR_CLUES: CSRClue[] = [ + // Empty container divs + { + name: "empty_div", + category: "empty_container", + pattern: + "\\s*", + description: "Empty container divs typical of SPA/CSR", + }, + { + name: "empty_div_self_closing", + category: "empty_container", + pattern: + "", + description: "Empty container divs typical of SPA/CSR (self-closing)", + }, + // Framework meta tags + { + name: "nextjs_generator", + category: "framework_meta", + pattern: "]*src=[\"'][^\"']*(?:webpack|vite|parcel|rollup|esbuild)[^\"']*\\.js[\"']", + description: "Bundler script", + }, + { + name: "bundled_script", + category: "bundler_scripts", + pattern: + "]*src=[\"'][^\"']*(?:bundle|chunk|vendor|runtime|main)[^\"']*\\.[a-f0-9]{8,}\\.js[\"']", + description: "Bundled script with hash", + }, + { + name: "webpack_data_attr", + category: "bundler_scripts", + pattern: "]*\\sdata-webpack=[\"'][^\"']*[\"']", + description: "Webpack data attribute", + }, + { + name: "vite_data_attr", + category: "bundler_scripts", + pattern: "]*\\sdata-vite-fid=[\"'][^\"']*[\"']", + description: "Vite data attribute", + }, + // Hydration data + { + name: "nextjs_hydration", + category: "hydration_data", + pattern: "]*id=[\"']__NEXT_DATA__[\"']", + description: "Next.js hydration data", + }, + { + name: "nuxt_hydration", + category: "hydration_data", + pattern: "]*id=[\"']__NUXT(?:_DATA)?__[\"']", + description: "Nuxt hydration data", + }, + { + name: "vue_hydration", + category: "hydration_data", + pattern: "]*id=[\"']__VUE__[\"']", + description: "Vue hydration data", + }, + { + name: "angular_hydration", + category: "hydration_data", + pattern: "]*id=[\"']ng-state[\"']", + description: "Angular hydration data", + }, + { + name: "svelte_hydration", + category: "hydration_data", + pattern: "data-sveltekit-hydrate", + description: "Svelte hydration data", + }, + // Noscript warnings + { + name: "noscript", + category: "noscript_warning", + pattern: "