From 581a18db0075d5fa2cc29cd924d7f47d9a63aae5 Mon Sep 17 00:00:00 2001 From: emma Date: Wed, 11 Mar 2026 14:36:49 -0400 Subject: [PATCH 01/22] implement client-side rendering detection in qa adds option to analyze replayed html content to detect csr clues such as framework indicators, hydration data, web components, and dom manipulation patterns to identify client-side rendered pages will follow with text analysis using & not using additional page resources --- src/replaycrawler.ts | 115 +++++++++++++++++- src/util/argParser.ts | 6 + src/util/clientSideRenderingClues.ts | 171 +++++++++++++++++++++++++++ 3 files changed, 291 insertions(+), 1 deletion(-) create mode 100644 src/util/clientSideRenderingClues.ts diff --git a/src/replaycrawler.ts b/src/replaycrawler.ts index 091c2e4c4..f6f3d41f5 100644 --- a/src/replaycrawler.ts +++ b/src/replaycrawler.ts @@ -21,6 +21,7 @@ import { WARCWriter } from "./util/warcwriter.js"; import { parseRx } from "./util/seeds.js"; import { getFileOrUrlJson } from "./util/file_reader.js"; import { WACZLoader } from "./util/wacz.js"; +import { CSR_CLUES } from "./util/clientSideRenderingClues.js"; // RWP Replay Prefix const REPLAY_PREFIX = "http://localhost:9990/replay/w/replay/"; @@ -50,6 +51,10 @@ type ComparisonData = { replayBad?: number; }; }; + csrClues?: { + clues: Record; + categories: Record; + }; }; type ReplayPageInfoRecord = PageInfoRecord & ComparisonData; @@ -409,7 +414,7 @@ export class ReplayCrawler extends Crawler { comparison: { resourceCounts: {} }, counts: { jsErrors: 0 }, tsStatus: 999, - }; + } satisfies ReplayPageInfoRecord; this.pageInfos.set(page, pageInfo); let replayFrame; @@ -467,6 +472,9 @@ export class ReplayCrawler extends Crawler { await this.compareResources(page, data, url, date); + if (this.params.qaDetectClientSideRendering) + await this.processCSRClues(page, data); + await this.processPageInfo(page, data); } @@ -774,6 +782,111 @@ export class ReplayCrawler extends Crawler { } } + async processCSRClues(page: Page, state?: PageState) { + const pageInfo = this.pageInfos.get(page); + if (!pageInfo) { + logger.warn("No pageInfo found for CSR clues processing", {}, "replay"); + return; + } + + if (!state?.isHTMLPage) { + logger.debug( + "Skipping CSR clues - not an HTML page", + { url: pageInfo.url }, + "replay", + ); + return; + } + + if (!pageInfo.ts) { + logger.warn("Missing page timestamp, skipping", { pageInfo }); + return; + } + + const timestamp = new Date(pageInfo.ts) + .toISOString() + .slice(0, 19) + .replace(/[T:-]/g, ""); + const replayUrl = REPLAY_PREFIX + `${timestamp}mp_/${pageInfo.url}`; + + const frame = page.frames()[1]; + if (!frame) { + logger.warn( + "Replay frame missing for CSR clues", + { url: pageInfo.url }, + "replay", + ); + return; + } + + const htmlContent = await frame.evaluate(async (url) => { + const response = await fetch(url, { + method: "GET", + credentials: "include", + }); + if (response.status !== 200) { + logger.warn("Got non-200 status code, proceeding", { url }); + } + return await response.text(); + }, replayUrl); + + if (!htmlContent) { + logger.warn( + "Could not fetch HTML content for CSR clues", + { url: pageInfo.url }, + "replay", + ); + return; + } + + const clueCounts: Record = {}; + const categoryCounts: Record = {}; + + for (const clue of CSR_CLUES) { + const regex = new RegExp(clue.pattern, "gi"); + const matches = htmlContent.match(regex); + + if (matches && matches.length > 0) { + // count by clue name + clueCounts[clue.name] = matches.length; + + // count by category (sum of all patterns in category) + categoryCounts[clue.category] = + (categoryCounts[clue.category] || 0) + matches.length; + + logger.debug( + "CSR clue detected", + { + pattern: clue.name, + category: clue.category, + url: pageInfo.url, + count: matches.length, + }, + "replay", + ); + } + } + + // add clue counts to pageInfo + if (Object.keys(clueCounts).length > 0) { + pageInfo["csrClues"] = { + clues: clueCounts, + categories: categoryCounts, + }; + logger.info( + "CSR clues detected", + { + url: pageInfo.url, + categories: categoryCounts, + clues: clueCounts, + }, + "replay", + ); + } else { + logger.debug("No CSR clues detected", { url: pageInfo.url }, "replay"); + } + } + protected pageEntryForRedis( entry: Record, state: PageState, diff --git a/src/util/argParser.ts b/src/util/argParser.ts index f1f41b082..d6aca66a7 100644 --- a/src/util/argParser.ts +++ b/src/util/argParser.ts @@ -704,6 +704,12 @@ class ArgParser { type: "boolean", }, + qaDetectClientSideRendering: { + describe: + "use heuristics to detect when a page might use client-side rendering (CSR)", + type: "boolean", + }, + sshProxyPrivateKeyFile: { describe: "path to SSH private key for SOCKS5 over SSH proxy connection", diff --git a/src/util/clientSideRenderingClues.ts b/src/util/clientSideRenderingClues.ts new file mode 100644 index 000000000..836af1eff --- /dev/null +++ b/src/util/clientSideRenderingClues.ts @@ -0,0 +1,171 @@ +export type CSRClue = { + name: string; + category: string; + pattern: string; + description: string; +}; + +/** + * Various clues to identify client-side rendering (CSR) in source HTML. + */ +export const CSR_CLUES: CSRClue[] = [ + // Empty container divs + { + name: "empty_div", + category: "empty_container", + pattern: + "\\s*", + description: "Empty container divs typical of SPA/CSR", + }, + // Framework meta tags + { + name: "nextjs_generator", + category: "framework_meta", + pattern: "]*src=[\"'][^\"']*(?:webpack|vite|parcel|rollup|esbuild)[^\"']*\\.js[\"']", + description: "Bundler script", + }, + { + name: "bundled_script", + category: "bundler_scripts", + pattern: + "]*src=[\"'][^\"']*(?:bundle|chunk|vendor|runtime|main)[^\"']*\\.[a-f0-9]{8,}\\.js[\"']", + description: "Bundled script with hash", + }, + { + name: "webpack_data_attr", + category: "bundler_scripts", + pattern: "]*\\sdata-webpack=[\"'][^\"']*[\"']", + description: "Webpack data attribute", + }, + { + name: "vite_data_attr", + category: "bundler_scripts", + pattern: "]*\\sdata-vite-fid=[\"'][^\"']*[\"']", + description: "Vite data attribute", + }, + // Hydration data + { + name: "nextjs_hydration", + category: "hydration_data", + pattern: "]*id=[\"']__NEXT_DATA__[\"']", + description: "Next.js hydration data", + }, + { + name: "nuxt_hydration", + category: "hydration_data", + pattern: "]*id=[\"']__NUXT__[\"']", + description: "Nuxt hydration data", + }, + { + name: "vue_hydration", + category: "hydration_data", + pattern: "]*id=[\"']__VUE__[\"']", + description: "Vue hydration data", + }, + { + name: "angular_hydration", + category: "hydration_data", + pattern: "]*id=[\"']ng-state[\"']", + description: "Angular hydration data", + }, + { + name: "svelte_hydration", + category: "hydration_data", + pattern: "data-sveltekit-hydrate", + description: "Svelte hydration data", + }, + // Noscript warnings + { + name: "noscript", + category: "noscript_warning", + pattern: "