refactor text extraction & generate text from raw response when missing

in qa

Author: emma-sg

src/replaycrawler.ts

@@ -22,6 +22,7 @@ import { parseRx } from "./util/seeds.js";
 import { getFileOrUrlJson } from "./util/file_reader.js";
 import { WACZLoader } from "./util/wacz.js";
 import { CSR_CLUES } from "./util/clientSideRenderingClues.js";
+import { extractTextFromDOM } from "./util/textextract.js";
 
 // RWP Replay Prefix
 const REPLAY_PREFIX = "http://localhost:9990/replay/w/replay/";
@@ -44,6 +45,7 @@ type ComparisonData = {
   comparison: {
     screenshotMatch?: number;
     textMatch?: number;
+    rawTextMatch?: number;
     resourceCounts: {
       crawlGood?: number;
       crawlBad?: number;
@@ -467,6 +469,8 @@ export class ReplayCrawler extends Crawler {
 
     await this.compareText(page, data, url, date);
 
+    await this.compareRawText(page, data, url, date);
+
     await this.compareResources(page, data, url, date);
 
     if (this.params.qaDetectClientSideRendering)
@@ -596,6 +600,112 @@ export class ReplayCrawler extends Crawler {
     }
   }
 
+  async compareRawText(page: Page, state: PageState, url: string, date: Date) {
+    // Try to fetch the stored crawl-time raw response text
+    let rawText = await this.fetchOrigText(
+      page,
+      "text-from-response",
+      url,
+      date.toISOString().replace(/[^\d]/g, ""),
+    );
+
+    // If not stored (older archive), extract it from the archived raw response on-the-fly
+    if (rawText === undefined) {
+      logger.info(
+        "Stored raw text not found, extracting from archived response",
+        { url },
+        "replay",
+      );
+      rawText = await this.extractRawTextFromArchive(page, url, date);
+    }
+
+    // Get the replay-time rendered text (extracted from browser after JS runs)
+    const renderedText = state.text;
+
+    if (rawText === undefined || renderedText === undefined) {
+      logger.warn(
+        "Text missing for raw vs rendered comparison",
+        {
+          url,
+          rawTextLen: rawText?.length,
+          renderedTextLen: renderedText?.length,
+        },
+        "replay",
+      );
+      return;
+    }
+
+    const dist = levenshtein(rawText, renderedText);
+    const maxLen = Math.max(rawText.length, renderedText.length);
+
+    let matchPercent = 1.0;
+    if (maxLen > 0) {
+      matchPercent = (maxLen - dist) / maxLen;
+    }
+    logger.info("Raw vs Rendered Text Levenshtein Dist", {
+      url,
+      dist,
+      matchPercent,
+      maxLen,
+      rawTextLen: rawText.length,
+      renderedTextLen: renderedText.length,
+    });
+
+    const pageInfo = this.pageInfos.get(page);
+    if (pageInfo) {
+      pageInfo.comparison.rawTextMatch = matchPercent;
+    }
+  }
+
+  async extractRawTextFromArchive(
+    page: Page,
+    url: string,
+    date: Date,
+  ): Promise<string | undefined> {
+    const timestamp = date.toISOString().slice(0, 19).replace(/[T:-]/g, "");
+    // `if_` suffix ensures wabac.js serves the unaltered source
+    const replayUrl = REPLAY_PREFIX + `${timestamp}if_/${url}`;
+
+    const frame = page.frames()[1];
+    if (!frame) {
+      logger.warn(
+        "Replay frame missing for raw text extraction",
+        { url },
+        "replay",
+      );
+      return undefined;
+    }
+
+    const htmlContent = await frame.evaluate(async (url) => {
+      const response = await fetch(url, {
+        method: "GET",
+        credentials: "include",
+      });
+      if (response.status !== 200) {
+        return undefined;
+      }
+      return await response.text();
+    }, replayUrl);
+
+    if (!htmlContent) {
+      logger.warn(
+        "Could not fetch HTML content for raw text extraction",
+        { url },
+        "replay",
+      );
+      return undefined;
+    }
+
+    // Extract text from the raw HTML using the shared extraction logic
+    const { JSDOM } = await import("jsdom");
+    const dom = new JSDOM(htmlContent);
+    try {
+      return extractTextFromDOM(dom.window.document);
+    } finally {
+      dom.window.close();
+    }
+  }
+
   async compareResources(
     page: Page,
     state: PageState,