Implement text extraction from raw HTML + client-side rendering (CSR) detection in QA

package.json

@@ -31,6 +31,7 @@
     "iso-639-1": "^3.1.5",
     "js-levenshtein": "^1.1.6",
     "js-yaml": "^4.1.0",
+    "jsdom": "^29.0.0",
     "normalize-url": "^9.0.0",
     "p-queue": "^7.3.4",
     "pixelmatch": "^5.3.0",
@@ -50,6 +51,7 @@
     "@types/get-folder-size": "^3.0.4",
     "@types/js-levenshtein": "^1.1.3",
     "@types/js-yaml": "^4.0.8",
+    "@types/jsdom": "^28.0.0",
     "@types/node": "^20.8.7",
     "@types/pixelmatch": "^5.2.6",
     "@types/pngjs": "^6.0.4",

src/crawler.ts

@@ -19,7 +19,10 @@ import yaml from "js-yaml";
 import { WACZ, WACZInitOpts, mergeCDXJ } from "./util/wacz.js";
 
 import { HealthChecker } from "./util/healthcheck.js";
-import { TextExtractViaSnapshot } from "./util/textextract.js";
+import {
+  TextExtractViaResponse,
+  TextExtractViaSnapshot,
+} from "./util/textextract.js";
 import {
   initStorage,
   getFileSize,
@@ -1178,11 +1181,55 @@ self.__bx_behaviors.selectMainBehavior();
       recorder.pageSeedDepth = depth;
     }
 
+    // Enable Network domain to track document request for raw text extraction
+    let documentRequestId: string | null = null;
+    let trackDocumentRequest:
+      | ((params: Protocol.Network.ResponseReceivedEvent) => void)
+      | null = null;
+
+    if (this.params.text.includes("to-warc-from-raw")) {
+      await cdp.send("Network.enable");
+      trackDocumentRequest = (
+        params: Protocol.Network.ResponseReceivedEvent,
+      ) => {
+        if (params.type === "Document" && !documentRequestId) {
+          documentRequestId = params.requestId;
+          logger.debug(
+            "Captured document request ID",
+            { requestId: documentRequestId, url: params.response.url },
+            "text",
+          );
+        }
+      };
+      cdp.on("Network.responseReceived", trackDocumentRequest);
+    }
+
     // run custom driver here, if any
-    if (this.driver) {
-      await this.driver({ page, data, crawler: this, seed });
-    } else {
-      await this.loadPage(page, data, seed);
+    try {
+      if (this.driver) {
+        await this.driver({ page, data, crawler: this, seed });
+      } else {
+        await this.loadPage(page, data, seed);
+      }
+    } finally {
+      // Store document request ID and clean up listener
+      if (documentRequestId) {
+        data.documentRequestId = documentRequestId;
+        logger.info(
+          "Stored document request ID for raw text extraction",
+          { url: data.url, requestId: documentRequestId },
+          "text",
+        );
+      } else if (trackDocumentRequest) {
+        logger.warn(
+          "No document request ID captured",
+          { url: data.url },
+          "text",
+        );
+      }
+      if (trackDocumentRequest) {
+        cdp.off("Network.responseReceived", trackDocumentRequest);
+      }
     }
 
     data.title = await timedRun(
@@ -1235,23 +1282,61 @@ self.__bx_behaviors.selectMainBehavior();
       }
     }
 
-    let textextract = null;
+    let textExtract = null;
 
     if (this.textWriter) {
-      textextract = new TextExtractViaSnapshot(cdp, {
+      textExtract = new TextExtractViaSnapshot(cdp, {
         writer: this.textWriter,
         url,
         skipDocs: this.skipTextDocs,
       });
-      const { text } = await textextract.extractAndStoreText(
+      const { text } = await textExtract.extractAndStoreText(
         "text",
         false,
         this.params.text.includes("to-warc"),
+        data.originalWarcRecordId,
       );
 
       if (text !== null && (this.textInPages || saveOutput)) {
         data.text = text;
       }
+
+      if (this.params.text.includes("to-warc-from-raw")) {
+        logger.info(
+          "Extracting text from raw response",
+          {
+            url,
+            hasRequestId: !!data.documentRequestId,
+            refersTo: String(data.originalWarcRecordId),
+          },
+          "text",
+        );
+        textExtract = new TextExtractViaResponse(cdp, {
+          writer: this.textWriter,
+          url,
+          skipDocs: this.skipTextDocs,
+          requestId: data.documentRequestId,
+        });
+        const { text: textFromResponse } =
+          await textExtract.extractAndStoreText(
+            "text-from-response",
+            false,
+            this.params.text.includes("to-warc-from-raw"),
+            data.originalWarcRecordId,
+          );
+        logger.info(
+          "Extracted text from raw response result",
+          {
+            url,
+            hasText: textFromResponse !== null,
+            refersTo: String(data.originalWarcRecordId),
+          },
+          "text",
+        );
+        if (textFromResponse !== null && (this.textInPages || saveOutput)) {
+          data.textFromResponse = textFromResponse;
+        }
+      }
     }
 
     data.loadState = LoadState.EXTRACTION_DONE;
@@ -1280,8 +1365,13 @@ self.__bx_behaviors.selectMainBehavior();
           data.loadState = LoadState.BEHAVIORS_DONE;
         }
 
-        if (textextract && this.params.text.includes("final-to-warc")) {
-          await textextract.extractAndStoreText("textFinal", true, true);
+        if (textExtract && this.params.text.includes("final-to-warc")) {
+          await textExtract.extractAndStoreText(
+            "textFinal",
+            true,
+            true,
+            data.originalWarcRecordId,
+          );
         }
 
         if (
@@ -2640,6 +2730,7 @@ self.__bx_behaviors.selectMainBehavior();
     logDetails: LogDetails = {},
     ts = 0,
     pageid?: string,
+    originalWarcRecordId?: string,
   ) {
     if (this.limitHit) {
       return false;
@@ -2658,7 +2749,7 @@ self.__bx_behaviors.selectMainBehavior();
     }
 
     const result = await this.crawlState.addToQueue(
-      { url, seedId, depth, extraHops, ts, pageid },
+      { url, seedId, depth, extraHops, ts, pageid, originalWarcRecordId },
       this.pageLimit,
     );