finish passing through refersTo uuid

Author: emma-sg

src/crawler.ts

@@ -1294,6 +1294,7 @@ self.__bx_behaviors.selectMainBehavior();
         "text",
         false,
         this.params.text.includes("to-warc"),
+        data.originalWarcRecordId,
       );
 
       if (text !== null && (this.textInPages || saveOutput)) {
@@ -1303,7 +1304,11 @@ self.__bx_behaviors.selectMainBehavior();
       if (this.params.text.includes("to-warc-from-raw")) {
         logger.info(
           "Extracting text from raw response",
-          { url, hasRequestId: !!data.documentRequestId },
+          {
+            url,
+            hasRequestId: !!data.documentRequestId,
+            refersTo: String(data.originalWarcRecordId),
+          },
           "text",
         );
         textExtract = new TextExtractViaResponse(cdp, {
@@ -1317,10 +1322,15 @@ self.__bx_behaviors.selectMainBehavior();
             "text-from-response",
             false,
             this.params.text.includes("to-warc-from-raw"),
+            data.originalWarcRecordId,
           );
         logger.info(
           "Extracted text from raw response result",
-          { url, hasText: textFromResponse !== null },
+          {
+            url,
+            hasText: textFromResponse !== null,
+            refersTo: String(data.originalWarcRecordId),
+          },
           "text",
         );
         if (textFromResponse !== null && (this.textInPages || saveOutput)) {
@@ -1356,7 +1366,12 @@ self.__bx_behaviors.selectMainBehavior();
         }
 
         if (textExtract && this.params.text.includes("final-to-warc")) {
-          await textExtract.extractAndStoreText("textFinal", true, true);
+          await textExtract.extractAndStoreText(
+            "textFinal",
+            true,
+            true,
+            data.originalWarcRecordId,
+          );
         }
 
         if (

src/replaycrawler.ts

@@ -236,6 +236,7 @@ export class ReplayCrawler extends Crawler {
   }
 
   async _addPageIfInScope({ url, ts, id, mime }: ReplayPage, depth: number) {
+    logger.debug("Adding page to queue", { url, id, hasId: !!id }, "replay");
     if (mime && mime !== "text/html") {
       logger.info("Skipping non-HTML page", { url, mime }, "replay");
       return;
@@ -779,7 +780,7 @@ export class ReplayCrawler extends Crawler {
       );
       logger.info(
         "Wrote HTML to WARC",
-        { url, length: htmlContent.length },
+        { url, length: htmlContent.length, refersTo: originalWarcRecordId },
         "replay",
       );
     }
@@ -967,6 +968,11 @@ export class ReplayCrawler extends Crawler {
         { type: "pageinfo", url: pageInfo.url },
         "replay",
       );
+      logger.debug(
+        "Wrote pageinfo to WARC",
+        { url: pageInfo.url, refersTo: state?.originalWarcRecordId },
+        "replay",
+      );
 
       this.pageInfos.delete(page);
     }

src/util/state.ts

@@ -125,6 +125,15 @@ export class PageState {
     this.status = 0;
     this.retry = redisData.retry || 0;
     this.originalWarcRecordId = redisData.originalWarcRecordId;
+    logger.debug(
+      "PageState created",
+      {
+        url: this.url,
+        pageid: this.pageid,
+        originalWarcRecordId: String(this.originalWarcRecordId),
+      },
+      "state",
+    );
   }
 }
 
@@ -1296,6 +1305,9 @@ return inx;
     if (pageid) {
       data.pageid = pageid;
     }
+    if (originalWarcRecordId) {
+      data.originalWarcRecordId = originalWarcRecordId;
+    }
 
     // return codes
     // 0 - url queued successfully

src/util/textextract.ts

@@ -33,6 +33,7 @@ export abstract class BaseTextExtract {
     resourceType: string,
     ignoreIfMatchesLast = false,
     saveToWarc = false,
+    refersTo?: string,
   ) {
     try {
       const text = await this.doGetText();
@@ -53,6 +54,7 @@ export abstract class BaseTextExtract {
             resourceType,
             contentType: "text/plain",
             url: this.url,
+            refersTo,
           },
           {
             resource: "text",

src/util/warcwriter.ts

@@ -265,7 +265,7 @@ export class WARCWriter implements IndexerOffsetLength {
     const warcRecordType = "resource";
     const warcHeaders: Record<string, string> = { "Content-Type": contentType };
     if (refersTo) {
-      warcHeaders["WARC-Refers-To"] = refersTo;
+      warcHeaders["WARC-Refers-To"] = `<url:uuid:${refersTo}>`;
     }
     async function* content() {
       yield buffer;