Sitemaps: parse /sitemap.xml if no sitemap listed in robots.txt (#933)

- Ensure the /sitemap.xml is parsed even if robots.txt exists, but no
sitemaps listed there.
- Resolve relative URLs listed in robots.txt, eg. 'Sitemap:
/my-sitemap.xml'
- Simplify sitemap detection logic, check robots first, then sitemap.xml
OR alternate url if provided via --useSitemap <url>
- Have two main methods, parseSitemap() and parseSitemapFromRobots()
that handle the parsing.
- follow-up to #930

---------

Co-authored-by: Tessa Walsh <tessa@bitarchivist.net>

Author: ikreymer

package.json

@@ -1,6 +1,6 @@
 {
   "name": "browsertrix-crawler",
-  "version": "1.10.0",
+  "version": "1.10.1",
   "main": "browsertrix-crawler",
   "type": "module",
   "repository": "https://github.com/webrecorder/browsertrix-crawler",

src/crawler.ts

@@ -2809,18 +2809,18 @@ self.__bx_behaviors.selectMainBehavior();
       });
     });
 
+    let found = false;
     try {
-      await sitemapper.parse(sitemap, url);
+      found = await sitemapper.parse(sitemap, url);
     } catch (e) {
-      logger.warn(
-        "Sitemap for seed failed",
-        { url, sitemap, ...formatErr(e) },
-        "sitemap",
-      );
-      return;
+      //
     }
 
-    await p;
+    if (found) {
+      await p;
+    } else {
+      logger.warn("No sitemap for seed", { url, sitemap }, "sitemap");
+    }
   }
 
   async combineWARC() {

src/util/sitemapper.ts

@@ -62,14 +62,23 @@ export class SitemapReader extends EventEmitter {
     return ct.split(";")[0];
   }
 
-  async _fetchWithRetry(url: string, message: string) {
+  async _fetchWithRetry(url: string, expectedCT = XML_CONTENT_TYPES) {
     while (true) {
       const resp = await fetch(url, {
         headers: this.headers,
         dispatcher: getProxyDispatcher(url),
       });
 
       if (resp.ok) {
+        const ct = resp.headers.get("content-type");
+        if (expectedCT && ct && !expectedCT.includes(ct.split(";")[0])) {
+          logger.debug(
+            "Not loading sitemap: invalid content-type",
+            { ct },
+            "sitemap",
+          );
+          return null;
+        }
         return resp;
       }
 
@@ -85,148 +94,108 @@ export class SitemapReader extends EventEmitter {
         continue;
       }
 
-      logger.debug(message, { status: resp.status }, "sitemap");
-      return null;
-    }
-  }
-
-  async tryFetch(url: string, expectedCT?: string[] | null) {
-    try {
       logger.debug(
-        "Detecting Sitemap: fetching",
-        { url, expectedCT },
+        "Not loading sitemap: invalid status code",
+        { status: resp.status },
         "sitemap",
       );
-
-      const resp = await this._fetchWithRetry(
-        url,
-        "Detecting Sitemap: invalid status code",
-      );
-
-      if (!resp) {
-        return null;
-      }
-
-      const ct = resp.headers.get("content-type");
-      if (expectedCT && ct && !expectedCT.includes(ct.split(";")[0])) {
-        logger.debug(
-          "Detecting Sitemap: invalid content-type",
-          { ct },
-          "sitemap",
-        );
-        return null;
-      }
-
-      return resp;
-    } catch (e) {
-      logger.debug("Detecting Sitemap: unknown error", e, "sitemap");
       return null;
     }
   }
 
   async parse(sitemap: string, seedUrl: string) {
-    let resp: Response | null = null;
-    let fullUrl: string | null = null;
-    let isRobots = false;
-    let isSitemap = false;
-
-    // if set to auto-detect, eg. --sitemap / --useSitemap with no URL
-    // 1. first check robots.txt
-    // 2. if not found, check /sitemap.xml
+    let found = false;
+
     if (sitemap === DETECT_SITEMAP) {
+      // if set to auto-detect, eg. --sitemap / --useSitemap with no URL
+      // 1. first check robots.txt
+      // 2. if not found, check /sitemap.xml
       logger.debug("Detecting sitemap for seed", { seedUrl }, "sitemap");
-      fullUrl = new URL("/robots.txt", seedUrl).href;
-      resp = await this.tryFetch(fullUrl, TEXT_CONTENT_TYPE);
-      if (resp) {
-        isRobots = true;
-      } else {
-        fullUrl = new URL("/sitemap.xml", seedUrl).href;
-        resp = await this.tryFetch(fullUrl, XML_CONTENT_TYPES);
-        if (resp) {
-          isSitemap = true;
-        }
+      const robotsUrl = new URL("/robots.txt", seedUrl).href;
+      found = await this.parseRobotsForSitemap(robotsUrl);
+
+      const sitemapUrl = new URL("/sitemap.xml", seedUrl).href;
+
+      if (!found) {
+        found = await this.parseSitemap(sitemapUrl);
+      }
+      if (!found) {
+        logger.debug(
+          "Sitemap not detected in robots.txt or sitemap.xml",
+          { robotsUrl, sitemapUrl, seedUrl },
+          "sitemap",
+        );
       }
     } else {
       // if specific URL provided, check if its a .xml file or a robots.txt file
-      fullUrl = new URL(sitemap, seedUrl).href;
-      let expected = null;
+      const fullUrl = new URL(sitemap, seedUrl).href;
       if (fullUrl.endsWith(".xml") || fullUrl.endsWith(".xml.gz")) {
-        expected = XML_CONTENT_TYPES;
-        isSitemap = true;
+        found = await this.parseSitemap(fullUrl);
       } else if (fullUrl.endsWith(".txt")) {
-        expected = TEXT_CONTENT_TYPE;
-        isRobots = true;
+        found = await this.parseRobotsForSitemap(fullUrl);
+      } else {
+        logger.debug(
+          "URL provided must be a sitemap XML or robots TXT file",
+          { sitemap, seedUrl, fullUrl },
+          "sitemap",
+        );
       }
-      resp = await this.tryFetch(fullUrl, expected);
     }
 
-    // fail if no successful response fetched
-    if (!resp) {
-      logger.debug(
-        "Sitemap not found",
-        { sitemap, seedUrl, fullUrl },
-        "sitemap",
-      );
-      throw new Error("not found");
-    }
-
-    // fail if neither an xml nor robots.txt
-    if (!isRobots && !isSitemap) {
-      logger.info("Sitemap not detected for seed", { seedUrl }, "sitemap");
-      throw new Error("not xml or robots.txt");
-    }
+    return found;
+  }
 
-    if (isRobots) {
-      logger.debug(
-        "Sitemap: parsing from robots.txt",
-        { fullUrl, seedUrl },
-        "sitemap",
-      );
-      await this._parseRobotsFromResponse(resp);
-    } else if (isSitemap) {
+  private async parseRobotsForSitemap(robotsUrl: string) {
+    let sitemapFound = false;
+    try {
       logger.debug(
-        "Sitemap: parsing from top-level sitemap XML",
-        { fullUrl, seedUrl },
+        "Sitemap: Parsing robots to detect sitemap",
+        { url: robotsUrl },
         "sitemap",
       );
-      await this.parseSitemapFromResponse(fullUrl, resp);
-    }
+      const resp = await this._fetchWithRetry(robotsUrl, TEXT_CONTENT_TYPE);
+      if (!resp) {
+        return sitemapFound;
+      }
 
-    await this.checkIfDone();
-  }
+      const text = await resp.text();
 
-  async parseFromRobots(url: string) {
-    const resp = await this._fetchWithRetry(
-      url,
-      "Sitemap robots.txt parse failed",
-    );
-    if (!resp) {
-      return;
+      text.replace(/^Sitemap:\s?([^\s]+)$/gim, (m, urlStr) => {
+        try {
+          const url = new URL(urlStr, robotsUrl).href;
+          logger.debug("Sitemap: Added from robots", { url }, "sitemap");
+          this.addNewSitemap(url, null);
+          sitemapFound = true;
+        } catch (e) {
+          // ignore invalid
+        }
+        return urlStr;
+      });
+    } catch (e) {
+      //
     }
-
-    await this._parseRobotsFromResponse(resp);
-  }
-
-  private async _parseRobotsFromResponse(resp: Response) {
-    const text = await resp.text();
-
-    text.replace(/^Sitemap:\s?([^\s]+)$/gim, (m, url) => {
-      this.addNewSitemap(url, null);
-      return url;
-    });
+    return sitemapFound;
   }
 
   async parseSitemap(url: string) {
-    this.seenSitemapSet.add(url);
+    try {
+      this.seenSitemapSet.add(url);
 
-    const resp = await this._fetchWithRetry(url, "Sitemap parse failed");
-    if (!resp) {
-      return;
-    }
+      logger.debug("Parsing sitemap XML", url, "sitemap");
+
+      const resp = await this._fetchWithRetry(url);
+      if (!resp) {
+        return false;
+      }
 
-    await this.parseSitemapFromResponse(url, resp);
+      await this.parseSitemapFromResponse(url, resp);
 
-    await this.checkIfDone();
+      await this.checkIfDone();
+      return true;
+    } catch (e) {
+      logger.warn("Sitemap parse failed", { url, ...formatErr(e) }, "sitemap");
+      return false;
+    }
   }
 
   private async parseSitemapFromResponse(url: string, resp: Response) {
@@ -430,14 +399,19 @@ export class SitemapReader extends EventEmitter {
     parserStream.on("text", (text: string) => processText(text));
     parserStream.on("cdata", (text: string) => processText(text));
 
+    let limitLogged = false;
+
     parserStream.on("error", (err: Error) => {
       const msg = { url, err, errCount };
       if (this.atLimit()) {
-        logger.warn(
-          "Sitemap parsing aborting, page limit reached",
-          msg,
-          "sitemap",
-        );
+        if (!limitLogged) {
+          logger.warn(
+            "Sitemap parsing aborting, page limit reached",
+            msg,
+            "sitemap",
+          );
+          limitLogged = true;
+        }
         resolve();
       } else {
         logger.warn("Sitemap error parsing XML", msg, "sitemap");
@@ -490,17 +464,7 @@ export class SitemapReader extends EventEmitter {
       return;
     }
 
-    void this.queue.add(async () => {
-      try {
-        await this.parseSitemap(url);
-      } catch (e) {
-        logger.warn(
-          "Sitemap parse failed",
-          { url, ...formatErr(e) },
-          "sitemap",
-        );
-      }
-    });
+    void this.queue.add(() => this.parseSitemap(url));
   }
 
   emitEntry(url: string | null, lastmod: Date | null) {