Merge pull request #28 from devopsabcs-engineering/feature/fix-crawl-seed-cap

fix(crawler): cap seed URLs to prevent crawlee premature termination

Author: emmanuelknafo

src/lib/crawler/__tests__/site-crawler.test.ts

@@ -216,6 +216,33 @@ describe('site-crawler', () => {
       const abortUpdate = calls.find(([, data]) => 'abortController' in data);
       expect(abortUpdate).toBeDefined();
     });
+
+    it('caps seed URLs below maxPages to prevent crawlee premature termination', async () => {
+      // Simulate a large sitemap that returns more URLs than maxPages
+      const sitemapUrls = Array.from({ length: 200 }, (_, i) => `https://example.com/page${i}`);
+      vi.mocked(discoverSitemapUrls).mockResolvedValueOnce(sitemapUrls);
+
+      const crawlee = await import('crawlee');
+      let capturedRunArg: string[] = [];
+      vi.mocked(crawlee.PlaywrightCrawler).mockImplementationOnce(function (this: Record<string, unknown>, options: Record<string, unknown>) {
+        _capturedOptions = options;
+        this.run = vi.fn().mockImplementation((urls: string[]) => {
+          capturedRunArg = urls;
+          return Promise.resolve();
+        });
+        return this;
+      });
+
+      const config = { ...defaultConfig, maxPages: 50 };
+      await startCrawl('crawl-cap', 'https://example.com', config);
+
+      // Should pass fewer than maxPages seed URLs to prevent the
+      // crawler from hitting maxRequestsPerCrawl during the enqueue phase
+      expect(capturedRunArg.length).toBeLessThan(config.maxPages);
+      expect(capturedRunArg.length).toBe(config.maxPages - 1);
+      // The original seed URL must always be included first
+      expect(capturedRunArg[0]).toBe('https://example.com');
+    });
   });
 
   describe('cancelCrawl', () => {

src/lib/crawler/site-crawler.ts

@@ -263,8 +263,17 @@ export async function startCrawl(
       },
     });
 
-    // Run the crawler with seed URLs
-    await crawler.run(Array.from(seedUrls));
+    // Run the crawler with seed URLs.
+    // Cap the seed list to (maxPages - 1) so crawlee's internal request counter
+    // does not reach maxRequestsPerCrawl during the initial enqueue phase.
+    // When the limit is hit before the processing loop starts, the crawler
+    // terminates immediately with 0 pages processed (observed with large
+    // sitemaps such as ontario.ca and microsoft.com).
+    const primarySeed = normalizeUrl(seedUrl);
+    const remainingSeeds = Array.from(seedUrls).filter(u => u !== primarySeed);
+    const maxSeeds = Math.max(1, config.maxPages - 1);
+    const cappedSeeds = [primarySeed, ...remainingSeeds].slice(0, maxSeeds);
+    await crawler.run(cappedSeeds);
 
     // Aggregation phase
     updateCrawl(crawlId, { status: 'aggregating', progress: 95, message: 'Aggregating results...' });