Merge pull request #36 from devopsabcs-engineering/feature/fix-crawl-requestqueue

fix(crawler): use per-crawl RequestQueue for isolation (crawlee v3 compatible)

Author: emmanuelknafo

src/lib/crawler/__tests__/site-crawler.test.ts

@@ -10,9 +10,15 @@ vi.mock('crawlee', () => ({
     this.run = vi.fn().mockResolvedValue(undefined);
     return this;
   }),
-  Configuration: vi.fn().mockImplementation(function () {
-    return { get: vi.fn(), set: vi.fn() };
-  }),
+  Configuration: {
+    getGlobalConfig: vi.fn().mockReturnValue({ set: vi.fn() }),
+  },
+  RequestQueue: {
+    open: vi.fn().mockResolvedValue({
+      drop: vi.fn().mockResolvedValue(undefined),
+    }),
+  },
+  purgeDefaultStorages: vi.fn().mockResolvedValue(undefined),
 }));
 
 vi.mock('uuid', () => ({ v4: vi.fn().mockReturnValue('mock-page-id') }));

src/lib/crawler/site-crawler.ts

@@ -1,4 +1,4 @@
-import { PlaywrightCrawler, Configuration, type PlaywrightCrawlingContext } from 'crawlee';
+import { PlaywrightCrawler, Configuration, RequestQueue, purgeDefaultStorages, type PlaywrightCrawlingContext } from 'crawlee';
 import { v4 as uuidv4 } from 'uuid';
 import { scanPage } from '../scanner/engine';
 import { parseAxeResults } from '../scanner/result-parser';
@@ -43,6 +43,9 @@ export async function startCrawl(
   // We update this on the first request so domain boundary checks use the post-redirect hostname.
   let effectiveSeedUrl = seedUrl;
 
+  // Per-crawl request queue — declared here so it can be dropped in finally
+  let requestQueue: Awaited<ReturnType<typeof RequestQueue.open>> | undefined;
+
   try {
     // Phase: discovering
     updateCrawl(crawlId, { status: 'discovering', progress: 5, message: 'Fetching robots.txt and sitemaps...' });
@@ -84,14 +87,14 @@ export async function startCrawl(
     });
     emitProgress(crawlId, completedPages, onProgress);
 
-    // Each crawl gets its own Configuration so crawlee uses a fresh,
-    // isolated request queue. Without this, the default singleton queue
-    // retains "already handled" URLs from previous crawls, causing
-    // subsequent crawls to immediately shut down with 0 results.
-    const crawlConfig = new Configuration({
-      persistStorage: false,
-      storageClientOptions: { localDataDirectory: `/tmp/crawlee-${crawlId}` },
-    });
+    // Purge default storages before each crawl to clear any stale state,
+    // then open a uniquely-named RequestQueue so this crawl is fully isolated
+    // from previous runs. Without this, crawlee's default singleton queue
+    // retains "already handled" URLs, causing subsequent crawls to shut down
+    // with 0 results.
+    Configuration.getGlobalConfig().set('persistStorage', false);
+    await purgeDefaultStorages();
+    requestQueue = await RequestQueue.open(`crawl-${crawlId}`);
 
     // Track depth per URL
     const urlDepth = new Map<string, number>();
@@ -102,7 +105,7 @@ export async function startCrawl(
       maxConcurrency: config.concurrency,
       requestHandlerTimeoutSecs: 60,
       navigationTimeoutSecs: 30,
-      configuration: crawlConfig,
+      requestQueue,
       launchContext: {
         launchOptions: {
           headless: true,
@@ -316,6 +319,10 @@ export async function startCrawl(
   } finally {
     activeAbortControllers.delete(crawlId);
     clearRobotsCache();
+    // Drop the per-crawl request queue to free memory
+    if (requestQueue) {
+      await requestQueue.drop().catch(() => {});
+    }
   }
 }