refactor: move blocked HTTP status code management to BasicCrawler (#3496)

Refactors the blocked HTTP status code handling so it's managed by the
Crawler instance rather than `SessionPool`.

Closes #2441

Author: barjin

docs/guides/session_management_basic.ts

@@ -34,9 +34,6 @@ const crawler = new BasicCrawler({
             throw e;
         }
 
-        // Automatically retires the session based on response HTTP status code.
-        session?.retireOnBlockedStatusCodes(response.status);
-
         if ((await response.text()).includes('You are blocked!')) {
             // You are sure it is blocked.
             // This will throw away the session.

docs/upgrading/upgrading_v4.md

@@ -84,9 +84,9 @@ const crawler = new CheerioCrawler({
 
 Previously, the crawling context extended a `Record` type, allowing to access any property. This was changed to a strict type, which means that you can only access properties that are defined in the context.
 
-## `additionalBlockedStatusCodes` parameter is removed
+## `retireOnBlockedStatusCodes` is removed from `Session`
 
-`additionalBlockedStatusCodes` parameter of `Session.retireOnBlockedStatusCodes` method is removed. Use the `blockedStatusCodes` crawler option instead.
+`Session.retireOnBlockedStatusCodes` is removed. Blocked status code handling is now internal to the crawler. Configure blocked status codes via the `blockedStatusCodes` crawler option (moved from `sessionPoolOptions`).
 
 ## Remove `experimentalContainers` option
 

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -31,6 +31,7 @@ import type {
 import {
     AutoscaledPool,
     bindMethodsToServiceLocator,
+    BLOCKED_STATUS_CODES,
     ContextPipeline,
     ContextPipelineCleanupError,
     ContextPipelineInitializationError,
@@ -338,6 +339,12 @@ export interface BasicCrawlerOptions<
      */
     statusMessageCallback?: StatusMessageCallback;
 
+    /**
+     * HTTP status codes that indicate the session should be retired.
+     * @default [401, 403, 429]
+     */
+    blockedStatusCodes?: number[];
+
     /**
      * If set to `true`, the crawler will automatically try to bypass any detected bot protection.
      *
@@ -426,6 +433,18 @@ export interface BasicCrawlerOptions<
      *
      */
     id?: string;
+
+    /**
+     * An array of HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be excluded from error consideration.
+     * By default, status codes >= 500 trigger errors.
+     */
+    ignoreHttpErrorStatusCodes?: number[];
+
+    /**
+     * An array of additional HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be treated as errors.
+     * By default, status codes >= 500 trigger errors.
+     */
+    additionalHttpErrorStatusCodes?: number[];
 }
 
 /**
@@ -546,7 +565,6 @@ export class BasicCrawler<
 
     /**
      * A reference to the underlying {@apilink SessionPool} class that manages the crawler's {@apilink Session|sessions}.
-     * Only available if used by the crawler.
      */
     sessionPool?: SessionPool;
 
@@ -624,6 +642,9 @@ export class BasicCrawler<
     protected statusMessageLoggingInterval: number;
     protected statusMessageCallback?: StatusMessageCallback;
     protected sessionPoolOptions: SessionPoolOptions;
+    protected blockedStatusCodes = new Set<number>();
+    protected additionalHttpErrorStatusCodes: Set<number>;
+    protected ignoreHttpErrorStatusCodes: Set<number>;
     protected autoscaledPoolOptions: AutoscaledPoolOptions;
     protected httpClient: BaseHttpClient;
     protected retryOnBlocked: boolean;
@@ -666,6 +687,10 @@ export class BasicCrawler<
         statusMessageLoggingInterval: ow.optional.number,
         statusMessageCallback: ow.optional.function,
 
+        additionalHttpErrorStatusCodes: ow.optional.array.ofType(ow.number),
+        ignoreHttpErrorStatusCodes: ow.optional.array.ofType(ow.number),
+
+        blockedStatusCodes: ow.optional.array.ofType(ow.number),
         retryOnBlocked: ow.optional.boolean,
         respectRobotsTxtFile: ow.optional.any(ow.boolean, ow.object),
         onSkippedRequest: ow.optional.function,
@@ -713,6 +738,9 @@ export class BasicCrawler<
             sessionPoolOptions = {},
             proxyConfiguration,
 
+            additionalHttpErrorStatusCodes = [],
+            ignoreHttpErrorStatusCodes = [],
+
             // Service locator options
             configuration,
             storageClient,
@@ -724,6 +752,7 @@ export class BasicCrawler<
             maxConcurrency,
             maxRequestsPerMinute,
 
+            blockedStatusCodes: blockedStatusCodesInput,
             retryOnBlocked = false,
             respectRobotsTxtFile = false,
             onSkippedRequest,
@@ -795,6 +824,9 @@ export class BasicCrawler<
             this.robotsTxtFileCache = new LruCache({ maxLength: 1000 });
             this.handleSkippedRequest = this.handleSkippedRequest.bind(this);
 
+            this.additionalHttpErrorStatusCodes = new Set([...additionalHttpErrorStatusCodes]);
+            this.ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);
+
             this.requestHandler = requestHandler ?? this.router;
             this.failedRequestHandler = failedRequestHandler;
             this.errorHandler = errorHandler;
@@ -836,14 +868,8 @@ export class BasicCrawler<
                 ...sessionPoolOptions,
                 log: this.log,
             };
-            if (this.retryOnBlocked) {
-                this.sessionPoolOptions.blockedStatusCodes = sessionPoolOptions.blockedStatusCodes ?? [];
-                if (this.sessionPoolOptions.blockedStatusCodes.length !== 0) {
-                    this.log.warning(
-                        `Both 'blockedStatusCodes' and 'retryOnBlocked' are set. Please note that the 'retryOnBlocked' feature might not work as expected.`,
-                    );
-                }
-            }
+            this.blockedStatusCodes = new Set(blockedStatusCodesInput ?? BLOCKED_STATUS_CODES);
+
             const maxSignedInteger = 2 ** 31 - 1;
             if (this.requestHandlerTimeoutMillis > maxSignedInteger) {
                 this.log.warning(
@@ -979,6 +1005,19 @@ export class BasicCrawler<
         }
     }
 
+    /**
+     * Determines if the given HTTP status code is an error status code given
+     * the default behaviour and user-set preferences.
+     * @param status
+     * @returns `true` if the status code is considered an error, `false` otherwise
+     */
+    protected isErrorStatusCode(status: number): boolean {
+        const excludeError = this.ignoreHttpErrorStatusCodes.has(status);
+        const includeError = this.additionalHttpErrorStatusCodes.has(status);
+
+        return (status >= 500 && !excludeError) || includeError;
+    }
+
     /**
      * Builds the basic context pipeline that transforms `{ request }` into a full `CrawlingContext`.
      * This handles base context creation, session resolution, and context helpers.
@@ -1640,11 +1679,11 @@ export class BasicCrawler<
     /**
      * Handles blocked request
      */
-    protected _throwOnBlockedRequest(session: Session, statusCode: number) {
-        const isBlocked = session.retireOnBlockedStatusCodes(statusCode);
+    protected _throwOnBlockedRequest(statusCode: number) {
+        if (this.retryOnBlocked) return;
 
-        if (isBlocked) {
-            throw new Error(`Request blocked - received ${statusCode} status code.`);
+        if (this.blockedStatusCodes.has(statusCode)) {
+            throw new SessionError(`Request blocked - received ${statusCode} status code.`);
         }
     }
 
@@ -2040,7 +2079,9 @@ export class BasicCrawler<
             }
 
             if (!request.noRetry) {
-                request.retryCount++;
+                if (!(error instanceof SessionError)) {
+                    request.retryCount++;
+                }
 
                 const { url, retryCount, id } = request;
 
@@ -2058,6 +2099,10 @@ export class BasicCrawler<
             }
         }
 
+        if (error instanceof SessionError) {
+            crawlingContext.session?.retire();
+        }
+
         // If the request is non-retryable, the error and snapshot aren't saved in the errorTrackerRetry object.
         // Therefore, we pass the crawlingContext to the errorTracker.add method, enabling snapshot capture.
         // This is to make sure the error snapshot is not duplicated in the errorTrackerRetry and errorTracker objects.

packages/browser-crawler/src/internals/browser-crawler.ts

@@ -15,7 +15,6 @@ import type {
 } from '@crawlee/basic';
 import {
     BasicCrawler,
-    BLOCKED_STATUS_CODES as DEFAULT_BLOCKED_STATUS_CODES,
     ContextPipeline,
     cookieStringToToughCookie,
     enqueueLinks,
@@ -429,13 +428,6 @@ export abstract class BrowserCrawler<
     ): Promise<string | false> {
         const { page, response } = crawlingContext;
 
-        const blockedStatusCodes =
-            // eslint-disable-next-line dot-notation
-            (this.sessionPool?.['blockedStatusCodes'].length ?? 0) > 0
-                ? // eslint-disable-next-line dot-notation
-                  this.sessionPool!['blockedStatusCodes']
-                : DEFAULT_BLOCKED_STATUS_CODES;
-
         // Cloudflare specific heuristic - wait 5 seconds if we get a 403 for the JS challenge to load / resolve.
         if ((await this.containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {
             await sleep(5000);
@@ -448,10 +440,10 @@ export abstract class BrowserCrawler<
         }
 
         const foundSelectors = await this.containsSelectors(page, RETRY_CSS_SELECTORS);
-        const blockedStatusCode = blockedStatusCodes.find((x) => x === (response?.status() ?? 0));
+        const statusCode = response?.status() ?? 0;
 
         if (foundSelectors) return `Found selectors: ${foundSelectors.join(', ')}`;
-        if (blockedStatusCode) return `Received blocked status code: ${blockedStatusCode}`;
+        if (this.blockedStatusCodes.has(statusCode)) return `Received blocked status code: ${statusCode}`;
 
         return false;
     }
@@ -651,11 +643,19 @@ export abstract class BrowserCrawler<
             const status: number = response.status();
 
             this.stats.registerStatusCode(status);
+
+            if (this.isErrorStatusCode(status)) {
+                if (this.additionalHttpErrorStatusCodes.has(status)) {
+                    throw new Error(`${status} - Error status code was set by user.`);
+                }
+
+                throw new Error(`${status} - Internal Server Error`);
+            }
         }
 
         if (this.sessionPool && response && session) {
             if (typeof response === 'object' && typeof response.status === 'function') {
-                this._throwOnBlockedRequest(session, response.status());
+                this._throwOnBlockedRequest(response.status());
             } else {
                 this.log.debug('Got a malformed Browser response.', { request, response });
             }

packages/core/src/session_pool/session.ts

@@ -288,23 +288,6 @@ export class Session implements ISession {
         this._maybeSelfRetire();
     }
 
-    /**
-     * With certain status codes: `401`, `403` or `429` we can be certain
-     * that the target website is blocking us. This function helps to do this conveniently
-     * by retiring the session when such code is received. Optionally, the default status
-     * codes can be extended in the second parameter.
-     * @param statusCode HTTP status code.
-     * @returns Whether the session was retired.
-     */
-    retireOnBlockedStatusCodes(statusCode: number): boolean {
-        // eslint-disable-next-line dot-notation -- accessing private property
-        const isBlocked = this.sessionPool['blockedStatusCodes'].includes(statusCode);
-        if (isBlocked) {
-            this.retire();
-        }
-        return isBlocked;
-    }
-
     /**
      * Saves cookies from an HTTP response to be used with the session.
      * It expects an object with a `headers` property that's either an `Object`

packages/core/src/session_pool/session_pool.ts

@@ -10,7 +10,7 @@ import { EventType } from '../events/event_manager.js';
 import type { CrawleeLogger } from '../log.js';
 import { serviceLocator } from '../service_locator.js';
 import { KeyValueStore } from '../storages/key_value_store.js';
-import { BLOCKED_STATUS_CODES, MAX_POOL_SIZE, PERSIST_STATE_KEY } from './consts.js';
+import { MAX_POOL_SIZE, PERSIST_STATE_KEY } from './consts.js';
 import type { SessionOptions } from './session.js';
 import { Session } from './session.js';
 
@@ -51,13 +51,6 @@ export interface SessionPoolOptions {
      */
     createSessionFunction?: CreateSession;
 
-    /**
-     * Specifies which response status codes are considered as blocked.
-     * Session connected to such request will be marked as retired.
-     * @default [401, 403, 429]
-     */
-    blockedStatusCodes?: number[];
-
     /** @internal */
     log?: CrawleeLogger;
 
@@ -133,7 +126,6 @@ export class SessionPool extends EventEmitter {
     protected persistStateKey: string;
     protected _listener!: () => Promise<void>;
     protected events: EventManager;
-    protected readonly blockedStatusCodes: number[];
     protected persistenceOptions: PersistenceOptions;
     protected isInitialized = false;
 
@@ -153,7 +145,6 @@ export class SessionPool extends EventEmitter {
                 persistStateKey: ow.optional.string,
                 createSessionFunction: ow.optional.function,
                 sessionOptions: ow.optional.object,
-                blockedStatusCodes: ow.optional.array.ofType(ow.number),
                 log: ow.optional.object,
                 persistenceOptions: ow.optional.object,
             }),
@@ -165,14 +156,12 @@ export class SessionPool extends EventEmitter {
             persistStateKey = PERSIST_STATE_KEY,
             createSessionFunction,
             sessionOptions = {},
-            blockedStatusCodes = BLOCKED_STATUS_CODES,
             log = serviceLocator.getLogger(),
             persistenceOptions = {
                 enable: true,
             },
         } = options;
 
-        this.blockedStatusCodes = blockedStatusCodes;
         this.events = serviceLocator.getEventManager();
         this.log = log.child({ prefix: 'SessionPool' });
         this.persistenceOptions = persistenceOptions;