refactor: use ContextPipeline to initialize BasicCrawler's context idiomatically (#3388)

Extracts all `CrawlingContext` initialization to `ContextPipeline` steps
to tighten the control over the `CrawlingContext` contents.

Blocks #3380

Author: barjin

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -411,7 +411,7 @@ export abstract class BrowserCrawler<
         });
     }
 
-    protected buildContextPipeline(): ContextPipeline<
+    protected override buildContextPipeline(): ContextPipeline<
         CrawlingContext,
         BrowserCrawlingContext<Page, Response, ProvidedController, Dictionary>
     > {

packages/browser-crawler/src/internals/browser-crawler.ts

@@ -176,17 +176,23 @@ export class CheerioCrawler<
      * All `CheerioCrawler` parameters are passed via an options object.
      */
     constructor(options?: CheerioCrawlerOptions<ContextExtension, ExtendedContext>) {
+        const { contextPipelineBuilder, ...rest } = options ?? {};
+
         super({
-            ...options,
-            contextPipelineBuilder: () =>
-                this.buildContextPipeline()
-                    .compose({
-                        action: async (context) => await this.parseContent(context),
-                    })
-                    .compose({ action: async (context) => await this.addHelpers(context) }),
+            ...rest,
+            contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
         });
     }
 
+    protected override buildContextPipeline() {
+        return super
+            .buildContextPipeline()
+            .compose({
+                action: async (context) => await this.parseContent(context),
+            })
+            .compose({ action: async (context) => await this.addHelpers(context) });
+    }
+
     private async parseContent(crawlingContext: InternalHttpCrawlingContext) {
         const isXml = crawlingContext.contentType.type.includes('xml');
         const body = Buffer.isBuffer(crawlingContext.body)

packages/cheerio-crawler/src/internals/cheerio-crawler.ts

@@ -1,5 +1,7 @@
 import type { Awaitable } from '@crawlee/types';
 
+import log from '@apify/log';
+
 import {
     ContextPipelineCleanupError,
     ContextPipelineInitializationError,
@@ -56,6 +58,19 @@ export abstract class ContextPipeline<TContextBase, TCrawlingContext extends TCo
         middleware: ContextMiddleware<TCrawlingContext, TCrawlingContextExtension>,
     ): ContextPipeline<TContextBase, TCrawlingContext & TCrawlingContextExtension>;
 
+    /**
+     * Chains another pipeline onto this one. The other pipeline's base context must match
+     * this pipeline's output context. Returns a new pipeline that runs this pipeline's
+     * middlewares first, then the other pipeline's middlewares.
+     *
+     * @template TFinalContext - The final context type after the chained pipeline's transformations
+     * @param other - The pipeline to append after this one
+     * @returns A new ContextPipeline combining both pipelines' middlewares
+     */
+    abstract chain<TFinalContext extends TCrawlingContext>(
+        other: ContextPipeline<TCrawlingContext, TFinalContext>,
+    ): ContextPipeline<TContextBase, TFinalContext>;
+
     /**
      * Executes the middleware pipeline and passes the final context to a consumer function.
      *
@@ -105,6 +120,21 @@ class ContextPipelineImpl<TContextBase, TCrawlingContext extends TContextBase> e
         );
     }
 
+    chain<TFinalContext extends TCrawlingContext>(
+        other: ContextPipeline<TCrawlingContext, TFinalContext>,
+    ): ContextPipeline<TContextBase, TFinalContext> {
+        const otherMiddlewares = Array.from(
+            (other as any).middlewareChain() as Iterable<ContextMiddleware<any, any>>,
+        ).reverse();
+
+        let result: ContextPipeline<TContextBase, any> = this as any;
+        for (const middleware of otherMiddlewares) {
+            result = result.compose(middleware as any);
+        }
+
+        return result as ContextPipeline<TContextBase, TFinalContext>;
+    }
+
     private *middlewareChain() {
         let step: ContextPipelineImpl<TContextBase, TContextBase> | undefined = this as any;
 
@@ -129,7 +159,25 @@ class ContextPipelineImpl<TContextBase, TCrawlingContext extends TContextBase> e
             for (const { action, cleanup } of middlewares) {
                 try {
                     const contextExtension = await action(crawlingContext);
-                    Object.defineProperties(crawlingContext, Object.getOwnPropertyDescriptors(contextExtension));
+
+                    const extensionNames = [
+                        ...Object.getOwnPropertyNames(contextExtension),
+                        ...Object.getOwnPropertySymbols(contextExtension),
+                    ];
+
+                    for (const key of extensionNames) {
+                        try {
+                            if (Object.getOwnPropertyDescriptor(crawlingContext, key)?.configurable !== false) {
+                                Object.defineProperty(
+                                    crawlingContext,
+                                    key,
+                                    Object.getOwnPropertyDescriptor(contextExtension, key)!,
+                                );
+                            }
+                        } catch (error: any) {
+                            log.debug(`Context pipeline failed to define property ${key.toString()}:`, error);
+                        }
+                    }
 
                     if (cleanup) {
                         cleanupStack.push(cleanup);

packages/core/src/crawlers/context_pipeline.ts

@@ -1,7 +1,7 @@
 import { Transform } from 'node:stream';
 
 import type { BasicCrawlerOptions } from '@crawlee/basic';
-import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
+import { BasicCrawler } from '@crawlee/basic';
 import type { CrawlingContext, LoadedRequest, Request } from '@crawlee/core';
 import { ResponseWithUrl } from '@crawlee/http-client';
 import type { Dictionary } from '@crawlee/types';
@@ -162,19 +162,22 @@ export class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {
     constructor(options: BasicCrawlerOptions<FileDownloadCrawlingContext> = {}) {
         super({
             ...options,
-            contextPipelineBuilder: () =>
-                ContextPipeline.create<CrawlingContext>().compose({
-                    action: async (context) => this.initiateDownload(context),
-                    cleanup: async (context) => {
-                        if (!context.response.bodyUsed) {
-                            // Nobody consumed the body — cancel it so the
-                            // underlying connection can be released.
-                            await context.response.body?.cancel();
-                        }
+            contextPipelineBuilder: () => this.buildContextPipeline(),
+        });
+    }
+
+    protected override buildContextPipeline() {
+        return super.buildContextPipeline().compose({
+            action: async (context) => this.initiateDownload(context),
+            cleanup: async (context) => {
+                if (!context.response.bodyUsed) {
+                    // Nobody consumed the body — cancel it so the
+                    // underlying connection can be released.
+                    await context.response.body?.cancel();
+                }
 
-                        await (context as { [kBodyDrained]: Promise<void> })[kBodyDrained];
-                    },
-                }),
+                await (context as { [kBodyDrained]: Promise<void> })[kBodyDrained];
+            },
         });
     }
 

packages/http-crawler/src/internals/file-download.ts

@@ -410,7 +410,7 @@ export class HttpCrawler<
         }
     }
 
-    protected buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext> {
+    protected override buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext> {
         return ContextPipeline.create<CrawlingContext>()
             .compose({
                 action: this.makeHttpRequest.bind(this),

packages/http-crawler/src/internals/http-crawler.ts

@@ -192,26 +192,30 @@ export class JSDOMCrawler<
     protected virtualConsole: VirtualConsole | null = null;
 
     constructor(options: JSDOMCrawlerOptions<ContextExtension, ExtendedContext> = {}) {
-        const { runScripts = false, hideInternalConsole = false, ...httpOptions } = options;
+        const { runScripts = false, hideInternalConsole = false, contextPipelineBuilder, ...httpOptions } = options;
 
         super({
             ...httpOptions,
-            contextPipelineBuilder: () =>
-                this.buildContextPipeline()
-                    .compose({
-                        action: async (context) => await this.parseContent(context),
-                        cleanup: async (context) => {
-                            this.getVirtualConsole().off('jsdomError', this.jsdomErrorHandler);
-                            context.window?.close();
-                        },
-                    })
-                    .compose({ action: async (context) => await this.addHelpers(context) }),
+            contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
         });
 
         this.runScripts = runScripts;
         this.hideInternalConsole = hideInternalConsole;
     }
 
+    protected override buildContextPipeline() {
+        return super
+            .buildContextPipeline()
+            .compose({
+                action: async (context) => await this.parseContent(context),
+                cleanup: async (context) => {
+                    this.getVirtualConsole().off('jsdomError', this.jsdomErrorHandler);
+                    context.window?.close();
+                },
+            })
+            .compose({ action: async (context) => await this.addHelpers(context) });
+    }
+
     /**
      * Returns the currently used `VirtualConsole` instance. Can be used to listen for the JSDOM's internal console messages.
      *

packages/jsdom-crawler/src/internals/jsdom-crawler.ts

@@ -167,17 +167,23 @@ export class LinkeDOMCrawler<
     private static parser = new DOMParser();
 
     constructor(options: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext>) {
+        const { contextPipelineBuilder, ...rest } = options;
+
         super({
-            ...options,
-            contextPipelineBuilder: () =>
-                this.buildContextPipeline()
-                    .compose({
-                        action: async (context) => this.parseContent(context),
-                    })
-                    .compose({ action: async (context) => this.addHelpers(context) }),
+            ...rest,
+            contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
         });
     }
 
+    protected override buildContextPipeline() {
+        return super
+            .buildContextPipeline()
+            .compose({
+                action: async (context) => this.parseContent(context),
+            })
+            .compose({ action: async (context) => this.addHelpers(context) });
+    }
+
     private async parseContent(crawlingContext: InternalHttpCrawlingContext) {
         const isXml = crawlingContext.contentType.type.includes('xml');
         const document = LinkeDOMCrawler.parser.parseFromString(

packages/linkedom-crawler/src/internals/linkedom-crawler.ts

@@ -150,7 +150,7 @@ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary =
      */
     parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
 
-    enqueueLinks(options?: EnqueueLinksOptions): Promise<void>;
+    enqueueLinks(options?: EnqueueLinksOptions): Promise<unknown>;
 }
 
 interface AdaptiveHook
@@ -299,17 +299,11 @@ export class AdaptivePlaywrightCrawler<
 
         super({
             ...rest,
-            // Pass error handlers to the "main" crawler - we only pluck them from `rest` so that they don't go to the sub crawlers
             errorHandler,
             failedRequestHandler,
-            // Same for request handler
             requestHandler,
-            // The builder intentionally returns null so that it crashes the crawler when it tries to use this instead of one of two the specialized context pipelines
-            // (that would be a logical error in this class)
-            contextPipelineBuilder: () =>
-                null as unknown as ContextPipeline<CrawlingContext, AdaptivePlaywrightCrawlerContext>,
+            contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
         });
-
         this.individualRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
 
         this.renderingTypePredictor =
@@ -408,6 +402,34 @@ export class AdaptivePlaywrightCrawler<
         return await super._init();
     }
 
+    protected override buildContextPipeline() {
+        const errorMessage = (prop: string) =>
+            `The \`${prop}\` property is not available on the outer context pipeline of AdaptivePlaywrightCrawler - it is provided by the inner (static/browser) pipelines`;
+
+        return super.buildContextPipeline().compose({
+            action: async ({ request }) => ({
+                get request(): LoadedRequest<Request<Dictionary>> {
+                    return request as LoadedRequest<Request<Dictionary>>;
+                },
+                get response(): Response {
+                    throw new Error(errorMessage('response'));
+                },
+                get page(): Page {
+                    throw new Error(errorMessage('page'));
+                },
+                get querySelector(): AdaptivePlaywrightCrawlerContext['querySelector'] {
+                    throw new Error(errorMessage('querySelector'));
+                },
+                get waitForSelector(): AdaptivePlaywrightCrawlerContext['waitForSelector'] {
+                    throw new Error(errorMessage('waitForSelector'));
+                },
+                get parseWithCheerio(): AdaptivePlaywrightCrawlerContext['parseWithCheerio'] {
+                    throw new Error(errorMessage('parseWithCheerio'));
+                },
+            }),
+        });
+    }
+
     private async adaptCheerioContext(cheerioContext: CheerioCrawlingContext) {
         // Capture the original response to avoid infinite recursion when the getter is copied to the context
         const result = this.resultObjects.get(cheerioContext);
@@ -507,28 +529,29 @@ export class AdaptivePlaywrightCrawler<
             pushData: result.pushData,
             useState: this.allowStorageAccess(useStateFunction),
             getKeyValueStore: this.allowStorageAccess(result.getKeyValueStore),
-            enqueueLinks: async (options: SetRequired<EnqueueLinksOptions, 'urls'>) => {
-                return await this.enqueueLinks(options, context.request, result);
-            },
             log: this.createLogProxy(context.log, logs),
             registerDeferredCleanup: (cleanup: () => Promise<unknown>) => deferredCleanup.push(cleanup),
         };
 
-        const subCrawlerContext = { ...context, ...resultBoundContextHelpers };
+        const subCrawlerContext = Object.defineProperties(
+            {},
+            Object.getOwnPropertyDescriptors(context),
+        ) as typeof context;
+
+        // Mark result-bound helpers as non-configurable so they survive the sub-crawler context pipeline
+        // (which would otherwise override them with the sub-crawler's own versions, losing the result binding).
+        for (const [key, descriptor] of Object.entries(Object.getOwnPropertyDescriptors(resultBoundContextHelpers))) {
+            Object.defineProperty(subCrawlerContext, key, { ...descriptor, configurable: false });
+        }
+
         this.resultObjects.set(subCrawlerContext, result);
 
         try {
             const callAdaptiveRequestHandler = async () => {
                 if (renderingType === 'static') {
-                    await this.staticContextPipeline.call(
-                        subCrawlerContext,
-                        async (finalContext) => await this.requestHandler(finalContext),
-                    );
+                    await this.staticContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this));
                 } else if (renderingType === 'clientOnly') {
-                    await this.browserContextPipeline.call(
-                        subCrawlerContext,
-                        async (finalContext) => await this.requestHandler(finalContext),
-                    );
+                    await this.browserContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this));
                 }
             };
 

packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts

@@ -202,7 +202,7 @@ export class PlaywrightCrawler<
     constructor(options: PlaywrightCrawlerOptions<ExtendedContext> = {}) {
         ow(options, 'PlaywrightCrawlerOptions', ow.object.exactShape(PlaywrightCrawler.optionsShape));
 
-        const { launchContext = {}, headless, ...browserCrawlerOptions } = options;
+        const { launchContext = {}, headless, contextPipelineBuilder, ...browserCrawlerOptions } = options;
 
         const browserPoolOptions = {
             ...options.browserPoolOptions,
@@ -234,11 +234,14 @@ export class PlaywrightCrawler<
             ...(browserCrawlerOptions as PlaywrightCrawlerOptions<ExtendedContext>),
             launchContext,
             browserPoolOptions,
-            contextPipelineBuilder: () =>
-                this.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) }),
+            contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
         });
     }
 
+    protected override buildContextPipeline() {
+        return super.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) });
+    }
+
     protected override async _navigationHandler(
         crawlingContext: PlaywrightCrawlingContext,
         gotoOptions: DirectNavigationOptions,