feat: decouple log configuration from apifylog (#3399)

Adds a pluggable logger abstraction so users can swap @apify/log for any                                                                    logger (Winston, Pino, Bunyan, etc.).                                                                                                       
                                                                                                                                              
What's added:                                                                                                                               
  - CrawleeLogger interface — the contract any logger must satisfy (13
    methods: error, warning, info, debug, exception, softFail,
    warningOnce, deprecated, perf, logWithLevel, child, getOptions,
    setOptions)
  - BaseCrawleeLogger abstract class — implement logWithLevel() +
    createChild(), get everything else for free
  - ServiceLocator.getLogger() / setLogger() — injection point; defaults
    to @apify/log

Author: l2ysho

docs/examples/file_download_stream.ts

@@ -1,9 +1,9 @@
 import { pipeline, Transform } from 'stream';
 
-import { FileDownload, type Log } from 'crawlee';
+import { FileDownload, type CrawleeLogger } from 'crawlee';
 
 // A sample Transform stream logging the download progress.
-function createProgressTracker({ url, log, totalBytes }: { url: URL; log: Log; totalBytes: number }) {
+function createProgressTracker({ url, log, totalBytes }: { url: URL; log: CrawleeLogger; totalBytes: number }) {
     let downloadedBytes = 0;
 
     return new Transform({

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -6,6 +6,7 @@ import type {
     AddRequestsBatchedResult,
     AutoscaledPoolOptions,
     Configuration,
+    CrawleeLogger,
     CrawlingContext,
     DatasetExportOptions,
     EnqueueLinksOptions,
@@ -40,6 +41,7 @@ import {
     EnqueueStrategy,
     EventType,
     KeyValueStore,
+    LogLevel,
     mergeCookies,
     NonRetryableError,
     purgeDefaultStorages,
@@ -77,8 +79,6 @@ import { getDomain } from 'tldts';
 import type { ReadonlyDeep, SetRequired } from 'type-fest';
 
 import { LruCache } from '@apify/datastructures';
-import type { Log } from '@apify/log';
-import defaultLog, { LogLevel } from '@apify/log';
 import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
 import { cryptoRandomObjectId } from '@apify/utilities';
 
@@ -370,9 +370,6 @@ export interface BasicCrawlerOptions<
      */
     onSkippedRequest?: SkippedRequestCallback;
 
-    /** @internal */
-    log?: Log;
-
     /**
      * Enables experimental features of Crawlee, which can alter the behavior of the crawler.
      * WARNING: these options are not guaranteed to be stable and may change or be removed at any time.
@@ -415,6 +412,12 @@ export interface BasicCrawlerOptions<
      */
     eventManager?: EventManager;
 
+    /**
+     * Custom logger to use for this crawler.
+     * If provided, the crawler will use its own ServiceLocator instance instead of the global one.
+     */
+    logger?: CrawleeLogger;
+
     /**
      * A unique identifier for the crawler instance. This ID is used to isolate the state returned by
      * {@apilink BasicCrawler.useState|`crawler.useState()`} from other crawler instances.
@@ -586,7 +589,12 @@ export class BasicCrawler<
     running = false;
     hasFinishedBefore = false;
 
-    readonly log: Log;
+    #log!: CrawleeLogger;
+
+    get log(): CrawleeLogger {
+        return this.#log;
+    }
+
     protected requestHandler!: RequestHandler<ExtendedContext>;
     protected errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
     protected failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
@@ -651,6 +659,7 @@ export class BasicCrawler<
         configuration: ow.optional.object,
         storageClient: ow.optional.object,
         eventManager: ow.optional.object,
+        logger: ow.optional.object,
 
         // AutoscaledPool shorthands
         minConcurrency: ow.optional.number,
@@ -659,7 +668,6 @@ export class BasicCrawler<
         keepAlive: ow.optional.boolean,
 
         // internal
-        log: ow.optional.object,
         experiments: ow.optional.object,
 
         statisticsOptions: ow.optional.object,
@@ -695,6 +703,7 @@ export class BasicCrawler<
             configuration,
             storageClient,
             eventManager,
+            logger,
 
             // AutoscaledPool shorthands
             minConcurrency,
@@ -714,7 +723,6 @@ export class BasicCrawler<
             httpClient,
 
             // internal
-            log = defaultLog.child({ prefix: this.constructor.name }),
             experiments = {},
 
             id,
@@ -730,15 +738,18 @@ export class BasicCrawler<
         if (
             storageClient ||
             eventManager ||
+            logger ||
             (configuration !== undefined && configuration !== serviceLocator.getConfiguration())
         ) {
-            const scopedServiceLocator = new ServiceLocator(configuration, eventManager, storageClient);
+            const scopedServiceLocator = new ServiceLocator(configuration, eventManager, storageClient, logger);
             serviceLocatorScope = bindMethodsToServiceLocator(scopedServiceLocator, this);
         }
 
         try {
             serviceLocatorScope.enterScope();
 
+            this.#log = serviceLocator.getLogger().child({ prefix: this.constructor.name });
+
             // Store whether the user explicitly provided an ID
             this.hasExplicitId = id !== undefined;
             // Store the user-provided ID, or generate a unique one for tracking purposes (not for state key)
@@ -793,7 +804,6 @@ export class BasicCrawler<
 
             this.httpClient = httpClient ?? new GotScrapingHttpClient();
             this.proxyConfiguration = proxyConfiguration;
-            this.log = log;
             this.statusMessageLoggingInterval = statusMessageLoggingInterval;
             this.statusMessageCallback = statusMessageCallback as StatusMessageCallback;
             this.domainAccessedTime = new Map();
@@ -833,19 +843,19 @@ export class BasicCrawler<
             this.sameDomainDelayMillis = sameDomainDelaySecs * 1000;
             this.maxSessionRotations = maxSessionRotations;
             this.stats = new Statistics({
-                logMessage: `${log.getOptions().prefix} request statistics:`,
-                log,
+                logMessage: `${this.constructor.name} request statistics:`,
+                log: this.log,
                 ...(this.hasExplicitId ? { id: this.crawlerId } : {}),
                 ...statisticsOptions,
             });
             this.sessionPoolOptions = {
                 ...sessionPoolOptions,
-                log,
+                log: this.log,
             };
             if (this.retryOnBlocked) {
                 this.sessionPoolOptions.blockedStatusCodes = sessionPoolOptions.blockedStatusCodes ?? [];
                 if (this.sessionPoolOptions.blockedStatusCodes.length !== 0) {
-                    log.warning(
+                    this.log.warning(
                         `Both 'blockedStatusCodes' and 'retryOnBlocked' are set. Please note that the 'retryOnBlocked' feature might not work as expected.`,
                     );
                 }
@@ -854,7 +864,7 @@ export class BasicCrawler<
 
             const maxSignedInteger = 2 ** 31 - 1;
             if (this.requestHandlerTimeoutMillis > maxSignedInteger) {
-                log.warning(
+                this.log.warning(
                     `requestHandlerTimeoutMillis ${this.requestHandlerTimeoutMillis}` +
                         ` does not fit a signed 32-bit integer. Limiting the value to ${maxSignedInteger}`,
                 );
@@ -885,7 +895,7 @@ export class BasicCrawler<
                 isTaskReadyFunction: async () => {
                     if (isMaxPagesExceeded()) {
                         if (this.shouldLogMaxProcessedRequestsExceeded) {
-                            log.info(
+                            this.log.info(
                                 'Crawler reached the maxRequestsPerCrawl limit of ' +
                                     `${this.maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`,
                             );
@@ -898,7 +908,7 @@ export class BasicCrawler<
                 },
                 isFinishedFunction: async () => {
                     if (isMaxPagesExceeded()) {
-                        log.info(
+                        this.log.info(
                             `Earlier, the crawler reached the maxRequestsPerCrawl limit of ${this.maxRequestsPerCrawl} requests ` +
                                 'and all requests that were in progress at that time have now finished. ' +
                                 `In total, the crawler processed ${this.handledRequestsCount} requests and will shut down.`,
@@ -914,12 +924,12 @@ export class BasicCrawler<
                         const reason = isFinishedFunction
                             ? "Crawler's custom isFinishedFunction() returned true, the crawler will shut down."
                             : 'All requests from the queue have been processed, the crawler will shut down.';
-                        log.info(reason);
+                        this.log.info(reason);
                     }
 
                     return isFinished;
                 },
-                log,
+                log: this.log,
             };
 
             this.autoscaledPoolOptions = { ...autoscaledPoolOptions, ...basicCrawlerAutoscaledPoolConfiguration };
@@ -944,7 +954,7 @@ export class BasicCrawler<
     async setStatusMessage(message: string, options: SetStatusMessageOptions = {}) {
         const data =
             options.isStatusMessageTerminal != null ? { terminal: options.isStatusMessageTerminal } : undefined;
-        this.log.internal(LogLevel[(options.level as 'DEBUG') ?? 'DEBUG'], message, data);
+        this.log.logWithLevel(LogLevel[(options.level as 'DEBUG') ?? 'DEBUG'], message, data);
 
         const client = serviceLocator.getStorageClient();
 
@@ -1099,7 +1109,7 @@ export class BasicCrawler<
                 retryHistogram: this.stats.requestRetryHistogram,
                 ...finalStats,
             };
-            this.log.info('Final request statistics:', stats);
+            this.log.info('Final request statistics:', stats as unknown as Record<string, unknown>);
 
             if (this.stats.errorTracker.total !== 0) {
                 const prettify = ([count, info]: [number, string[]]) =>
@@ -1193,12 +1203,14 @@ export class BasicCrawler<
         BasicCrawler.useStateCrawlerIds.add(this.crawlerId);
 
         if (BasicCrawler.useStateCrawlerIds.size > 1) {
-            defaultLog.warningOnce(
-                'Multiple crawler instances are calling useState() without an explicit `id` option. \n' +
-                    'This means they will share the same state object, which is likely unintended. \n' +
-                    'To fix this, provide a unique `id` option to each crawler instance. \n' +
-                    'Example: new BasicCrawler({ id: "my-crawler-1", ... })',
-            );
+            serviceLocator
+                .getLogger()
+                .warningOnce(
+                    'Multiple crawler instances are calling useState() without an explicit `id` option. \n' +
+                        'This means they will share the same state object, which is likely unintended. \n' +
+                        'To fix this, provide a unique `id` option to each crawler instance. \n' +
+                        'Example: new BasicCrawler({ id: "my-crawler-1", ... })',
+                );
         }
 
         return kvs.getAutoSavedValue<State>(BasicCrawler.CRAWLEE_STATE_KEY, defaultValue);

packages/browser-pool/src/abstract-classes/browser-controller.ts

@@ -1,3 +1,4 @@
+import { type CrawleeLogger, serviceLocator } from '@crawlee/core';
 import type { Cookie, Dictionary } from '@crawlee/types';
 import { nanoid } from 'nanoid';
 import { TypedEmitter } from 'tiny-typed-emitter';
@@ -6,7 +7,6 @@ import { tryCancel } from '@apify/timeout';
 
 import { BROWSER_CONTROLLER_EVENTS } from '../events.js';
 import type { LaunchContext } from '../launch-context.js';
-import { log } from '../logger.js';
 import type { UnwrapPromise } from '../utils.js';
 import type { BrowserPlugin, CommonBrowser, CommonLibrary } from './browser-plugin.js';
 
@@ -40,6 +40,7 @@ export abstract class BrowserController<
     NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,
 > extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>> {
     id = nanoid();
+    protected log!: CrawleeLogger;
 
     /**
      * The `BrowserPlugin` instance used to launch the browser.
@@ -90,6 +91,7 @@ export abstract class BrowserController<
 
     constructor(browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>) {
         super();
+        this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
         this.browserPlugin = browserPlugin;
     }
 
@@ -136,14 +138,14 @@ export abstract class BrowserController<
             // TODO: shouldn't this go in a finally instead?
             this.isActive = false;
         } catch (error) {
-            log.debug(`Could not close browser.\nCause: ${(error as Error).message}`, { id: this.id });
+            this.log.debug(`Could not close browser.\nCause: ${(error as Error).message}`, { id: this.id });
         }
 
         this.emit(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, this);
 
         setTimeout(() => {
             this._kill().catch((err) => {
-                log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
+                this.log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
             });
         }, PROCESS_KILL_TIMEOUT_MILLIS);
     }

packages/browser-pool/src/abstract-classes/browser-plugin.ts

@@ -1,4 +1,4 @@
-import { CriticalError } from '@crawlee/core';
+import { type CrawleeLogger, CriticalError, serviceLocator } from '@crawlee/core';
 import type { Dictionary } from '@crawlee/types';
 import merge from 'lodash.merge';
 
@@ -105,6 +105,7 @@ export abstract class BrowserPlugin<
     NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,
 > {
     name = this.constructor.name;
+    protected log!: CrawleeLogger;
     library: Library;
     launchOptions: LibraryOptions;
     proxyUrl?: string;
@@ -121,6 +122,7 @@ export abstract class BrowserPlugin<
             browserPerProxy = false,
         } = options;
 
+        this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
         this.library = library;
         this.launchOptions = launchOptions;
         this.proxyUrl = proxyUrl && new URL(proxyUrl).href.slice(0, -1);

packages/browser-pool/src/browser-pool.ts

@@ -1,4 +1,4 @@
-import type { TieredProxy } from '@crawlee/core';
+import { type CrawleeLogger, serviceLocator, type TieredProxy } from '@crawlee/core';
 import type { BrowserFingerprintWithHeaders } from 'fingerprint-generator';
 import { FingerprintGenerator } from 'fingerprint-generator';
 import { FingerprintInjector } from 'fingerprint-injector';
@@ -20,7 +20,6 @@ import {
 } from './fingerprinting/hooks.js';
 import type { FingerprintGeneratorOptions } from './fingerprinting/types.js';
 import type { LaunchContext } from './launch-context.js';
-import { log } from './logger.js';
 import type { InferBrowserPluginArray, UnwrapPromise } from './utils.js';
 
 const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
@@ -334,9 +333,11 @@ export class BrowserPool<
     private browserRetireInterval?: NodeJS.Timeout;
 
     private limiter = pLimit(1);
+    private log!: CrawleeLogger;
 
     constructor(options: Options & BrowserPoolHooks<BrowserControllerReturn, LaunchContextReturn, PageReturn>) {
         super();
+        this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
 
         this.browserKillerInterval!.unref();
 
@@ -708,7 +709,7 @@ export class BrowserPool<
             throw err;
         }
 
-        log.debug('Launched new browser.', { id: browserController.id });
+        this.log.debug('Launched new browser.', { id: browserController.id });
         browserController.proxyTier = proxyTier;
         browserController.proxyUrl = proxyUrl;
 
@@ -719,7 +720,7 @@ export class BrowserPool<
         } catch (err) {
             this.startingBrowserControllers.delete(browserController);
             browserController.close().catch((closeErr) => {
-                log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
+                this.log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
                     id: browserController.id,
                 });
             });
@@ -774,15 +775,15 @@ export class BrowserPool<
 
             if (isBrowserIdle || isBrowserEmpty) {
                 const { id } = controller;
-                log.debug('Closing retired browser.', { id });
+                this.log.debug('Closing retired browser.', { id });
                 await controller.close();
                 this.retiredBrowserControllers.delete(controller);
                 closedBrowserIds.push(id);
             }
         }
 
         if (closedBrowserIds.length) {
-            log.debug('Closed retired browsers.', {
+            this.log.debug('Closed retired browsers.', {
                 count: closedBrowserIds.length,
                 closedBrowserIds,
             });
@@ -798,7 +799,7 @@ export class BrowserPool<
             await this._executeHooks(this.prePageCloseHooks, page, browserController);
 
             await originalPageClose.apply(page, args).catch((err: Error) => {
-                log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
+                this.log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
             });
 
             await this._executeHooks(this.postPageCloseHooks, pageId, browserController);
@@ -821,7 +822,7 @@ export class BrowserPool<
             // Run this with a delay, otherwise page.close()
             // might fail with "Protocol error (Target.closeTarget): Target closed."
             setTimeout(() => {
-                log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
+                this.log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
                 void browserController.close().finally(() => {
                     this.retiredBrowserControllers.delete(browserController);
                 });