diff --git a/docs/guides/configuration.mdx b/docs/guides/configuration.mdx
index 22ce4ec2e583..0e16c845d3b7 100644
--- a/docs/guides/configuration.mdx
+++ b/docs/guides/configuration.mdx
@@ -15,13 +15,13 @@ There are three ways of changing the configuration parameters:
- using the `Configuration` class
You could also combine all the above, but you should keep in mind, that the precedence for these 3 options is the following:
-***`crawlee.json`*** < ***constructor options*** < ***environment variables***.
+***constructor options*** > ***environment variables*** > ***`crawlee.json`***.
-`crawlee.json` is a baseline. The options provided in the `Configuration` constructor will override the options provided in the JSON. Environment variables will override both.
+Constructor options have the highest priority. Environment variables override `crawlee.json`. The JSON file serves as a baseline.
## `crawlee.json`
-The first option you could use for configuring Crawlee is `crawlee.json` file. The only thing you need to do is specify the `ConfigurationOptions` in the file, place the file in the root of your project, and Crawlee will use provided options as global configuration.
+The first option you could use for configuring Crawlee is `crawlee.json` file. The only thing you need to do is specify the configuration options in the file, place the file in the root of your project, and Crawlee will use provided options as global configuration. See the `Configuration` class for the full list of supported options.
```json title="crawlee.json"
{
@@ -133,24 +133,28 @@ the autoscaling feature will only use up to 2048 MB of memory.
## Configuration class
-The last option to adjust Crawlee configuration is to use the `Configuration` class in the code.
+The last option to adjust Crawlee configuration is to use the `Configuration` class in the code. Configuration is immutable — values are set via the constructor and cannot be changed afterwards.
### Global Configuration
-By default, there is a global singleton instance of `Configuration` class, it is used by the crawlers and some other classes that depend on a configurable behavior. In most cases you don't need to adjust any options there, but if needed - you can get access to it via `Configuration.getGlobalConfig()` function. Now you can easily `get` and `set` the `ConfigurationOptions`.
+By default, there is a global singleton instance of `Configuration` class, it is used by the crawlers and some other classes that depend on a configurable behavior. In most cases you don't need to adjust any options there, but if needed - you can access it via `Configuration.getGlobalConfig()`, which delegates to the global `serviceLocator` — the single source of truth for Crawlee's shared services (for example the configuration, event manager, storage client, and logger). You can also reach the same instance directly via `serviceLocator.getConfiguration()` or swap services globally with `serviceLocator.setConfiguration(...)` before any crawler is created. Configuration values are accessible directly as properties on the instance.
```js
import { CheerioCrawler, Configuration, sleep } from 'crawlee';
// Get the global configuration
const config = Configuration.getGlobalConfig();
-// Set the 'persistStateIntervalMillis' option
-// of global configuration to 10 seconds
-config.set('persistStateIntervalMillis', 10_000);
+// Access configuration values directly as properties
+console.log(config.persistStateIntervalMillis);
-// Note, that we are not passing the configuration to the crawler
-// as it's using the global configuration
-const crawler = new CheerioCrawler();
+// To use custom configuration values, create a new Configuration instance
+const configuration = new Configuration({
+ // Set the 'persistStateIntervalMillis' option to 10 seconds
+ persistStateIntervalMillis: 10_000,
+});
+
+// Pass the configuration to the crawler
+const crawler = new CheerioCrawler({ configuration });
crawler.router.addDefaultHandler(async ({ request }) => {
// For the first request we wait for 5 seconds,
@@ -170,15 +174,13 @@ crawler.router.addDefaultHandler(async ({ request }) => {
await crawler.run(['https://www.example.com/1']);
```
-This is pretty much the same example we used for showing `crawlee.json` usage,
-but now we're using the global configuration, which is the only difference.
-If you run this example - you will find the `SDK_CRAWLER_STATISTICS` file in default Key-Value store as before,
-which would show the same number of finishes requests (one) and the same crawler runtime (~10 seconds).
-This confirms that provided parameters worked: the state was persisted after 10 seconds, as it was set in the global configuration.
+If you run this example - you will find the `SDK_CRAWLER_STATISTICS` file in default Key-Value store,
+which would show the same number of finished requests (one) and the same crawler runtime (~10 seconds).
+This confirms that provided parameters worked: the state was persisted after 10 seconds, as it was set in the configuration.
:::note
-After running the same example with commented two lines of code related to `Configuration` there will be
+After running the same example without the custom configuration, there will be
no `SDK_CRAWLER_STATISTICS` file stored in the default Key-Value store:
as we did not change the `persistStateIntervalMillis`, Crawlee used the default value of 60 seconds,
and the crawler was forcefully aborted after ~15 seconds of run time before it persisted the state for the first time.
@@ -187,19 +189,19 @@ and the crawler was forcefully aborted after ~15 seconds of run time before it p
### Custom configuration
-Alternatively, you can create a custom configuration. In this case you need to pass it to the class that is going to use it, e.g. to the crawler. Let's adjust the previous example:
+You can create a custom configuration and pass it to the crawler via the `configuration` option:
```js
import { CheerioCrawler, Configuration, sleep } from 'crawlee';
// Create new configuration
-const config = new Configuration({
+const configuration = new Configuration({
// Set the 'persistStateIntervalMillis' option to 10 seconds
persistStateIntervalMillis: 10_000,
});
-// Now we need to pass the configuration to the crawler
-const crawler = new CheerioCrawler({}, config);
+// Pass the configuration to the crawler
+const crawler = new CheerioCrawler({ configuration });
crawler.router.addDefaultHandler(async ({ request }) => {
// for the first request we wait for 5 seconds,
diff --git a/docs/guides/parallel-scraping/parallel-scraper.mjs b/docs/guides/parallel-scraping/parallel-scraper.mjs
index 6bee4f4ff13a..ad6b9364ba17 100644
--- a/docs/guides/parallel-scraping/parallel-scraper.mjs
+++ b/docs/guides/parallel-scraping/parallel-scraper.mjs
@@ -73,15 +73,13 @@ if (!process.env.IN_WORKER_THREAD) {
// or a configuration option. This is just for show 😈
workerLogger.setLevel(log.LEVELS.DEBUG);
- // Disable the automatic purge on start
- // This is needed when running locally, as otherwise multiple processes will try to clear the default storage (and that will cause clashes)
- Configuration.set('purgeOnStart', false);
-
// Get the request queue
const requestQueue = await getOrInitQueue(false);
- // Configure crawlee to store the worker-specific data in a separate directory (needs to be done AFTER the queue is initialized when running locally)
+ // Disable the automatic purge on start and configure crawlee to store the worker-specific data in a separate directory
+ // (needs to be done AFTER the queue is initialized when running locally)
const config = new Configuration({
+ purgeOnStart: false,
storageClientOptions: {
localDataDirectory: `./storage/worker-${process.env.WORKER_INDEX}`,
},
diff --git a/docs/guides/parallel-scraping/parallel-scraping.mdx b/docs/guides/parallel-scraping/parallel-scraping.mdx
index 5e05532c859b..a717e81bfdac 100644
--- a/docs/guides/parallel-scraping/parallel-scraping.mdx
+++ b/docs/guides/parallel-scraping/parallel-scraping.mdx
@@ -132,22 +132,19 @@ We use this to ensure the parent process stays alive until all the worker proces
There are three steps we want to do for the worker processes:
-- ensure the default storages do **not** get purged on start, as otherwise we'd lose the queue we prepared
- get the queue that supports locking from the same location as the parent process
-- initialize a special storage for worker processes so they do not collide with each other
+- ensure the default storages do **not** get purged on start, as otherwise we'd lose the queue we prepared, and initialize a special storage for worker processes so they do not collide with each other
In order, that's what these lines do:
```javascript title="src/parallel-scraper.mjs"
-// Disable the automatic purge on start (step 1)
-// This is needed when running locally, as otherwise multiple processes will try to clear the default storage (and that will cause clashes)
-Configuration.set('purgeOnStart', false);
-
-// Get the request queue from the parent process (step 2)
+// Get the request queue from the parent process (step 1)
const requestQueue = await getOrInitQueue(false);
-// Configure crawlee to store the worker-specific data in a separate directory (needs to be done AFTER the queue is initialized when running locally) (step 3)
+// Disable the automatic purge on start and configure crawlee to store the worker-specific data
+// in a separate directory (needs to be done AFTER the queue is initialized when running locally) (step 2)
const config = new Configuration({
+ purgeOnStart: false,
storageClientOptions: {
localDataDirectory: `./storage/worker-${process.env.WORKER_INDEX}`,
},
diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.md
index 24d304107761..a5ff920265f5 100644
--- a/docs/upgrading/upgrading_v4.md
+++ b/docs/upgrading/upgrading_v4.md
@@ -55,6 +55,7 @@ The crawler following options are removed:
- `FileDownloadOptions.streamHandler` - streaming should now be handled directly in the `requestHandler` instead
- `playwrightUtils.registerUtilsToContext` and `puppeteerUtils.registerUtilsToContext` - this is now added to the context via `ContextPipeline` composition
- `puppeteerUtils.blockResources` and `puppeteerUtils.cacheResponses` (deprecated)
+- `Configuration.systemInfoV2` / `CRAWLEE_SYSTEM_INFO_V2` environment variable — the v2 behavior is now the default (see [Available resource detection](#available-resource-detection))
### The protected `BasicCrawler.crawlingContexts` map is removed
@@ -154,6 +155,40 @@ The `KeyValueStore.getPublicUrl` method is now asynchronous and reads the public
The `preNavigationHooks` option in `HttpCrawler` subclasses no longer accepts the `gotOptions` object as a second parameter. Modify the `crawlingContext` fields (e.g. `.request`) directly instead.
+## Configuration class redesign
+
+The `Configuration` class has been redesigned for v4. The main changes are:
+
+### Direct property access replaces `get()` and `set()`
+
+**Before:**
+```ts
+const config = Configuration.getGlobalConfig();
+config.set('persistStateIntervalMillis', 10_000);
+const headless = config.get('headless');
+```
+
+**After:**
+```ts
+// Configuration is now immutable — set options via the constructor
+const config = new Configuration({ persistStateIntervalMillis: 10_000 });
+const headless = config.headless;
+```
+
+The `get()` and `set()` methods are removed. Access config values directly as properties.
+Configuration instances are immutable — attempting to assign a property throws a `TypeError`.
+
+### Constructor options now take precedence over environment variables
+
+**New priority order (highest to lowest):**
+1. Constructor options
+2. Environment variables
+3. `crawlee.json`
+4. Schema defaults
+
+Previously, environment variables always won. Now `new Configuration({ headless: false })`
+works even when `CRAWLEE_HEADLESS=true` is set.
+
## Service management moved from `Configuration` to `ServiceLocator`
The service management functionality has been extracted from `Configuration` into a new `ServiceLocator` class, following the pattern established in Crawlee for Python.
@@ -166,6 +201,7 @@ The following methods and properties have been removed from `Configuration`:
- `Configuration.getEventManager()` - moved to `ServiceLocator.getEventManager()`
- `Configuration.useStorageClient()` - use `ServiceLocator.setStorageClient()` instead
- `Configuration.useEventManager()` - use `ServiceLocator.setEventManager()` instead
+- `Configuration.resetGlobalState()` - use `serviceLocator.reset()` instead
- `Configuration.storageManagers` - moved to `ServiceLocator.storageManagers`
The `EventManager` and `LocalEventManager` constructors now accept an options object for configuring event intervals (e.g. `persistStateIntervalMillis`, `systemInfoIntervalMillis`). You can also use the new `LocalEventManager.fromConfig()` factory method to create an instance with intervals derived from a `Configuration` object.
diff --git a/packages/browser-crawler/src/internals/browser-launcher.ts b/packages/browser-crawler/src/internals/browser-launcher.ts
index cdff657de995..bfac152053e4 100644
--- a/packages/browser-crawler/src/internals/browser-launcher.ts
+++ b/packages/browser-crawler/src/internals/browser-launcher.ts
@@ -189,7 +189,7 @@ export abstract class BrowserLauncher<
...this.launchOptions,
};
- if (this.config.get('disableBrowserSandbox')) {
+ if (this.config.disableBrowserSandbox) {
launchOptions.args.push('--no-sandbox');
}
@@ -209,11 +209,11 @@ export abstract class BrowserLauncher<
}
protected _getDefaultHeadlessOption(): boolean {
- return this.config.get('headless')! && !this.config.get('xvfb', false);
+ return this.config.headless && !this.config.xvfb;
}
protected _getChromeExecutablePath(): string {
- return this.config.get('chromeExecutablePath', this._getTypicalChromeExecutablePath());
+ return this.config.chromeExecutablePath ?? this._getTypicalChromeExecutablePath();
}
/**
diff --git a/packages/core/package.json b/packages/core/package.json
index 8e6778b7644c..0bef02294a6a 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -59,7 +59,6 @@
"@sapphire/async-queue": "^1.5.5",
"@vladfrangu/async_event_emitter": "^2.4.6",
"csv-stringify": "^6.5.2",
- "fs-extra": "^11.3.0",
"json5": "^2.2.3",
"minimatch": "^10.0.1",
"ow": "^2.0.0",
@@ -67,6 +66,7 @@
"tldts": "^7.0.6",
"tough-cookie": "^6.0.0",
"tslib": "^2.8.1",
- "type-fest": "^4.41.0"
+ "type-fest": "^4.41.0",
+ "zod": "^3.24.0 || ^4.0.0"
}
}
diff --git a/packages/core/src/autoscaling/snapshotter.ts b/packages/core/src/autoscaling/snapshotter.ts
index a98472ea9925..8df93b597c0e 100644
--- a/packages/core/src/autoscaling/snapshotter.ts
+++ b/packages/core/src/autoscaling/snapshotter.ts
@@ -171,21 +171,19 @@ export class Snapshotter {
* Starts capturing snapshots at configured intervals.
*/
async start(): Promise {
- const memoryMbytes = serviceLocator.getConfiguration().get('memoryMbytes', 0);
+ const memoryMbytes = serviceLocator.getConfiguration().memoryMbytes ?? 0;
if (memoryMbytes > 0) {
this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
} else {
- const containerized = serviceLocator.getConfiguration().get('containerized', await isContainerized());
+ const containerized = serviceLocator.getConfiguration().containerized ?? (await isContainerized());
const memInfo = await getMemoryInfo({
containerized,
logger: serviceLocator.getLogger(),
});
const totalBytes = memInfo.totalBytes;
- this.maxMemoryBytes = Math.ceil(
- totalBytes * serviceLocator.getConfiguration().get('availableMemoryRatio')!,
- );
+ this.maxMemoryBytes = Math.ceil(totalBytes * serviceLocator.getConfiguration().availableMemoryRatio);
this.log.debug(
`Setting max memory of this run to ${Math.round(this.maxMemoryBytes / 1024 / 1024)} MB. ` +
'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.',
diff --git a/packages/core/src/configuration.ts b/packages/core/src/configuration.ts
index 61fcf8f9a362..fa2e275a7472 100644
--- a/packages/core/src/configuration.ts
+++ b/packages/core/src/configuration.ts
@@ -2,152 +2,111 @@ import { EventEmitter } from 'node:events';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
-import type { Dictionary } from '@crawlee/types';
-import { pathExistsSync } from 'fs-extra/esm';
+import { z } from 'zod';
import { log, LogLevel } from './log.js';
import { serviceLocator } from './service_locator.js';
-import { entries } from './typedefs.js';
-export interface ConfigurationOptions {
- /**
- * Default dataset id.
- *
- * Alternative to `CRAWLEE_DEFAULT_DATASET_ID` environment variable.
- * @default 'default'
- */
- defaultDatasetId?: string;
-
- /**
- * Defines whether to purge the default storage folders before starting the crawler run.
- *
- * Alternative to `CRAWLEE_PURGE_ON_START` environment variable.
- * @default true
- */
- purgeOnStart?: boolean;
-
- /**
- * Default key-value store id.
- *
- * Alternative to `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` environment variable.
- * @default 'default'
- */
- defaultKeyValueStoreId?: string;
-
- /**
- * Default request queue id.
- *
- * Alternative to `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` environment variable.
- * @default 'default'
- */
- defaultRequestQueueId?: string;
+// Crawlee attaches many listeners to shared EventEmitters (one per crawler/session/autoscaled pool),
+// which can exceed Node's default limit of 10 and trigger spurious MaxListenersExceededWarning logs.
+// Raising the global default avoids false positives; real leaks will still manifest as unbounded growth.
+// TODO: tracked in https://github.com/apify/crawlee/issues/3615 — find a less side-effecting place for this.
+EventEmitter.defaultMaxListeners = 50;
- /**
- * Sets the ratio, defining the maximum CPU usage.
- * When the CPU usage is higher than the provided ratio, the CPU is considered overloaded.
- * @default 0.95
- */
- maxUsedCpuRatio?: number;
+// --- Field definition helpers ---
- /**
- * Sets the ratio, defining the amount of system memory that could be used by the {@apilink AutoscaledPool}.
- * When the memory usage is more than the provided ratio, the memory is considered overloaded.
- *
- * Alternative to `CRAWLEE_AVAILABLE_MEMORY_RATIO` environment variable.
- * @default 0.25
- */
- availableMemoryRatio?: number;
-
- /**
- * Sets the amount of system memory in megabytes to be used by the {@apilink AutoscaledPool}.
- * By default, the maximum memory is set to one quarter of total system memory.
- *
- * Alternative to `CRAWLEE_MEMORY_MBYTES` environment variable.
- */
- memoryMbytes?: number;
-
- /**
- * Defines the interval of emitting the `persistState` event.
- *
- * Alternative to `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` environment variable.
- * @default 60_000
- */
- persistStateIntervalMillis?: number;
-
- /**
- Defines the interval of emitting the `systemInfo` event.
- @default 1_000
- */
- systemInfoIntervalMillis?: number;
-
- /**
- * Defines the default input key, i.e. the key that is used to get the crawler input value
- * from the default {@apilink KeyValueStore} associated with the current crawler run.
- *
- * Alternative to `CRAWLEE_INPUT_KEY` environment variable.
- * @default 'INPUT'
- */
- inputKey?: string;
-
- /**
- * Defines whether web browsers launched by Crawlee will run in the headless mode.
- *
- * Alternative to `CRAWLEE_HEADLESS` environment variable.
- * @default true
- */
- headless?: boolean;
-
- /**
- * Defines whether to run X virtual framebuffer on the web browsers launched by Crawlee.
- *
- * Alternative to `CRAWLEE_XVFB` environment variable.
- * @default false
- */
- xvfb?: boolean;
-
- /**
- * Defines a path to Chrome executable.
- *
- * Alternative to `CRAWLEE_CHROME_EXECUTABLE_PATH` environment variable.
- */
- chromeExecutablePath?: string;
-
- /**
- * Defines a path to default browser executable.
- *
- * Alternative to `CRAWLEE_DEFAULT_BROWSER_PATH` environment variable.
- */
- defaultBrowserPath?: string;
-
- /**
- * Defines whether to disable browser sandbox by adding `--no-sandbox` flag to `launchOptions`.
- *
- * Alternative to `CRAWLEE_DISABLE_BROWSER_SANDBOX` environment variable.
- */
- disableBrowserSandbox?: boolean;
+export interface ConfigField {
+ schema: T;
+ envVar?: string | string[];
+}
- /**
- * Sets the log level to the given value.
- *
- * Alternative to `CRAWLEE_LOG_LEVEL` environment variable.
- * @default 'INFO'
- */
- logLevel?: LogLevel | LogLevel[keyof LogLevel];
+export function field(schema: T, envVar?: string | string[]): ConfigField {
+ return { schema, envVar };
+}
- /**
- * Defines whether the storage client used should persist the data it stores.
- *
- * Alternative to `CRAWLEE_PERSIST_STORAGE` environment variable.
- */
- persistStorage?: boolean;
+// --- Zod preprocessors ---
- /**
- * Used in place of `isContainerized()` when collecting system metrics.
- *
- * Alternative to `CRAWLEE_CONTAINERIZED` environment variable.
- */
- containerized?: boolean;
-}
+/** Zod preprocessor treating `'0'` and `'false'` as falsy. */
+export const coerceBoolean = z.preprocess((val) => {
+ if (typeof val === 'string') {
+ return !['0', 'false'].includes(val.toLowerCase());
+ }
+ return val;
+}, z.boolean());
+
+export const coerceNumber = z.preprocess((val) => {
+ if (typeof val === 'string') return Number(val);
+ return val;
+}, z.number());
+
+/** Zod schema accepting both LogLevel enum values and string names (case-insensitive). */
+const logLevelSchema = z.preprocess((val) => {
+ if (val == null) return val;
+ const s = String(val);
+ if (Number.isFinite(+s)) return +s;
+ const key = s.toUpperCase() as keyof typeof LogLevel;
+ if (key in LogLevel) return LogLevel[key];
+ return val;
+}, z.nativeEnum(LogLevel));
+
+// --- Crawlee config field definitions ---
+
+export const crawleeConfigFields = {
+ /** @default 'default' */
+ defaultDatasetId: field(z.string().default('default'), 'CRAWLEE_DEFAULT_DATASET_ID'),
+ /** @default true */
+ purgeOnStart: field(coerceBoolean.default(true), 'CRAWLEE_PURGE_ON_START'),
+ /** @default 'default' */
+ defaultKeyValueStoreId: field(z.string().default('default'), 'CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID'),
+ /** @default 'default' */
+ defaultRequestQueueId: field(z.string().default('default'), 'CRAWLEE_DEFAULT_REQUEST_QUEUE_ID'),
+ /** @default 0.95 */
+ maxUsedCpuRatio: field(coerceNumber.default(0.95)),
+ /** @default 0.25 */
+ availableMemoryRatio: field(coerceNumber.default(0.25), 'CRAWLEE_AVAILABLE_MEMORY_RATIO'),
+ memoryMbytes: field(coerceNumber.optional(), 'CRAWLEE_MEMORY_MBYTES'),
+ /** @default 60_000 */
+ persistStateIntervalMillis: field(coerceNumber.default(60_000), 'CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS'),
+ /** @default 1_000 */
+ systemInfoIntervalMillis: field(coerceNumber.default(1_000)),
+ /** @default 'INPUT' */
+ inputKey: field(z.string().default('INPUT'), 'CRAWLEE_INPUT_KEY'),
+ /** @default true */
+ headless: field(coerceBoolean.default(true), 'CRAWLEE_HEADLESS'),
+ /** @default false */
+ xvfb: field(coerceBoolean.default(false), 'CRAWLEE_XVFB'),
+ chromeExecutablePath: field(z.string().optional(), 'CRAWLEE_CHROME_EXECUTABLE_PATH'),
+ defaultBrowserPath: field(z.string().optional(), 'CRAWLEE_DEFAULT_BROWSER_PATH'),
+ /** @default false */
+ disableBrowserSandbox: field(coerceBoolean.default(false), 'CRAWLEE_DISABLE_BROWSER_SANDBOX'),
+ logLevel: field(logLevelSchema.optional(), 'CRAWLEE_LOG_LEVEL'),
+ /** @default true */
+ persistStorage: field(coerceBoolean.default(true), 'CRAWLEE_PERSIST_STORAGE'),
+ containerized: field(coerceBoolean.optional(), 'CRAWLEE_CONTAINERIZED'),
+};
+
+// --- Type utilities ---
+
+export type FieldsInput> = {
+ [K in keyof F]?: z.output;
+};
+
+export type FieldsOutput> = {
+ [K in keyof F]: z.output;
+};
+
+export type ConfigurationInput = FieldsInput;
+export type ResolvedConfigValues = FieldsOutput;
+
+/** @deprecated Use {@link ConfigurationInput} instead. */
+export type ConfigurationOptions = ConfigurationInput;
+
+// --- Configuration class ---
+
+// Declaration merging: adds resolved config properties to the Configuration type.
+// Properties are defined at runtime via Object.defineProperties in registerAccessors().
+// eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging
+export interface Configuration extends ResolvedConfigValues {}
/**
* `Configuration` is a value object holding Crawlee configuration. By default, there is a
@@ -161,13 +120,9 @@ export interface ConfigurationOptions {
*
* // Get the global configuration
* const config = Configuration.getGlobalConfig();
- * // Set the 'persistStateIntervalMillis' option
- * // of global configuration to 10 seconds
- * config.set('persistStateIntervalMillis', 10_000);
- *
- * // No need to pass the configuration to the crawler,
- * // as it's using the global configuration by default
- * const crawler = new BasicCrawler();
+ * // Access configuration values directly as properties
+ * console.log(config.headless);
+ * console.log(config.persistStateIntervalMillis);
* ```
*
* *Using custom configuration:*
@@ -177,15 +132,14 @@ export interface ConfigurationOptions {
* // Create a new configuration
* const config = new Configuration({ persistStateIntervalMillis: 30_000 });
* // Pass the configuration to the crawler
- * const crawler = new BasicCrawler({ configuration: config, ... });
+ * const crawler = new BasicCrawler({ configuration: config });
* ```
*
- * The configuration provided via environment variables always takes precedence. We can also
- * define the `crawlee.json` file in the project root directory which will serve as a baseline,
- * so the options provided in constructor will override those. In other words, the precedence is:
+ * Configuration is immutable — values are set via the constructor and cannot be changed afterwards.
+ * The priority order for resolving values is (highest to lowest):
*
* ```text
- * crawlee.json < constructor options < environment variables
+ * constructor options > environment variables > crawlee.json > schema defaults
* ```
*
* ## Supported Configuration Options
@@ -207,169 +161,134 @@ export interface ConfigurationOptions {
* Key | Environment Variable | Default Value
* ---|---|---
* `inputKey` | `CRAWLEE_INPUT_KEY` | `'INPUT'`
- * `xvfb` | `CRAWLEE_XVFB` | -
+ * `xvfb` | `CRAWLEE_XVFB` | `false`
* `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
* `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
* `disableBrowserSandbox` | `CRAWLEE_DISABLE_BROWSER_SANDBOX` | -
* `availableMemoryRatio` | `CRAWLEE_AVAILABLE_MEMORY_RATIO` | `0.25`
- * `containerized | `CRAWLEE_CONTAINERIZED | -
+ * `containerized` | `CRAWLEE_CONTAINERIZED` | -
*/
+// eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging
export class Configuration {
/**
- * Maps environment variables to config keys (e.g. `CRAWLEE_MEMORY_MBYTES` to `memoryMbytes`)
+ * Field definitions for this configuration class.
+ * Subclasses override this to register additional fields.
*/
- protected static ENV_MAP: Dictionary = {
- CRAWLEE_AVAILABLE_MEMORY_RATIO: 'availableMemoryRatio',
- CRAWLEE_PURGE_ON_START: 'purgeOnStart',
- CRAWLEE_MEMORY_MBYTES: 'memoryMbytes',
- CRAWLEE_DEFAULT_DATASET_ID: 'defaultDatasetId',
- CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID: 'defaultKeyValueStoreId',
- CRAWLEE_DEFAULT_REQUEST_QUEUE_ID: 'defaultRequestQueueId',
- CRAWLEE_INPUT_KEY: 'inputKey',
- CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS: 'persistStateIntervalMillis',
- CRAWLEE_HEADLESS: 'headless',
- CRAWLEE_XVFB: 'xvfb',
- CRAWLEE_CHROME_EXECUTABLE_PATH: 'chromeExecutablePath',
- CRAWLEE_DEFAULT_BROWSER_PATH: 'defaultBrowserPath',
- CRAWLEE_DISABLE_BROWSER_SANDBOX: 'disableBrowserSandbox',
- CRAWLEE_LOG_LEVEL: 'logLevel',
- CRAWLEE_PERSIST_STORAGE: 'persistStorage',
- CRAWLEE_CONTAINERIZED: 'containerized',
- };
-
- protected static BOOLEAN_VARS = [
- 'purgeOnStart',
- 'headless',
- 'xvfb',
- 'disableBrowserSandbox',
- 'persistStorage',
- 'containerized',
- ];
-
- protected static INTEGER_VARS = ['memoryMbytes', 'persistStateIntervalMillis', 'systemInfoIntervalMillis'];
-
- protected static COMMA_SEPARATED_LIST_VARS: string[] = [];
-
- protected static DEFAULTS: Dictionary = {
- defaultKeyValueStoreId: 'default',
- defaultDatasetId: 'default',
- defaultRequestQueueId: 'default',
- inputKey: 'INPUT',
- maxUsedCpuRatio: 0.95,
- availableMemoryRatio: 0.25,
- storageClientOptions: {},
- purgeOnStart: true,
- headless: true,
- persistStateIntervalMillis: 60_000,
- systemInfoIntervalMillis: 1_000,
- persistStorage: true,
- };
-
- protected options!: Map;
+ protected static fields: Record = crawleeConfigFields;
+
+ private resolvedValues: Record;
/**
- * Creates new `Configuration` instance with provided options. Env vars will have precedence over those.
+ * Creates new `Configuration` instance with provided options.
+ * Constructor options take precedence over environment variables, which take precedence
+ * over crawlee.json values, which take precedence over schema defaults.
*/
- constructor(options: ConfigurationOptions = {}) {
- this.buildOptions(options);
-
- // Increase the global limit for event emitter memory leak warnings.
- EventEmitter.defaultMaxListeners = 50;
-
- // set the log level to support CRAWLEE_ prefixed env var too
- const logLevel = this.get('logLevel');
-
- if (logLevel) {
- const level = Number.isFinite(+logLevel)
- ? +logLevel
- : LogLevel[String(logLevel).toUpperCase() as unknown as LogLevel];
- log.setLevel(level as LogLevel);
+ constructor(options: ConfigurationInput = {}) {
+ const fields = (this.constructor as typeof Configuration).fields;
+ const fileOptions = Configuration.loadFileOptions();
+ this.resolvedValues = Configuration.resolveAll(fields, options as Record, fileOptions);
+ this.registerAccessors();
+
+ // Set the log level
+ const logLevel = this.logLevel;
+ if (logLevel != null) {
+ log.setLevel(logLevel);
}
}
/**
- * Returns configured value. First checks the environment variables, then provided configuration,
- * fallbacks to the `defaultValue` argument if provided, otherwise uses the default value as described
- * in the above section.
+ * Returns the global configuration instance. It will respect the environment variables.
+ *
+ * Delegates to the global ServiceLocator, making it the single source of truth for service management.
*/
- get(key: T, defaultValue?: U): U {
- // prefer env vars, always iterate through the whole map as there might be duplicate env vars for the same option
- let envValue: string | undefined;
+ static getGlobalConfig(): Configuration {
+ return serviceLocator.getConfiguration();
+ }
- for (const [k, v] of entries(Configuration.ENV_MAP)) {
- if (key === v) {
- envValue = process.env[k as string];
+ /**
+ * Resolves all field values once using the priority chain:
+ * constructor options > env vars > crawlee.json > schema defaults.
+ */
+ private static resolveAll(
+ fields: Record,
+ userOptions: Record,
+ fileOptions: Record,
+ ): Record {
+ const values: Record = {};
+
+ for (const [key, fieldDef] of Object.entries(fields)) {
+ // 1. Constructor options (highest priority)
+ if (key in userOptions && userOptions[key] !== undefined) {
+ values[key] = fieldDef.schema.parse(userOptions[key]);
+ continue;
+ }
- if (envValue) {
- break;
- }
+ // 2. Environment variables
+ const envValue = Configuration.readEnvVar(fieldDef);
+ if (envValue != null) {
+ values[key] = fieldDef.schema.parse(envValue);
+ continue;
}
- }
- if (envValue != null) {
- return this._castEnvValue(key, envValue) as U;
- }
+ // 3. crawlee.json file options
+ if (key in fileOptions && fileOptions[key] !== undefined) {
+ values[key] = fieldDef.schema.parse(fileOptions[key]);
+ continue;
+ }
- // check instance level options
- if (this.options.has(key)) {
- return this.options.get(key) as U;
+ // 4. Schema default (by parsing undefined through the schema)
+ const result = fieldDef.schema.safeParse(undefined);
+ values[key] = result.success ? result.data : undefined;
}
- // fallback to defaults
- return (defaultValue ?? Configuration.DEFAULTS[key as keyof typeof Configuration.DEFAULTS] ?? envValue) as U;
+ return values;
}
- protected _castEnvValue(key: keyof ConfigurationOptions, value: number | string | boolean) {
- if (Configuration.INTEGER_VARS.includes(key)) {
- return +value;
- }
-
- if (Configuration.BOOLEAN_VARS.includes(key)) {
- // 0, false and empty string are considered falsy values
- return !['0', 'false', ''].includes(String(value).toLowerCase());
- }
-
- if (Configuration.COMMA_SEPARATED_LIST_VARS.includes(key)) {
- if (!value) return [];
- return String(value)
- .split(',')
- .map((v) => v.trim());
+ /**
+ * Registers getters (and throwing setters) on the instance for each field.
+ */
+ private registerAccessors(): void {
+ const fields = (this.constructor as typeof Configuration).fields;
+ const descriptors: PropertyDescriptorMap = {};
+
+ for (const key of Object.keys(fields)) {
+ descriptors[key] = {
+ get: () => this.resolvedValues[key],
+ set() {
+ throw new TypeError('Configuration is immutable. Pass options via the constructor instead.');
+ },
+ enumerable: true,
+ configurable: false,
+ };
}
- return value;
+ Object.defineProperties(this, descriptors);
}
/**
- * Sets value for given option. Only affects this `Configuration` instance, the value will not be propagated down to the env var.
- * To reset a value, we can omit the `value` argument or pass `undefined` there.
+ * Reads the first defined env var value for a field definition.
+ * Empty strings are treated as unset, falling through to crawlee.json or schema defaults.
+ * (Crawlee v3 coerced `''` to `false`/`0`/`''` per type — v4 drops that for consistency.)
*/
- set(key: keyof ConfigurationOptions, value?: any): void {
- this.options.set(key, value);
+ private static readEnvVar(fieldDef: ConfigField): string | undefined {
+ if (!fieldDef.envVar) return undefined;
+ const envVars = Array.isArray(fieldDef.envVar) ? fieldDef.envVar : [fieldDef.envVar];
+ for (const envVar of envVars) {
+ const value = process.env[envVar];
+ if (value != null && value !== '') return value;
+ }
+ return undefined;
}
/**
- * Returns the global configuration instance. It will respect the environment variables.
- *
- * Delegates to the global ServiceLocator, making it the single source of truth for service management.
+ * Loads config options from crawlee.json in the current working directory.
*/
- static getGlobalConfig(): Configuration {
- return serviceLocator.getConfiguration();
- }
-
- protected buildOptions(options: ConfigurationOptions) {
- // try to load configuration from crawlee.json as the baseline
- const path = join(process.cwd(), 'crawlee.json');
-
- if (pathExistsSync(path)) {
- try {
- const file = readFileSync(path);
- const optionsFromFileConfig = JSON.parse(file.toString());
- Object.assign(options, optionsFromFileConfig);
- } catch {
- // ignore
- }
+ private static loadFileOptions(): Record {
+ try {
+ const file = readFileSync(join(process.cwd(), 'crawlee.json'));
+ return JSON.parse(file.toString());
+ } catch {
+ return {};
}
-
- this.options = new Map(entries(options));
}
}
diff --git a/packages/core/src/crawlers/crawler_commons.ts b/packages/core/src/crawlers/crawler_commons.ts
index 25de68e03961..343a7265d297 100644
--- a/packages/core/src/crawlers/crawler_commons.ts
+++ b/packages/core/src/crawlers/crawler_commons.ts
@@ -280,7 +280,7 @@ export class RequestHandlerResult {
const storeId = store.id;
return {
- id: storeId ?? this.config.get('defaultKeyValueStoreId'),
+ id: storeId ?? this.config.defaultKeyValueStoreId,
name: store.name,
getValue: async (key) => this.getKeyValueStoreChangedValue(storeId, key) ?? (await store.getValue(key)),
setValue: async (key, value, options) => {
@@ -292,7 +292,7 @@ export class RequestHandlerResult {
};
private getKeyValueStoreChangedValue = (storeKey: string | undefined, key: string) => {
- const id = storeKey ?? this.config.get('defaultKeyValueStoreId') ?? 'default';
+ const id = storeKey ?? this.config.defaultKeyValueStoreId;
this._keyValueStoreChanges[id] ??= {};
return this.keyValueStoreChanges[id][key]?.changedValue ?? null;
};
@@ -303,7 +303,7 @@ export class RequestHandlerResult {
changedValue: unknown,
options?: RecordOptions,
) => {
- const id = storeKey ?? this.config.get('defaultKeyValueStoreId') ?? 'default';
+ const id = storeKey ?? this.config.defaultKeyValueStoreId;
this._keyValueStoreChanges[id] ??= {};
this._keyValueStoreChanges[id][key] = { changedValue, options };
};
diff --git a/packages/core/src/crawlers/statistics.ts b/packages/core/src/crawlers/statistics.ts
index d96d43452873..29aaae3f537c 100644
--- a/packages/core/src/crawlers/statistics.ts
+++ b/packages/core/src/crawlers/statistics.ts
@@ -331,7 +331,7 @@ export class Statistics {
this.log.debug('Persisting state', { persistStateKey: this.persistStateKey });
// use half the interval of `persistState` to avoid race conditions
- const persistStateIntervalMillis = serviceLocator.getConfiguration().get('persistStateIntervalMillis')!;
+ const persistStateIntervalMillis = serviceLocator.getConfiguration().persistStateIntervalMillis;
const timeoutSecs = persistStateIntervalMillis / 2_000;
await this.keyValueStore
.setValue(this.persistStateKey, this.toJSON(), {
diff --git a/packages/core/src/events/local_event_manager.ts b/packages/core/src/events/local_event_manager.ts
index 2056cc66af8b..213d0cc2fbd5 100644
--- a/packages/core/src/events/local_event_manager.ts
+++ b/packages/core/src/events/local_event_manager.ts
@@ -26,8 +26,8 @@ export class LocalEventManager extends EventManager {
const resolvedConfig = config ?? serviceLocator.getConfiguration();
return new LocalEventManager({
- persistStateIntervalMillis: resolvedConfig.get('persistStateIntervalMillis'),
- systemInfoIntervalMillis: resolvedConfig.get('systemInfoIntervalMillis'),
+ persistStateIntervalMillis: resolvedConfig.persistStateIntervalMillis,
+ systemInfoIntervalMillis: resolvedConfig.systemInfoIntervalMillis,
});
}
@@ -66,7 +66,7 @@ export class LocalEventManager extends EventManager {
*/
async emitSystemInfoEvent(intervalCallback: () => unknown) {
const info = await this.createSystemInfo({
- maxUsedCpuRatio: serviceLocator.getConfiguration().get('maxUsedCpuRatio'),
+ maxUsedCpuRatio: serviceLocator.getConfiguration().maxUsedCpuRatio,
});
this.events.emit(EventType.SYSTEM_INFO, info);
intervalCallback();
@@ -77,7 +77,7 @@ export class LocalEventManager extends EventManager {
*/
async isContainerizedWrapper() {
const { isContainerized } = await import('@crawlee/utils');
- return serviceLocator.getConfiguration().get('containerized', await isContainerized());
+ return serviceLocator.getConfiguration().containerized ?? (await isContainerized());
}
/**
diff --git a/packages/core/src/service_locator.ts b/packages/core/src/service_locator.ts
index 4aac074030e2..4051c75fe02f 100644
--- a/packages/core/src/service_locator.ts
+++ b/packages/core/src/service_locator.ts
@@ -220,7 +220,7 @@ export class ServiceLocator implements ServiceLocatorInterface {
}
const config = this.getConfiguration();
this.storageClient = new MemoryStorage({
- persistStorage: config.get('persistStorage'),
+ persistStorage: config.persistStorage,
logger: this.getLogger().child({ prefix: 'MemoryStorage' }),
});
}
diff --git a/packages/core/src/session_pool/session_pool.ts b/packages/core/src/session_pool/session_pool.ts
index c2cfcc042000..0a4d20b8bd5a 100644
--- a/packages/core/src/session_pool/session_pool.ts
+++ b/packages/core/src/session_pool/session_pool.ts
@@ -391,7 +391,7 @@ export class SessionPool extends EventEmitter {
});
// use half the interval of `persistState` to avoid race conditions
- const persistStateIntervalMillis = serviceLocator.getConfiguration().get('persistStateIntervalMillis')!;
+ const persistStateIntervalMillis = serviceLocator.getConfiguration().persistStateIntervalMillis;
const timeoutSecs = persistStateIntervalMillis / 2_000;
await this.keyValueStore
?.setValue(this.persistStateKey, await this.getState(), {
diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.ts
index 7324748748dc..c1c49f2539c2 100644
--- a/packages/core/src/storages/key_value_store.ts
+++ b/packages/core/src/storages/key_value_store.ts
@@ -273,7 +273,7 @@ export class KeyValueStore {
}
// use half the interval of `persistState` to avoid race conditions
- const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis')!;
+ const persistStateIntervalMillis = this.config.persistStateIntervalMillis;
const timeoutSecs = persistStateIntervalMillis / 2_000;
serviceLocator.getEventManager().on('persistState', async () => {
@@ -785,7 +785,7 @@ export class KeyValueStore {
*/
static async getInput(): Promise {
const store = await this.open();
- const inputKey = store.config.get('inputKey')!;
+ const inputKey = store.config.inputKey;
const cwd = process.cwd();
const possibleExtensions = ['', '.json', '.txt'];
diff --git a/packages/core/src/storages/storage_manager.ts b/packages/core/src/storages/storage_manager.ts
index d38724ae075a..66600013eae7 100644
--- a/packages/core/src/storages/storage_manager.ts
+++ b/packages/core/src/storages/storage_manager.ts
@@ -123,7 +123,7 @@ export class StorageManager {
}
const defaultIdConfigKey = DEFAULT_ID_CONFIG_KEYS[this.name];
- const defaultId = this.config.get(defaultIdConfigKey) as string;
+ const defaultId = this.config[defaultIdConfigKey] as string;
return { id: defaultId };
}
diff --git a/packages/core/src/storages/utils.ts b/packages/core/src/storages/utils.ts
index c07770e3e0fb..e3562c9c59b6 100644
--- a/packages/core/src/storages/utils.ts
+++ b/packages/core/src/storages/utils.ts
@@ -61,7 +61,7 @@ export async function purgeDefaultStorages(
const casted = client as StorageClient & { __purged?: boolean };
// if `onlyPurgeOnce` is true, will purge anytime this function is called, otherwise - only on start
- if (!onlyPurgeOnce || (config.get('purgeOnStart') && !casted.__purged)) {
+ if (!onlyPurgeOnce || (config.purgeOnStart && !casted.__purged)) {
casted.__purged = true;
await casted.purge?.();
}
diff --git a/packages/core/test/core/configuration.test.ts b/packages/core/test/core/configuration.test.ts
new file mode 100644
index 000000000000..a382e8a41f78
--- /dev/null
+++ b/packages/core/test/core/configuration.test.ts
@@ -0,0 +1,320 @@
+import { existsSync, unlinkSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+import { coerceBoolean, Configuration, crawleeConfigFields, field, LogLevel } from '@crawlee/core';
+
+describe('Configuration', () => {
+ const envBackup: Record = {};
+
+ function setEnv(key: string, value: string) {
+ envBackup[key] ??= process.env[key];
+ process.env[key] = value;
+ }
+
+ beforeEach(() => {
+ // Clean all CRAWLEE_ env vars so tests are isolated
+ for (const key of Object.keys(process.env)) {
+ if (key.startsWith('CRAWLEE_')) {
+ envBackup[key] ??= process.env[key];
+ delete process.env[key];
+ }
+ }
+ });
+
+ afterEach(() => {
+ // Restore env vars
+ for (const [key, val] of Object.entries(envBackup)) {
+ if (val === undefined) {
+ delete process.env[key];
+ } else {
+ process.env[key] = val;
+ }
+ }
+ });
+
+ describe('defaults', () => {
+ it('returns schema defaults when nothing is set', () => {
+ const config = new Configuration();
+ expect(config.defaultDatasetId).toBe('default');
+ expect(config.defaultKeyValueStoreId).toBe('default');
+ expect(config.defaultRequestQueueId).toBe('default');
+ expect(config.inputKey).toBe('INPUT');
+ expect(config.headless).toBe(true);
+ expect(config.xvfb).toBe(false);
+ expect(config.disableBrowserSandbox).toBe(false);
+ expect(config.purgeOnStart).toBe(true);
+ expect(config.persistStorage).toBe(true);
+ expect(config.maxUsedCpuRatio).toBe(0.95);
+ expect(config.availableMemoryRatio).toBe(0.25);
+ expect(config.persistStateIntervalMillis).toBe(60_000);
+ expect(config.systemInfoIntervalMillis).toBe(1_000);
+ });
+
+ // `containerized` and `logLevel` are intentionally optional — `undefined` is a meaningful signal:
+ // - `containerized`: consumers fall back to runtime detection via `isContainerized()` using
+ // `config.containerized ?? (await isContainerized())`; defaulting to `false` would disable
+ // auto-detection.
+ // - `logLevel`: the log system already has its own default (INFO). Defaulting here would
+ // cause every `new Configuration()` to override any previously-configured log level.
+ it('returns undefined for optional fields with no default', () => {
+ const config = new Configuration();
+ expect(config.memoryMbytes).toBeUndefined();
+ expect(config.chromeExecutablePath).toBeUndefined();
+ expect(config.defaultBrowserPath).toBeUndefined();
+ expect(config.containerized).toBeUndefined();
+ expect(config.logLevel).toBeUndefined();
+ });
+ });
+
+ describe('priority: constructor > env > crawlee.json > defaults', () => {
+ it('constructor options override env vars', () => {
+ setEnv('CRAWLEE_HEADLESS', 'true');
+ const config = new Configuration({ headless: false });
+ expect(config.headless).toBe(false);
+ });
+
+ it('env vars override defaults', () => {
+ setEnv('CRAWLEE_HEADLESS', 'false');
+ const config = new Configuration();
+ expect(config.headless).toBe(false);
+ });
+
+ it('constructor options override defaults', () => {
+ const config = new Configuration({ persistStateIntervalMillis: 30_000 });
+ expect(config.persistStateIntervalMillis).toBe(30_000);
+ });
+
+ it('constructor options override env vars for string fields', () => {
+ setEnv('CRAWLEE_DEFAULT_DATASET_ID', 'from-env');
+ const config = new Configuration({ defaultDatasetId: 'from-constructor' });
+ expect(config.defaultDatasetId).toBe('from-constructor');
+ });
+
+ it('constructor options override env vars for number fields', () => {
+ setEnv('CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS', '99999');
+ const config = new Configuration({ persistStateIntervalMillis: 10_000 });
+ expect(config.persistStateIntervalMillis).toBe(10_000);
+ });
+
+ describe('with crawlee.json', () => {
+ const crawleeJsonPath = join(process.cwd(), 'crawlee.json');
+
+ afterEach(() => {
+ if (existsSync(crawleeJsonPath)) {
+ unlinkSync(crawleeJsonPath);
+ }
+ });
+
+ it('crawlee.json overrides defaults', () => {
+ writeFileSync(crawleeJsonPath, JSON.stringify({ persistStateIntervalMillis: 5_000 }));
+ const config = new Configuration();
+ expect(config.persistStateIntervalMillis).toBe(5_000);
+ });
+
+ it('env vars override crawlee.json', () => {
+ writeFileSync(crawleeJsonPath, JSON.stringify({ persistStateIntervalMillis: 5_000 }));
+ setEnv('CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS', '7000');
+ const config = new Configuration();
+ expect(config.persistStateIntervalMillis).toBe(7_000);
+ });
+
+ it('constructor options override crawlee.json and env vars', () => {
+ writeFileSync(crawleeJsonPath, JSON.stringify({ persistStateIntervalMillis: 5_000 }));
+ setEnv('CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS', '7000');
+ const config = new Configuration({ persistStateIntervalMillis: 10_000 });
+ expect(config.persistStateIntervalMillis).toBe(10_000);
+ });
+ });
+ });
+
+ describe('env var coercion', () => {
+ it('coerces boolean env vars', () => {
+ setEnv('CRAWLEE_HEADLESS', 'false');
+ expect(new Configuration().headless).toBe(false);
+
+ setEnv('CRAWLEE_HEADLESS', '0');
+ expect(new Configuration().headless).toBe(false);
+
+ setEnv('CRAWLEE_HEADLESS', 'true');
+ expect(new Configuration().headless).toBe(true);
+
+ setEnv('CRAWLEE_HEADLESS', '1');
+ expect(new Configuration().headless).toBe(true);
+ });
+
+ it('treats empty-string env var as unset across all field types', () => {
+ // Empty boolean env var falls through to default (true), not coerced to false
+ setEnv('CRAWLEE_HEADLESS', '');
+ expect(new Configuration().headless).toBe(true);
+
+ // Empty number env var falls through to default (60_000), not coerced to 0
+ setEnv('CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS', '');
+ expect(new Configuration().persistStateIntervalMillis).toBe(60_000);
+
+ // Empty string env var falls through to default ('default'), not coerced to ''
+ setEnv('CRAWLEE_DEFAULT_DATASET_ID', '');
+ expect(new Configuration().defaultDatasetId).toBe('default');
+
+ // Optional fields with no default stay undefined
+ setEnv('CRAWLEE_MEMORY_MBYTES', '');
+ expect(new Configuration().memoryMbytes).toBeUndefined();
+ });
+
+ it('coerces number env vars', () => {
+ setEnv('CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS', '30000');
+ expect(new Configuration().persistStateIntervalMillis).toBe(30_000);
+
+ setEnv('CRAWLEE_MEMORY_MBYTES', '512');
+ expect(new Configuration().memoryMbytes).toBe(512);
+ });
+
+ it('coerces log level from string name', () => {
+ setEnv('CRAWLEE_LOG_LEVEL', 'DEBUG');
+ expect(new Configuration().logLevel).toBe(LogLevel.DEBUG);
+ });
+
+ it('coerces log level from numeric string', () => {
+ setEnv('CRAWLEE_LOG_LEVEL', '5');
+ expect(new Configuration().logLevel).toBe(LogLevel.DEBUG);
+ });
+
+ it('coerces log level case-insensitively', () => {
+ setEnv('CRAWLEE_LOG_LEVEL', 'info');
+ expect(new Configuration().logLevel).toBe(LogLevel.INFO);
+ });
+ });
+
+ describe('direct property access', () => {
+ it('exposes resolved values as instance properties', () => {
+ const config = new Configuration({
+ headless: false,
+ defaultDatasetId: 'my-dataset',
+ persistStateIntervalMillis: 5_000,
+ });
+ expect(config.headless).toBe(false);
+ expect(config.defaultDatasetId).toBe('my-dataset');
+ expect(config.persistStateIntervalMillis).toBe(5_000);
+ });
+ });
+
+ describe('immutability', () => {
+ it('throws TypeError when assigning to a config property', () => {
+ const config = new Configuration();
+ expect(() => {
+ (config as any).headless = false;
+ }).toThrow(TypeError);
+ expect(() => {
+ (config as any).headless = false;
+ }).toThrow('Configuration is immutable');
+ });
+ });
+
+ describe('schema validation of constructor options', () => {
+ it('validates and coerces constructor options through the schema', () => {
+ // Pass a string where a number is expected — schema should coerce it
+ const config = new Configuration({ memoryMbytes: '512' as any });
+ expect(config.memoryMbytes).toBe(512);
+ expect(typeof config.memoryMbytes).toBe('number');
+ });
+
+ it('validates boolean constructor options through the schema', () => {
+ // Pass a string where a boolean is expected — schema should coerce it
+ const config = new Configuration({ headless: '0' as any });
+ expect(config.headless).toBe(false);
+ });
+
+ it('rejects invalid constructor options via schema', () => {
+ // Pass a completely invalid value — schema should throw
+ expect(() => {
+ const config = new Configuration({ memoryMbytes: 'not-a-number' as any });
+ // Access the property to trigger resolution
+ void config.memoryMbytes;
+ }).toThrow();
+ });
+ });
+
+ describe('crawlee.json file loading', () => {
+ const crawleeJsonPath = join(process.cwd(), 'crawlee.json');
+ let fileCreated = false;
+
+ afterEach(() => {
+ if (fileCreated) {
+ try {
+ unlinkSync(crawleeJsonPath);
+ } catch {
+ /* ignore */
+ }
+ fileCreated = false;
+ }
+ });
+
+ it('loads values from crawlee.json', () => {
+ writeFileSync(crawleeJsonPath, JSON.stringify({ defaultDatasetId: 'from-file' }));
+ fileCreated = true;
+
+ const config = new Configuration();
+ expect(config.defaultDatasetId).toBe('from-file');
+ });
+
+ it('constructor options override crawlee.json', () => {
+ writeFileSync(crawleeJsonPath, JSON.stringify({ defaultDatasetId: 'from-file' }));
+ fileCreated = true;
+
+ const config = new Configuration({ defaultDatasetId: 'from-constructor' });
+ expect(config.defaultDatasetId).toBe('from-constructor');
+ });
+
+ it('env vars override crawlee.json', () => {
+ writeFileSync(crawleeJsonPath, JSON.stringify({ headless: false }));
+ fileCreated = true;
+ setEnv('CRAWLEE_HEADLESS', 'true');
+
+ const config = new Configuration();
+ expect(config.headless).toBe(true);
+ });
+
+ it('validates and coerces crawlee.json values through the schema', () => {
+ // JSON numbers are already numbers, but string values should be coerced
+ writeFileSync(crawleeJsonPath, JSON.stringify({ memoryMbytes: '256' }));
+ fileCreated = true;
+
+ const config = new Configuration();
+ expect(config.memoryMbytes).toBe(256);
+ expect(typeof config.memoryMbytes).toBe('number');
+ });
+
+ it('handles missing crawlee.json gracefully', () => {
+ // No file created — should fall through to defaults
+ const config = new Configuration();
+ expect(config.defaultDatasetId).toBe('default');
+ });
+
+ it('handles malformed crawlee.json gracefully', () => {
+ writeFileSync(crawleeJsonPath, 'not valid json{{{');
+ fileCreated = true;
+
+ const config = new Configuration();
+ expect(config.defaultDatasetId).toBe('default');
+ });
+ });
+
+ describe('subclass field registration', () => {
+ it('subclass can define additional fields via static fields override', () => {
+ const extendedFields = {
+ ...crawleeConfigFields,
+ customFlag: field(coerceBoolean.default(false), 'MY_CUSTOM_FLAG'),
+ };
+
+ class ExtendedConfig extends Configuration {
+ protected static override fields = extendedFields;
+ }
+
+ const config = new ExtendedConfig();
+ expect((config as any).customFlag).toBe(false);
+
+ setEnv('MY_CUSTOM_FLAG', 'true');
+ const config2 = new ExtendedConfig();
+ expect((config2 as any).customFlag).toBe(true);
+ });
+ });
+});
diff --git a/packages/core/test/enqueue_links/add-enqueue-strategy-to-requests.ts b/packages/core/test/enqueue_links/add-enqueue-strategy-to-requests.ts
index 553a1d1f8391..bea08e92c7bf 100644
--- a/packages/core/test/enqueue_links/add-enqueue-strategy-to-requests.ts
+++ b/packages/core/test/enqueue_links/add-enqueue-strategy-to-requests.ts
@@ -1,10 +1,10 @@
import { load } from 'cheerio';
import type { Source } from 'crawlee';
-import { cheerioCrawlerEnqueueLinks, Configuration, EnqueueStrategy, RequestQueue } from 'crawlee';
+import { cheerioCrawlerEnqueueLinks, EnqueueStrategy, RequestQueue, serviceLocator } from 'crawlee';
import log from '@apify/log';
-const apifyClient = Configuration.getStorageClient();
+const apifyClient = serviceLocator.getStorageClient();
const HTML = `
diff --git a/packages/playwright-crawler/src/internals/playwright-launcher.ts b/packages/playwright-crawler/src/internals/playwright-launcher.ts
index a8bf13c86c50..bdb538c2b274 100644
--- a/packages/playwright-crawler/src/internals/playwright-launcher.ts
+++ b/packages/playwright-crawler/src/internals/playwright-launcher.ts
@@ -123,7 +123,7 @@ export class PlaywrightLauncher extends BrowserLauncher {
* @ignore
*/
function getDefaultExecutablePath(launchContext: PlaywrightLaunchContext, config: Configuration): string | undefined {
- const pathFromPlaywrightImage = config.get('defaultBrowserPath');
+ const pathFromPlaywrightImage = config.defaultBrowserPath;
const { launchOptions = {} } = launchContext;
if (launchOptions.executablePath) {
diff --git a/packages/stagehand-crawler/src/internals/stagehand-launcher.ts b/packages/stagehand-crawler/src/internals/stagehand-launcher.ts
index fcce8a461a61..c7c08bbec2bc 100644
--- a/packages/stagehand-crawler/src/internals/stagehand-launcher.ts
+++ b/packages/stagehand-crawler/src/internals/stagehand-launcher.ts
@@ -131,7 +131,7 @@ export class StagehandLauncher extends BrowserLauncher