feat!: replace globs/regexps/pseudoUrls with unified include/exclude API

Align `EnqueueLinksOptions` with crawlee-python (#3409):

- Replace `globs`, `regexps`, `pseudoUrls` options with `include`/`exclude` accepting `UrlPatternInput[]`
- Strip request options (label, method, payload, userData, headers) from pattern objects — patterns are pure URL matchers
- `transformRequestFunction` is now the only way to customize per-request options, runs after all filtering
- Add `'skip'` and `'unchanged'` return values to `RequestTransform` (aligned with Python's `RequestTransformAction`)
- Apply same changes to `enqueueLinksByClickingElements` (Playwright + Puppeteer) and `SitemapRequestList`
- Remove `@apify/pseudo_url` dependency and `PseudoUrl` re-export
- Update all templates from `globs` to `include`

BREAKING CHANGE: `globs`, `regexps`, and `pseudoUrls` options removed. Use `include`/`exclude` instead.

Author: l2ysho

packages/core/package.json

@@ -50,7 +50,6 @@
         "@apify/consts": "^2.41.0",
         "@apify/datastructures": "^2.0.3",
         "@apify/log": "^2.5.18",
-        "@apify/pseudo_url": "^2.0.59",
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
         "@crawlee/memory-storage": "4.0.0",

packages/core/src/crawlers/crawler_commons.ts

@@ -54,8 +54,7 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
      * This function automatically finds and enqueues links from the current page, adding them to the {@apilink RequestQueue}
      * currently used by the crawler.
      *
-     * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
-     * and override settings of the enqueued {@apilink Request} objects.
+     * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
      *
      * Check out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example
      * for more details regarding its usage.
@@ -65,7 +64,7 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
      * ```ts
      * async requestHandler({ enqueueLinks }) {
      *     await enqueueLinks({
-     *       globs: [
+     *       include: [
      *           'https://www.example.com/handbags/*',
      *       ],
      *     });
@@ -112,8 +111,7 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
      * This function automatically finds and enqueues links from the current page, adding them to the {@apilink RequestQueue}
      * currently used by the crawler.
      *
-     * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
-     * and override settings of the enqueued {@apilink Request} objects.
+     * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
      *
      * Check out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example
      * for more details regarding its usage.
@@ -123,7 +121,7 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
      * ```ts
      * async requestHandler({ enqueueLinks }) {
      *     await enqueueLinks({
-     *       globs: [
+     *       include: [
      *           'https://www.example.com/handbags/*',
      *       ],
      *     });

packages/core/src/enqueue_links/enqueue_links.ts

@@ -6,27 +6,22 @@ import type { SetRequired } from 'type-fest';
 
 import type { RequestOptions } from '../request.js';
 import { Request } from '../request.js';
-import { serviceLocator } from '../service_locator.js';
 import type {
     AddRequestsBatchedOptions,
     AddRequestsBatchedResult,
     RequestProvider,
     RequestQueueOperationOptions,
 } from '../storages/request_provider.js';
 import type {
-    GlobInput,
-    PseudoUrlInput,
-    RegExpInput,
     RequestTransform,
     SkippedRequestCallback,
     SkippedRequestReason,
+    UrlPatternInput,
     UrlPatternObject,
 } from './shared.js';
 import {
     applyRequestTransform,
-    constructGlobObjectsFromGlobs,
-    constructRegExpObjectsFromPseudoUrls,
-    constructRegExpObjectsFromRegExps,
+    constructUrlPatternObjects,
     createRequestOptions,
     filterRequestOptionsByPatterns,
 } from './shared.js';
@@ -50,8 +45,7 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
     /**
      * Sets {@apilink Request.label} for newly enqueued requests.
      *
-     * This option has the lowest priority and can be overwritten by request options
-     * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as by `transformRequestFunction`.
+     * Can be overwritten by `transformRequestFunction`.
      */
     label?: string;
 
@@ -68,65 +62,30 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
     baseUrl?: string;
 
     /**
-     * An array of glob pattern strings or plain objects
-     * containing glob pattern strings matching the URLs to be enqueued.
+     * An array of URL patterns that URLs must match to be enqueued.
      *
-     * The plain objects must include at least the `glob` property, which holds the glob pattern string.
-     * All remaining keys will be used as request options for the corresponding enqueued {@apilink Request} objects.
-     *
-     * The matching is always case-insensitive.
-     * If you need case-sensitive matching, use `regexps` property directly.
-     *
-     * If `globs` is an empty array or `undefined`, and `regexps` are also not defined, then the function
-     * enqueues the links with the same subdomain.
-     */
-    globs?: readonly GlobInput[];
-
-    /**
-     * An array of glob pattern strings, regexp patterns or plain objects
-     * containing patterns matching URLs that will **never** be enqueued.
-     *
-     * The plain objects must include either the `glob` property or the `regexp` property.
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
      *
      * Glob matching is always case-insensitive.
-     * If you need case-sensitive matching, provide a regexp.
-     */
-    exclude?: readonly (GlobInput | RegExpInput)[];
-
-    /**
-     * An array of regular expressions or plain objects
-     * containing regular expressions matching the URLs to be enqueued.
-     *
-     * The plain objects must include at least the `regexp` property, which holds the regular expression.
-     * All remaining keys will be used as request options for the corresponding enqueued {@apilink Request} objects.
+     * If you need case-sensitive matching, use a `RegExp`.
      *
-     * If `regexps` is an empty array or `undefined`, and `globs` are also not defined, then the function
+     * If `include` is an empty array or `undefined`, then the function
      * enqueues the links with the same subdomain.
      */
-    regexps?: readonly RegExpInput[];
+    include?: readonly UrlPatternInput[];
 
     /**
-     * *NOTE:* In future versions of SDK the options will be removed.
-     * Please use `globs` or `regexps` instead.
+     * An array of URL patterns. Matching URLs will **not** be enqueued.
      *
-     * An array of {@apilink PseudoUrl} strings or plain objects
-     * containing {@apilink PseudoUrl} strings matching the URLs to be enqueued.
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
      *
-     * The plain objects must include at least the `purl` property, which holds the pseudo-URL string.
-     * All remaining keys will be used as request options for the corresponding enqueued {@apilink Request} objects.
-     *
-     * With a pseudo-URL string, the matching is always case-insensitive.
-     * If you need case-sensitive matching, use `regexps` property directly.
-     *
-     * If `pseudoUrls` is an empty array or `undefined`, then the function
-     * enqueues the links with the same subdomain.
-     *
-     * @deprecated prefer using `globs` or `regexps` instead
+     * Glob matching is always case-insensitive.
+     * If you need case-sensitive matching, use a `RegExp`.
      */
-    pseudoUrls?: readonly PseudoUrlInput[];
+    exclude?: readonly UrlPatternInput[];
 
     /**
-     * After request options are filtered by patterns, this function can be used
+     * After request options are filtered by `include`/`exclude` patterns, this function can be used
      * to remove them or modify their contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
      * when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
      * or to dynamically update or create `userData`.
@@ -145,8 +104,8 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      * }
      * ```
      *
-     * Note that `transformRequestFunction` has the highest priority and can overwrite request options
-     * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as the global `label` option.
+     * Note that `transformRequestFunction` has the highest priority and can overwrite
+     * the global `label` option.
      *
      * The function receives a {@apilink RequestOptions} object and can return either:
      * - The modified {@apilink RequestOptions} object
@@ -256,8 +215,7 @@ export enum EnqueueStrategy {
  * This function enqueues the urls provided to the {@apilink RequestQueue} provided. If you want to automatically find and enqueue links,
  * you should use the context-aware `enqueueLinks` function provided on the crawler contexts.
  *
- * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
- * and override settings of the enqueued {@apilink Request} objects.
+ * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
  *
  * **Example usage**
  *
@@ -266,7 +224,7 @@ export enum EnqueueStrategy {
  *   urls: aListOfFoundUrls,
  *   requestQueue,
  *   selector: 'a.product-detail',
- *   globs: [
+ *   include: [
  *       'https://www.example.com/handbags/*',
  *       'https://www.example.com/purses/*'
  *   ],
@@ -295,6 +253,8 @@ export async function enqueueLinks(
         );
     }
 
+    const urlPatternValidator = ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'));
+
     ow(
         options as any,
         ow.object.exactShape({
@@ -309,12 +269,8 @@ export async function enqueueLinks(
             baseUrl: ow.optional.string,
             userData: ow.optional.object,
             label: ow.optional.string,
-            pseudoUrls: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('purl'))),
-            globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
-            exclude: ow.optional.array.ofType(
-                ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp')),
-            ),
-            regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
+            include: ow.optional.array.ofType(urlPatternValidator),
+            exclude: ow.optional.array.ofType(urlPatternValidator),
             transformRequestFunction: ow.optional.function,
             strategy: ow.optional.string.oneOf(Object.values(EnqueueStrategy)),
             waitForAllRequestsToBeAdded: ow.optional.boolean,
@@ -325,42 +281,17 @@ export async function enqueueLinks(
         requestQueue,
         limit,
         urls,
-        pseudoUrls,
+        include,
         exclude,
-        globs,
-        regexps,
         transformRequestFunction,
         forefront,
         waitForAllRequestsToBeAdded,
         robotsTxtFile,
         onSkippedRequest,
     } = options;
 
-    const urlExcludePatternObjects: UrlPatternObject[] = [];
-    const urlPatternObjects: UrlPatternObject[] = [];
-
-    if (exclude?.length) {
-        for (const excl of exclude) {
-            if (typeof excl === 'string' || 'glob' in excl) {
-                urlExcludePatternObjects.push(...constructGlobObjectsFromGlobs([excl]));
-            } else if (excl instanceof RegExp || 'regexp' in excl) {
-                urlExcludePatternObjects.push(...constructRegExpObjectsFromRegExps([excl]));
-            }
-        }
-    }
-
-    if (pseudoUrls?.length) {
-        serviceLocator.getLogger().deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
-        urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));
-    }
-
-    if (globs?.length) {
-        urlPatternObjects.push(...constructGlobObjectsFromGlobs(globs));
-    }
-
-    if (regexps?.length) {
-        urlPatternObjects.push(...constructRegExpObjectsFromRegExps(regexps));
-    }
+    const urlExcludePatternObjects: UrlPatternObject[] = exclude?.length ? constructUrlPatternObjects(exclude) : [];
+    const urlPatternObjects: UrlPatternObject[] = include?.length ? constructUrlPatternObjects(include) : [];
 
     if (!urlPatternObjects.length) {
         options.strategy ??= EnqueueStrategy.SameHostname;
@@ -445,8 +376,7 @@ export async function enqueueLinks(
     async function createFilteredRequests() {
         const skippedRequests: string[] = [];
 
-        // Step 1: Filter request options by exclude patterns, user patterns (globs/regexps), and strategy patterns.
-        // Pattern-level options (label, userData, method, etc.) are merged during this step.
+        // Step 1: Filter request options by exclude patterns, user include patterns, and strategy patterns.
         let filteredOptions: RequestOptions[];
         if (urlPatternObjects.length === 0) {
             filteredOptions = filterRequestOptionsByPatterns(
@@ -565,7 +495,7 @@ export interface ResolveBaseUrl {
 }
 
 /**
- * Internal function that changes the enqueue globs to match both http and https
+ * Internal function that changes the enqueue glob patterns to match both http and https
  */
 function ignoreHttpSchema(pattern: string): string {
     return pattern.replace(/^(https?):\/\//, 'http{s,}://');