fix: rectify transformRequestFunction precedence rules (#3372)

Fixes `transformRequestFunction` precedence in `enqueueLinks` — it now runs **after** pattern filtering instead of before, giving it the highest priority so its modifications can't be silently overwritten by `globs`/`regexps`/`pseudoUrls` options.

Previously, pattern-specific options (like `method` or `userData` on a glob) would override changes made by `transformRequestFunction`, which was unintuitive since the transform is the user's explicit escape hatch for customizing requests.

**Priority order (lowest to highest):**
1. Global `label`/`userData`
2. Pattern-specific options (`globs`, `regexps`, `pseudoUrls`)
3. `transformRequestFunction`

**What changed:**
- Reordered the processing pipeline: patterns filter and enrich first, then `transformRequestFunction` runs last
- Requests rejected by the transform (returning `null`/`false`) now report a `'transform'` skip reason via `onSkippedRequest`
- Updated Playwright and Puppeteer click-element utilities for consistent behavior
- Added upgrade guide documentation and comprehensive tests

Author: l2ysho

docs/upgrading/upgrading_v4.md

@@ -114,3 +114,18 @@ The `KeyValueStore.getPublicUrl` method is now asynchronous and reads the public
 ## `preNavigationHooks` in `HttpCrawler` no longer accepts `gotOptions` object
 
 The `preNavigationHooks` option in `HttpCrawler` subclasses no longer accepts the `gotOptions` object as a second parameter. Modify the `crawlingContext` fields (e.g. `.request`) directly instead.
+
+## `transformRequestFunction` precedence in `enqueueLinks`
+
+The `transformRequestFunction` callback in `enqueueLinks` now runs **after** URL pattern filtering (`globs`, `regexps`, `pseudoUrls`) instead of before. This means it has the highest priority and can overwrite any request options set by patterns or the global `label` option.
+
+The priority order is now (lowest to highest):
+1. Global `label` / `userData` options
+2. Pattern-specific options from `globs`, `regexps`, or `pseudoUrls` objects
+3. `transformRequestFunction`
+
+The `transformRequestFunction` callback receives a `RequestOptions` object and can return either:
+- The modified `RequestOptions` object
+- A new `RequestOptions` plain object
+- `'unchanged'` to keep the original options as-is
+- A falsy value or `'skip'` to exclude the request from the queue

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -1723,16 +1723,19 @@ export class BasicCrawler<
         request: Request<Dictionary>,
         requestQueue: RequestProvider,
     ): Promise<BatchAddRequestsResult> {
-        const transformRequestFunctionWrapper: RequestTransform = (newRequest) => {
-            newRequest.crawlDepth = request.crawlDepth + 1;
-
-            if (this.maxCrawlDepth !== undefined && newRequest.crawlDepth > this.maxCrawlDepth) {
-                newRequest.skippedReason = 'depth';
+        const transformRequestFunctionWrapper: RequestTransform = (requestOptions) => {
+            requestOptions.crawlDepth = request.crawlDepth + 1;
+
+            if (this.maxCrawlDepth !== undefined && requestOptions.crawlDepth! > this.maxCrawlDepth) {
+                // Setting `skippedReason` before returning `false` ensures that `reportSkippedRequests`
+                // reports `'depth'` as the reason (via `request.skippedReason ?? reason` fallback),
+                // rather than the generic `'transform'` reason.
+                requestOptions.skippedReason = 'depth';
                 return false;
             }
 
             // After injecting the crawlDepth, we call the user-provided transform function, if there is one.
-            return options.transformRequestFunction?.(newRequest) ?? newRequest;
+            return options.transformRequestFunction?.(requestOptions) ?? requestOptions;
         };
 
         return await enqueueLinks({

packages/core/src/enqueue_links/enqueue_links.ts

@@ -6,7 +6,8 @@ import type { SetRequired } from 'type-fest';
 
 import log from '@apify/log';
 
-import type { Request, RequestOptions } from '../request.js';
+import type { RequestOptions } from '../request.js';
+import { Request } from '../request.js';
 import type {
     AddRequestsBatchedOptions,
     AddRequestsBatchedResult,
@@ -23,12 +24,12 @@ import type {
     UrlPatternObject,
 } from './shared.js';
 import {
+    applyRequestTransform,
     constructGlobObjectsFromGlobs,
     constructRegExpObjectsFromPseudoUrls,
     constructRegExpObjectsFromRegExps,
     createRequestOptions,
-    createRequests,
-    filterRequestsByPatterns,
+    filterRequestOptionsByPatterns,
 } from './shared.js';
 
 export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
@@ -50,8 +51,8 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
     /**
      * Sets {@apilink Request.label} for newly enqueued requests.
      *
-     * Note that the request options specified in `globs`, `regexps`, or `pseudoUrls` objects
-     * have priority over this option.
+     * This option has the lowest priority and can be overwritten by request options
+     * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as by `transformRequestFunction`.
      */
     label?: string;
 
@@ -126,12 +127,12 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
     pseudoUrls?: readonly PseudoUrlInput[];
 
     /**
-     * Just before a new {@apilink Request} is constructed and enqueued to the {@apilink RequestQueue}, this function can be used
-     * to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
+     * After request options are filtered by patterns, this function can be used
+     * to remove them or modify their contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
      * when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
      * or to dynamically update or create `userData`.
      *
-     * For example: by adding `keepUrlFragment: true` to the `request` object, URL fragments will not be removed
+     * For example: by adding `keepUrlFragment: true` to the request options, URL fragments will not be removed
      * when `uniqueKey` is computed.
      *
      * **Example:**
@@ -145,8 +146,13 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      * }
      * ```
      *
-     * Note that the request options specified in `globs`, `regexps`, or `pseudoUrls` objects
-     * have priority over this function. Some request options returned by `transformRequestFunction` may be overwritten by pattern-based options from `globs`, `regexps`, or `pseudoUrls`.
+     * Note that `transformRequestFunction` has the highest priority and can overwrite request options
+     * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as the global `label` option.
+     *
+     * The function receives a {@apilink RequestOptions} object and can return either:
+     * - The modified {@apilink RequestOptions} object
+     * - `'unchanged'` to keep the original options as-is
+     * - A falsy value or `'skip'` to exclude the request from the queue
      */
     transformRequestFunction?: RequestTransform;
 
@@ -437,58 +443,59 @@ export async function enqueueLinks(
         await reportSkippedRequests(skippedRequests, 'robotsTxt');
     }
 
-    if (transformRequestFunction) {
-        const skippedRequests: RequestOptions[] = [];
-
-        requestOptions = requestOptions
-            .map((request) => {
-                const transformedRequest = transformRequestFunction(request);
-                if (!transformedRequest) {
-                    skippedRequests.push(request);
-                }
-                return transformedRequest;
-            })
-            .filter((r) => Boolean(r)) as RequestOptions[];
-
-        await reportSkippedRequests(skippedRequests, 'filters');
-    }
-
     async function createFilteredRequests() {
         const skippedRequests: string[] = [];
 
-        // No user provided patterns means we can skip an extra filtering step
+        // Step 1: Filter request options by exclude patterns, user patterns (globs/regexps), and strategy patterns.
+        // Pattern-level options (label, userData, method, etc.) are merged during this step.
+        let filteredOptions: RequestOptions[];
         if (urlPatternObjects.length === 0) {
-            return createRequests(
+            filteredOptions = filterRequestOptionsByPatterns(
+                requestOptions,
+                enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined,
+                urlExcludePatternObjects,
+                options.strategy,
+                (url) => skippedRequests.push(url),
+            );
+        } else {
+            // Filter by user patterns first (with exclude)
+            const afterUserPatterns = filterRequestOptionsByPatterns(
                 requestOptions,
-                enqueueStrategyPatterns,
+                urlPatternObjects,
                 urlExcludePatternObjects,
                 options.strategy,
                 (url) => skippedRequests.push(url),
             );
+            // ...then filter by the enqueue links strategy (making this an AND check)
+            filteredOptions = filterRequestOptionsByPatterns(
+                afterUserPatterns,
+                enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined,
+                [],
+                options.strategy,
+                (url) => skippedRequests.push(url),
+            );
         }
 
-        // Generate requests based on the user patterns first
-        const generatedRequestsFromUserFilters = createRequests(
-            requestOptions,
-            urlPatternObjects,
-            urlExcludePatternObjects,
-            options.strategy,
-            (url) => skippedRequests.push(url),
-        );
-        // ...then filter them by the enqueue links strategy (making this an AND check)
-        const filtered = filterRequestsByPatterns(generatedRequestsFromUserFilters, enqueueStrategyPatterns, (url) =>
-            skippedRequests.push(url),
-        );
-
         await reportSkippedRequests(
             skippedRequests.map((url) => ({ url })),
             'filters',
         );
 
-        return filtered;
+        // Step 2: Apply transformRequestFunction on request options - it has the highest priority
+        if (transformRequestFunction) {
+            const skippedByTransform: RequestOptions[] = [];
+            filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) =>
+                skippedByTransform.push(r),
+            );
+            await reportSkippedRequests(skippedByTransform, 'transform');
+        }
+
+        // Step 3: Create Request instances from the final request options
+        return filteredOptions.map((opts) => new Request(opts));
     }
 
     let requests = await createFilteredRequests();
+
     if (typeof limit === 'number' && limit < requests.length) {
         await reportSkippedRequests(requests.slice(limit), 'enqueueLimit');
         requests = requests.slice(0, limit);

packages/core/src/enqueue_links/shared.ts

@@ -6,7 +6,6 @@ import { Minimatch } from 'minimatch';
 import { purlToRegExp } from '@apify/pseudo_url';
 
 import type { RequestOptions } from '../request.js';
-import { Request } from '../request.js';
 import type { EnqueueLinksOptions } from './enqueue_links.js';
 
 export { tryAbsoluteURL } from '@crawlee/utils';
@@ -47,7 +46,14 @@ export type RegExpObject = { regexp: RegExp } & Pick<
 
 export type RegExpInput = RegExp | RegExpObject;
 
-export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'redirect' | 'depth';
+export type SkippedRequestReason =
+    | 'robotsTxt'
+    | 'limit'
+    | 'enqueueLimit'
+    | 'filters'
+    | 'transform'
+    | 'redirect'
+    | 'depth';
 
 export type SkippedRequestCallback = (args: { url: string; reason: SkippedRequestReason }) => Awaitable<void>;
 
@@ -164,76 +170,46 @@ export function constructRegExpObjectsFromRegExps(regexps: readonly RegExpInput[
 }
 
 /**
+ * Filters request options by URL patterns and merges pattern-level options (label, userData, method, payload, headers)
+ * from the first matching pattern into each RequestOptions entry.
+ *
+ * When `includePatterns` is empty/undefined, all options pass through (only exclude filtering applies).
  * @ignore
  */
-export function createRequests(
-    requestOptions: (string | RequestOptions)[],
-    urlPatternObjects?: UrlPatternObject[],
-    excludePatternObjects: UrlPatternObject[] = [],
+export function filterRequestOptionsByPatterns(
+    requestOptions: RequestOptions[],
+    includePatterns: UrlPatternObject[] | undefined,
+    excludePatterns: UrlPatternObject[] = [],
     strategy?: EnqueueLinksOptions['strategy'],
     onSkippedUrl?: (url: string) => void,
-): Request[] {
-    const excludePatternObjectMatchers = excludePatternObjects.map(createPatternObjectMatcher);
-    const urlPatternObjectMatchers = urlPatternObjects?.map(createPatternObjectMatcher);
+): RequestOptions[] {
+    const excludeMatchers = excludePatterns.map(createPatternObjectMatcher);
+    const includeMatchers = includePatterns?.length ? includePatterns.map(createPatternObjectMatcher) : undefined;
 
     return requestOptions
-        .map((opts) => ({ url: typeof opts === 'string' ? opts : opts.url, opts }))
         .filter(({ url }) => {
-            const matchesExcludePatterns = excludePatternObjectMatchers.some(({ match }) => match(url));
-
-            if (matchesExcludePatterns) {
+            const matchesExclude = excludeMatchers.some(({ match }) => match(url));
+            if (matchesExclude) {
                 onSkippedUrl?.(url);
             }
-
-            return !matchesExcludePatterns;
+            return !matchesExclude;
         })
-        .map(({ url, opts }) => {
-            if (!urlPatternObjectMatchers || !urlPatternObjectMatchers.length) {
-                return new Request(typeof opts === 'string' ? { url: opts, enqueueStrategy: strategy } : { ...opts });
+        .map((opts) => {
+            if (!includeMatchers) {
+                return { ...opts, enqueueStrategy: strategy };
             }
 
-            for (const urlPatternObject of urlPatternObjectMatchers) {
-                const { match, glob, regexp, ...requestRegExpOptions } = urlPatternObject;
-                if (match(url)) {
-                    const request =
-                        typeof opts === 'string'
-                            ? { url: opts, ...requestRegExpOptions, enqueueStrategy: strategy }
-                            : { ...opts, ...requestRegExpOptions, enqueueStrategy: strategy };
-
-                    return new Request(request);
+            for (const { match, glob, regexp, ...patternOptions } of includeMatchers) {
+                if (match(opts.url)) {
+                    return { ...opts, ...patternOptions, enqueueStrategy: strategy };
                 }
             }
 
             // didn't match any positive pattern
-            onSkippedUrl?.(url);
+            onSkippedUrl?.(opts.url);
             return null;
         })
-        .filter((request) => request) as Request[];
-}
-
-export function filterRequestsByPatterns(
-    requests: Request[],
-    patterns?: UrlPatternObject[],
-    onSkippedUrl?: (url: string) => void,
-): Request[] {
-    if (!patterns?.length) {
-        return requests;
-    }
-
-    const filtered: Request[] = [];
-    const patternMatchers = patterns?.map(createPatternObjectMatcher);
-
-    for (const request of requests) {
-        const matchingPattern = patternMatchers.find(({ match }) => match(request.url));
-
-        if (matchingPattern !== undefined) {
-            filtered.push(request);
-        } else {
-            onSkippedUrl?.(request.url);
-        }
-    }
-
-    return filtered;
+        .filter((opts) => opts !== null);
 }
 
 /**
@@ -293,13 +269,45 @@ function createPatternObjectMatcher(urlPatternObject: UrlPatternObject) {
 }
 
 /**
- * Takes an Apify {@apilink RequestOptions} object and changes its attributes in a desired way. This user-function is used
- * {@apilink enqueueLinks} to modify requests before enqueuing them.
+ * Takes a {@apilink RequestOptions} object and changes its attributes in a desired way. This user-function is used
+ * by {@apilink enqueueLinks} to modify request options before they are converted to {@apilink Request} instances.
  */
 export interface RequestTransform {
     /**
      * @param original Request options to be modified.
-     * @returns The modified request options to enqueue.
+     * @returns The modified request options to enqueue, `'unchanged'` to keep the original options as-is,
+     *   or a falsy value / `'skip'` to exclude the request from the queue.
      */
-    (original: RequestOptions): RequestOptions | false | undefined | null;
+    (original: RequestOptions): RequestOptions | false | undefined | null | 'skip' | 'unchanged';
+}
+
+/**
+ * Applies a {@apilink RequestTransform} function to a list of request options.
+ * Options for which the transform returns a falsy value are removed from the list.
+ * @param onSkipped Called with the original request options when the transform returns a falsy value (i.e. the request is skipped).
+ * @ignore
+ * @internal
+ */
+export function applyRequestTransform(
+    requestOptions: RequestOptions[],
+    transformFn: RequestTransform,
+    onSkipped?: (requestOptions: RequestOptions) => void,
+): RequestOptions[] {
+    return requestOptions
+        .map((opts) => {
+            const transformed = transformFn(opts);
+            if (transformed === 'skip') {
+                onSkipped?.(opts);
+                return null;
+            }
+            if (transformed === 'unchanged') {
+                return opts;
+            }
+            if (!transformed) {
+                onSkipped?.(opts);
+                return null;
+            }
+            return transformed;
+        })
+        .filter((r): r is RequestOptions => r !== null);
 }