chore(haste-map): replace walker and hand-rolled traversal with fdir (#16187)

Author: SimenB

CHANGELOG.md

@@ -15,6 +15,7 @@
 ### Chore & Maintenance
 
 - `[jest-haste-map]` Refactor massive class into multiple files ([#16180](https://github.com/jestjs/jest/pull/16180))
+- `[jest-haste-map]` Drop `walker` dependency; replace hand-rolled directory recursion in the JS crawler and watcher startup with `fdir` ([#16187](https://github.com/jestjs/jest/pull/16187))
 - `[jest-runtime]` Avoid magical `null` value in ESM loader ([#16160](https://github.com/jestjs/jest/pull/16160))
 
 ## 30.4.2

docs/Configuration.md

@@ -913,11 +913,22 @@ type HasteConfig = {
   computeSha1?: boolean;
   /** The platform to use as the default, e.g. 'ios'. */
   defaultPlatform?: string | null;
-  /** Force use of Node's `fs` APIs rather than shelling out to `find` */
+  /**
+   * Force use of Node's `fs` APIs (via `fdir`) rather than shelling out to
+   * the system `find` binary. Defaults to `false` on Linux/macOS.
+   *
+   * **Consider setting this to `true`**: `find(1)` receives no ignore
+   * predicate, so it traverses ignored directories (e.g. `node_modules`,
+   * `.git`) in full and discards the results afterward. The Node `fs` crawler
+   * prunes those subtrees at `readdir` time. For most projects where
+   * `node_modules` dwarfs source files, this makes the Node crawler faster
+   * in practice despite `find(1)` being native code. A future Jest version
+   * may flip this default.
+   */
   forceNodeFilesystemAPI?: boolean;
   /**
    * Whether to follow symlinks when crawling for files.
-   *   This options cannot be used in projects which use watchman.
+   *   This option cannot be used in projects which use watchman.
    *   Projects with `watchman` set to true will error if this option is set to true.
    */
   enableSymlinks?: boolean;

packages/jest-haste-map/CLAUDE.md

@@ -8,10 +8,11 @@ Key files to know:
 
 - `lib/FileProcessor.ts` — `processFile` (worker dispatch, haste-id extraction, duplicate tracking) and `buildHasteMap` (initial build loop).
 - `lib/CacheManager.ts` — v8 serialize/deserialize for the on-disk cache. **Sync I/O is intentional** — at haste-map's scale, async overhead adds no value and switching to `fs.promises` is not a free win.
+- `lib/walk.ts` — shared `fdir`-backed directory walker used by the node crawler (`crawlers/node.ts` `find`), `FSEventsWatcher.ts` startup, and `NodeWatcher.js` startup (via `watchers/common.js` `recReaddir`). Exports `walk(opts, done): void` (callback-based) and `WalkOptions`, `WalkEntryKind`. Uses `fdir`'s `.withCallback()` API with a bounded callback-driven `lstat` pool (default `Math.max(os.availableParallelism() * 4, 32)` inflight). `fdir` is constructed with `new Fdir({fs})` so it uses `graceful-fs` for `readdir` — mock `graceful-fs.readdir` in tests to control directory traversal. `WalkOptions.statCache` is an optional `Map<string, Stats>` passed by callers that invoke `walk()` multiple times (e.g. once per root): a path already in the map skips its `lstat` call. **The cache must not be stored on watcher instances or passed to runtime (post-startup) walks** — stale stats would be returned for files changed after startup. `find()` and `WatcherDriver.start()` each create a fresh cache, use it for all per-root startup walks, then discard it.
 - `watchers/ChangeQueue.ts` — 30 ms debounce, O(1) mtime-dedup via `Set<string>`, copy-on-write for the live map, file-processing dispatch.
-- `crawlers/watchman.ts` — fb-watchman with clock-based incremental updates. `crawlers/node.ts` — pure Node.js fallback.
-- `watchers/types.ts` — `IWatcher`, `WatcherOptions`, `WatcherCtor`, `WatcherBackend`. New backends must implement `IWatcher` and accept `(root: string, opts: WatcherOptions)`.
-- `watchers/WatchmanWatcher.js` — macOS/Linux watchman. `FSEventsWatcher.ts` — macOS native. `NodeWatcher.js` — cross-platform fallback.
+- `crawlers/watchman.ts` — fb-watchman with clock-based incremental updates. `crawlers/node.ts` — `findNative` (`find(1)` shell-out) + `find` (`fdir` via `lib/walk`); `forceNodeFilesystemAPI` gates shell-out vs `fdir`.
+- `watchers/types.ts` — `IWatcher`, `WatcherOptions`, `WatcherCtor`. New backends must implement `IWatcher` and accept `(root: string, opts: WatcherOptions)`.
+- `watchers/WatchmanWatcher.js` — macOS/Linux watchman. `FSEventsWatcher.ts` — macOS native (startup walk via `lib/walk`). `NodeWatcher.js` — cross-platform fallback.
 
 ## Data model
 
@@ -40,6 +41,8 @@ Key files to know:
 
 **`enableSymlinks` guard** fires when `enableSymlinks && useWatchman`.
 
+**`enableSymlinks` symlink semantics.** When `false` (default), symlinks are excluded by all walkers. When `true`: `find` (fdir path) passes `enableSymlinks` to `walk()`, which sets `excludeSymlinks: false` and keeps `resolveSymlinks: false` (fdir mode 2) — symlinks are included at their **original link path** (not realpath); `walk()` then calls `fs.stat` (follows the link) to get target stats (mtime/size). This preserves the path Jest uses to `require` the file while reporting the target's metadata. `findNative` (shell `find` path) uses `( -type f -o -type l )` when `enableSymlinks` is true to include symlinks, then `fs.stat`s each result. Do NOT set fdir's `resolveSymlinks: true` — it calls `realpath` and returns the resolved path, which would cause haste-map to track files under the wrong path.
+
 **Config wiring from jest-config to jest-haste-map:** `HasteMap.Options` fields come from two places in `ProjectConfig`: `haste.enableSymlinks` → `enableSymlinks`, `haste.forceNodeFilesystemAPI` → `forceNodeFilesystemAPI`. The `useWatchman` field comes from the caller (e.g. `jest-runtime` passes `options?.watchman`; `jest-core` passes `globalConfig.watchman`). If you add a new `haste.*` config key that needs to reach `HasteMap`, add it to `HasteConfig` in `jest-types/src/Config.ts`, `HasteConfig` schema in `jest-schemas/src/raw-types.ts`, `Defaults.ts` (if it has a default), `ValidConfig.ts` (both `initialOptions.haste` and `initialProjectOptions.haste`), and the `HasteMap.create(...)` call in `jest-runtime/src/index.ts`.
 
 ## Tests

packages/jest-haste-map/README.md

@@ -73,9 +73,9 @@ console.log(files);
 | dependencyExtractor    | string \| null      | No       | `null`        |
 | enableSymlinks         | boolean             | No       | `false`       |
 | extensions             | Array<string> | Yes      | -             |
-| forceNodeFilesystemAPI | boolean             | Yes      | -             |
-| hasteImplModulePath    | string              | Yes      | -             |
-| hasteMapModulePath     | string              | Yes      | -             |
+| forceNodeFilesystemAPI | boolean             | No       | `false`       |
+| hasteImplModulePath    | string              | No       | -             |
+| hasteMapModulePath     | string              | No       | -             |
 | id                     | string              | Yes      | -             |
 | ignorePattern          | HasteRegExp         | No       | -             |
 | maxWorkers             | number              | Yes      | -             |
@@ -85,8 +85,8 @@ console.log(files);
 | retainAllFiles         | boolean             | Yes      | -             |
 | rootDir                | string              | Yes      | -             |
 | roots                  | Array<string> | Yes      | -             |
-| skipPackageJson        | boolean             | Yes      | -             |
-| throwOnModuleCollision | boolean             | Yes      | -             |
+| skipPackageJson        | boolean             | No       | `false`       |
+| throwOnModuleCollision | boolean             | No       | `false`       |
 | useWatchman            | boolean             | No       | `true`        |
 
 For more, you can check [github](https://github.com/jestjs/jest/tree/main/packages/jest-haste-map)

packages/jest-haste-map/package.json

@@ -23,12 +23,12 @@
     "@types/node": "*",
     "anymatch": "^3.1.3",
     "fb-watchman": "^2.0.2",
+    "fdir": "^6.5.0",
     "graceful-fs": "^4.2.11",
     "jest-regex-util": "workspace:*",
     "jest-util": "workspace:*",
     "jest-worker": "workspace:*",
-    "picomatch": "^4.0.3",
-    "walker": "^1.0.8"
+    "picomatch": "^4.0.3"
   },
   "devDependencies": {
     "@types/fb-watchman": "^2.0.5",

packages/jest-haste-map/src/crawlers/__tests__/node.test.js

@@ -71,44 +71,52 @@ jest.mock('graceful-fs', () => {
         throw new Error('readdir: callback is not a function!');
       }
 
-      if (slash(dir) === '/project/fruits') {
+      const normalizedDir = slash(dir).replace(/\/+$/, '');
+      if (normalizedDir === '/project/fruits') {
         setTimeout(
           () =>
             callback(null, [
               {
                 isDirectory: () => true,
+                isFile: () => false,
                 isSymbolicLink: () => false,
                 name: 'directory',
               },
               {
                 isDirectory: () => false,
+                isFile: () => true,
                 isSymbolicLink: () => false,
                 name: 'tomato.js',
               },
               {
                 isDirectory: () => false,
+                isFile: () => false,
                 isSymbolicLink: () => true,
                 name: 'symlink',
               },
             ]),
           0,
         );
-      } else if (slash(dir) === '/project/fruits/directory') {
+      } else if (normalizedDir === '/project/fruits/directory') {
         setTimeout(
           () =>
             callback(null, [
               {
                 isDirectory: () => false,
+                isFile: () => true,
                 isSymbolicLink: () => false,
                 name: 'strawberry.js',
               },
             ]),
           0,
         );
-      } else if (slash(dir) === '/error') {
+      } else if (normalizedDir === '/error') {
         setTimeout(() => callback({code: 'ENOTDIR'}, undefined), 0);
       }
     }),
+    readdirSync: jest.fn(() => []),
+    realpath: jest.fn(),
+    realpathSync: jest.fn(),
     stat: jest.fn(stat),
   };
 });
@@ -364,6 +372,73 @@ describe('node crawler', () => {
     expect(removedFiles).toEqual(new Map());
   });
 
+  it('deduplicates results when roots overlap', async () => {
+    nodeCrawl = require('../node').nodeCrawl;
+
+    const {hasteMap} = await nodeCrawl({
+      data: {files: new Map()},
+      extensions: ['js'],
+      forceNodeFilesystemAPI: true,
+      ignore: pearMatcher,
+      rootDir,
+      // /project/fruits/directory is a subdirectory of /project/fruits, so
+      // strawberry.js is reachable from both roots.
+      roots: ['/project/fruits', '/project/fruits/directory'],
+    });
+
+    expect(hasteMap.files).toEqual(
+      createMap({
+        'fruits/directory/strawberry.js': expect.any(Array),
+        'fruits/tomato.js': expect.any(Array),
+      }),
+    );
+  });
+
+  it('passes symlink args to find when enableSymlinks is true', async () => {
+    childProcess = require('child_process');
+    nodeCrawl = require('../node').nodeCrawl;
+
+    await nodeCrawl({
+      data: {files: new Map()},
+      enableSymlinks: true,
+      extensions: ['js'],
+      ignore: pearMatcher,
+      rootDir,
+      roots: ['/project/fruits'],
+    });
+
+    expect(childProcess.spawn).toHaveBeenLastCalledWith('find', [
+      '/project/fruits',
+      '(',
+      '-type',
+      'f',
+      '-o',
+      '-type',
+      'l',
+      ')',
+      '(',
+      '-iname',
+      '*.js',
+      ')',
+    ]);
+  });
+
+  it('handles empty results from find after filtering', async () => {
+    nodeCrawl = require('../node').nodeCrawl;
+    // All paths match the ignore pattern, so the filtered list is empty.
+    mockResponse = '/project/fruits/pear.js';
+
+    const {hasteMap} = await nodeCrawl({
+      data: {files: new Map()},
+      extensions: ['js'],
+      ignore: pearMatcher,
+      rootDir,
+      roots: ['/project/fruits'],
+    });
+
+    expect(hasteMap.files).toEqual(new Map());
+  });
+
   it('avoids calling lstat for directories and symlinks', async () => {
     nodeCrawl = require('../node').nodeCrawl;
     const fs = require('graceful-fs');

packages/jest-haste-map/src/crawlers/node.ts

@@ -10,6 +10,7 @@ import * as path from 'node:path';
 import * as fs from 'graceful-fs';
 import H from '../constants';
 import * as fastPath from '../lib/fast_path';
+import {walk} from '../lib/walk';
 import type {
   CrawlerOptions,
   FileData,
@@ -63,70 +64,36 @@ function find(
   enableSymlinks: boolean,
   callback: Callback,
 ): void {
+  const extSet = new Set(extensions);
   const result: Result = [];
-  let activeCalls = 0;
-
-  function search(directory: string): void {
-    activeCalls++;
-    fs.readdir(directory, {withFileTypes: true}, (err, entries) => {
-      activeCalls--;
-      if (err) {
-        if (activeCalls === 0) {
-          callback(result);
-        }
-        return;
-      }
-      for (const entry of entries) {
-        const file = path.join(directory, entry.name);
-
-        if (ignore(file)) {
-          continue;
-        }
-
-        if (entry.isSymbolicLink()) {
-          continue;
-        }
-        if (entry.isDirectory()) {
-          search(file);
-          continue;
-        }
+  const statCache = new Map<string, fs.Stats>();
+  let remaining = roots.length;
 
-        activeCalls++;
-
-        const stat = enableSymlinks ? fs.stat : fs.lstat;
-
-        stat(file, (err, stat) => {
-          activeCalls--;
-
-          // This logic is unnecessary for node > v10.10, but leaving it in
-          // since we need it for backwards-compatibility still.
-          if (!err && stat && !stat.isSymbolicLink()) {
-            if (stat.isDirectory()) {
-              search(file);
-            } else {
-              const ext = path.extname(file).slice(1);
-              if (extensions.includes(ext)) {
-                result.push([file, stat.mtime.getTime(), stat.size]);
-              }
-            }
-          }
-
-          if (activeCalls === 0) {
-            callback(result);
-          }
-        });
-      }
-
-      if (activeCalls === 0) {
-        callback(result);
-      }
-    });
+  if (remaining === 0) {
+    callback(result);
+    return;
   }
 
-  if (roots.length > 0) {
-    for (const root of roots) search(root);
-  } else {
-    callback(result);
+  for (const root of roots) {
+    walk(
+      {
+        enableSymlinks,
+        exclude: ignore,
+        onEntry: (kind, filePath, stats) => {
+          if (kind === 'file' && extSet.has(path.extname(filePath).slice(1))) {
+            result.push([filePath, stats.mtime.getTime(), stats.size]);
+          }
+        },
+        root,
+        statCache,
+      },
+      () => {
+        remaining--;
+        if (remaining === 0) {
+          callback(result);
+        }
+      },
+    );
   }
 }
 
@@ -158,20 +125,21 @@ function findNative(
   }
 
   const child = spawn('find', args);
-  let stdout = '';
   if (child.stdout === null) {
     throw new Error(
       'stdout is null - this should never happen. Please open up an issue at https://github.com/jestjs/jest',
     );
   }
   child.stdout.setEncoding('utf8');
-  child.stdout.on('data', data => (stdout += data));
+  const chunks: Array<string> = [];
+  child.stdout.on('data', data => chunks.push(data));
 
   child.stdout.on('close', () => {
-    const lines = stdout
+    const lines = chunks
+      .join('')
       .trim()
       .split('\n')
-      .filter(x => !ignore(x));
+      .filter(x => x && !ignore(x));
     const result: Result = [];
     let count = lines.length;
     if (count) {
@@ -198,11 +166,11 @@ export async function nodeCrawl(options: CrawlerOptions): Promise<{
 }> {
   const {
     data,
+    enableSymlinks,
     extensions,
     forceNodeFilesystemAPI,
     ignore,
     rootDir,
-    enableSymlinks,
     roots,
   } = options;
 
@@ -233,6 +201,12 @@ export async function nodeCrawl(options: CrawlerOptions): Promise<{
     };
 
     if (useNativeFind) {
+      // TODO: consider making forceNodeFilesystemAPI the default. find(1) does
+      // not receive the ignore predicate, so it traverses ignored directories
+      // (e.g. node_modules, .git) in full and discards results afterward.
+      // find() via fdir prunes those subtrees at readdir time. For a typical
+      // project where node_modules dwarfs source files, the wasted traversal
+      // likely outweighs find(1)'s native speed advantage.
       findNative(roots, extensions, ignore, enableSymlinks, callback);
     } else {
       find(roots, extensions, ignore, enableSymlinks, callback);