Add internal 'recrawl' watcher backend event and implement re-crawling a subdirectory.

Summary:
In order to interpret low-level OS file events and emit an accurate set of changes to all affected paths, we sometimes need to fall back to recursively scanning a directory and comparing it with previous state.

In Metro we have three watchers
 - Watchman, which has its own internal recrawl mechanism and always (/should!) give us a complete, high-level set of changes.
 - `FallbackWatcher`, currently used if Watchman is disabled on Windows and Linux, which implements flat inotify watches and *must* already internally crawl any "new" directories in order to watch them(*).
 - `NativeWatcher`, currently used if Watchman is disabled on macOS - uses `fsevents` to watch a whole tree *without* keeping track of contents.

For `NativeWatcher`, if we receive a 'rename' event and we find that the path is a directory, it *may* be new and it *may* have contents.

One approach would be for `NativeWatcher` to keep track of all files, internally scan any "renamed" directories, and emit any changes to files. However, that's redundant when we already have all of the mechanics for crawling, comparing with a previous state, and emitting the difference, downstream (because we do exactly that on startup, for current state vs cache).

Here, we implement a way for a watcher backend to let downstream know there may be changes.

Note that we don't need to worry about duplicate events (eg, a directory is created and immediately some contents are created - we might emit a new file event from the watcher as well as recrawling - but that's fine, because we aggregate changes at emit time).

Changelog: Internal

Reviewed By: vzaidman

Differential Revision: D94241640

fbshipit-source-id: 1ae41f07f17fbec864953ee2111cb3872065e261

Author: robhogan

packages/metro-file-map/src/Watcher.js

@@ -36,6 +36,13 @@ const debug = require('debug')('Metro:Watcher');
 
 const MAX_WAIT_TIME = 240000;
 
+type InternalCrawlOptions = Readonly<{
+  previousState: CrawlerOptions['previousState'],
+  roots: ReadonlyArray<string>,
+  subpath?: string,
+  useWatchman: boolean,
+}>;
+
 type WatcherOptions = {
   abortSignal: AbortSignal,
   computeSha1: boolean,
@@ -79,12 +86,41 @@ export class Watcher extends EventEmitter {
 
   async crawl(): Promise<CrawlResult> {
     this.#options.perfLogger?.point('crawl_start');
+    const options = this.#options;
+
+    const result = await this.#crawl({
+      previousState: options.previousState,
+      roots: options.roots,
+      useWatchman: options.useWatchman,
+    });
+
+    this.#options.perfLogger?.point('crawl_end');
+    return result;
+  }
+
+  async recrawl(
+    subpath: string,
+    currentFileSystem: CrawlerOptions['previousState']['fileSystem'],
+  ): Promise<CrawlResult> {
+    return this.#crawl({
+      previousState: {
+        clocks: new Map(),
+        fileSystem: currentFileSystem,
+      },
+      roots: [path.join(this.#options.rootDir, subpath)],
+      subpath,
+      useWatchman: false,
+    });
+  }
 
+  async #crawl(crawlOptions: InternalCrawlOptions): Promise<CrawlResult> {
     const options = this.#options;
+    const {useWatchman, subpath} = crawlOptions;
+
     const ignoreForCrawl = (filePath: string) =>
       options.ignoreForCrawl(filePath) ||
       path.basename(filePath).startsWith(this.#options.healthCheckFilePrefix);
-    const crawl = options.useWatchman ? watchmanCrawl : nodeCrawl;
+    const crawl = useWatchman ? watchmanCrawl : nodeCrawl;
     let crawler = crawl === watchmanCrawl ? 'watchman' : 'node';
 
     options.abortSignal.throwIfAborted();
@@ -101,12 +137,13 @@ export class Watcher extends EventEmitter {
         this.emit('status', status);
       },
       perfLogger: options.perfLogger,
-      previousState: options.previousState,
+      previousState: crawlOptions.previousState,
       rootDir: options.rootDir,
-      roots: options.roots,
+      roots: crawlOptions.roots,
+      subpath,
     };
 
-    debug('Beginning crawl with "%s".', crawler);
+    debug('Crawling roots: %s with %s crawler.', crawlOptions.roots, crawler);
 
     let delta: CrawlResult;
     try {
@@ -143,7 +180,6 @@ export class Watcher extends EventEmitter {
       delta.removedFiles.size,
       delta.clocks?.size ?? 0,
     );
-    this.#options.perfLogger?.point('crawl_end');
     return delta;
   }
 
@@ -202,6 +238,15 @@ export class Watcher extends EventEmitter {
             }
             return;
           }
+          // Watchman handles recrawls internally - receiving a recrawl event
+          // when using Watchman would indicate a bug. Log an error and ignore.
+          if (change.event === 'recrawl' && useWatchman) {
+            this.#options.console.error(
+              'metro-file-map: Received unexpected recrawl event while using ' +
+                'Watchman. Watchman recrawls are not implemented.',
+            );
+            return;
+          }
           onChange(change);
         });
         await watcher.startWatching();

packages/metro-file-map/src/__tests__/index-test.js

@@ -64,7 +64,11 @@ jest.mock('jest-worker', () => ({
   }),
 }));
 
-jest.mock('../crawlers/node', () => ({__esModule: true, default: jest.fn()}));
+const mockNodeCrawler = jest.fn();
+jest.mock('../crawlers/node', () => ({
+  __esModule: true,
+  default: mockNodeCrawler,
+}));
 jest.mock('../crawlers/watchman', () => ({
   __esModule: true,
   default: jest.fn(options => {
@@ -127,9 +131,14 @@ class MockWatcher extends AbstractWatcher {
     super(root, opts);
     mockEmitters[root] = this;
   }
+
+  static isSupported(): boolean {
+    return true;
+  }
 }
 
 jest.mock('../watchers/FallbackWatcher', () => MockWatcher);
+jest.mock('../watchers/NativeWatcher', () => MockWatcher);
 jest.mock('../watchers/WatchmanWatcher', () => MockWatcher);
 
 type MockFS = {[path: string]: ?string | {link: string}, __proto__: null};
@@ -277,6 +286,7 @@ let cacheContent = null;
 describe('FileMap', () => {
   beforeEach(() => {
     jest.resetModules();
+    mockNodeCrawler.mockClear();
 
     mockEmitters = Object.create(null);
     mockFs = object({
@@ -2482,5 +2492,243 @@ describe('FileMap', () => {
         },
       );
     });
+
+    describe('recrawl events', () => {
+      // Recrawl events only come from non-Watchman watchers (NativeWatcher,
+      // FallbackWatcher), because Watchman handles its own recrawls internally.
+      // These tests use useWatchman: false to simulate a non-Watchman watcher,
+      // so we need to mock nodeCrawl for the initial build.
+      beforeEach(() => {
+        mockNodeCrawler.mockImplementationOnce(async () => ({
+          changedFiles: new Map([
+            [path.join('fruits', 'Banana.js'), [32, 42, 0, null, 0, 'Banana']],
+            [path.join('fruits', 'Pear.js'), [32, 42, 0, null, 0, 'Pear']],
+            [
+              path.join('fruits', 'Strawberry.js'),
+              [32, 42, 0, null, 0, 'Strawberry'],
+            ],
+            [
+              path.join('fruits', '__mocks__', 'Pear.js'),
+              [32, 42, 0, null, 0, null],
+            ],
+            [
+              path.join('vegetables', 'Melon.js'),
+              [32, 42, 0, null, 0, 'Melon'],
+            ],
+          ]),
+          removedFiles: new Set<string>(),
+        }));
+      });
+
+      fm_it(
+        'recrawl event triggers subdirectory crawl and detects added files',
+        async ({fileMap, hasteMap}) => {
+          const {fileSystem: _fileSystem} = await fileMap.build();
+          const fruitsRoot = path.join('/', 'project', 'fruits');
+          const e = mockEmitters[fruitsRoot];
+
+          // Simulate a directory move-in: a new subdirectory appears with files
+          const newDir = path.join(fruitsRoot, 'tropical');
+          const newFile1 = path.join(newDir, 'Mango.js');
+          const newFile2 = path.join(newDir, 'Papaya.js');
+
+          mockFs[newFile1] = `// Mango!`;
+          mockFs[newFile2] = `// Papaya!`;
+
+          // Set up node crawler mock to return the new files
+          mockNodeCrawler.mockImplementationOnce(
+            async (options: $FlowFixMe) => {
+              const {rootDir} = options;
+              const changedFiles: Map<string, FileMetadata> = new Map();
+
+              // Return files found in the crawled subdirectory
+              changedFiles.set(path.relative(rootDir, newFile1), [
+                100,
+                50,
+                0,
+                null,
+                0,
+                null,
+              ]);
+              changedFiles.set(path.relative(rootDir, newFile2), [
+                101,
+                60,
+                0,
+                null,
+                0,
+                null,
+              ]);
+
+              return {
+                changedFiles,
+                removedFiles: new Set<string>(),
+              };
+            },
+          );
+
+          // Emit a recrawl event for the new directory
+          e.emitFileEvent({
+            event: 'recrawl',
+            relativePath: 'tropical',
+          });
+
+          await waitForItToChange(fileMap);
+
+          // Verify crawl was called with the correct directory
+          expect(mockNodeCrawler).toHaveBeenNthCalledWith(
+            2, // Second call is the recrawl (first call is initial build)
+            expect.objectContaining({
+              roots: [newDir],
+            }),
+          );
+        },
+        {config: {useWatchman: false}},
+      );
+
+      fm_it(
+        'recrawl event detects removed files from a moved-out directory',
+        async ({fileMap, hasteMap}) => {
+          const {fileSystem} = await fileMap.build();
+          const fruitsRoot = path.join('/', 'project', 'fruits');
+          const e = mockEmitters[fruitsRoot];
+
+          // Verify the file exists initially
+          const existingFile = path.join(fruitsRoot, 'Banana.js');
+          expect(fileSystem.exists(existingFile)).toBe(true);
+          expect(hasteMap.getModule('Banana')).toBe(existingFile);
+
+          // Set up node crawler mock to return the file as removed
+          mockNodeCrawler.mockImplementationOnce(
+            async (options: $FlowFixMe) => {
+              const {rootDir} = options;
+              const removedFiles: Set<string> = new Set();
+              removedFiles.add(path.relative(rootDir, existingFile));
+
+              return {
+                changedFiles: new Map<string, FileMetadata>(),
+                removedFiles,
+              };
+            },
+          );
+
+          // Emit a recrawl event (simulating directory being moved out)
+          e.emitFileEvent({
+            event: 'recrawl',
+            relativePath: '',
+          });
+
+          const {changes} = await waitForItToChange(fileMap);
+
+          // Verify deletion was emitted
+          expect(countFileChanges(changes)).toBe(1);
+          expect([...changes.removedFiles]).toHaveLength(1);
+
+          // Verify file is no longer in the file system
+          expect(fileSystem.exists(existingFile)).toBe(false);
+
+          // Verify haste map was updated
+          expect(hasteMap.getModule('Banana')).toBeNull();
+        },
+        {config: {useWatchman: false}},
+      );
+
+      fm_it(
+        'recrawl event detects both added and removed files',
+        async ({fileMap, hasteMap}) => {
+          const {fileSystem} = await fileMap.build();
+          const fruitsRoot = path.join('/', 'project', 'fruits');
+          const e = mockEmitters[fruitsRoot];
+
+          // Initial state
+          const existingFile = path.join(fruitsRoot, 'Pear.js');
+          expect(fileSystem.exists(existingFile)).toBe(true);
+
+          // New file to be added
+          const newFile = path.join(fruitsRoot, 'Kiwi.js');
+          mockFs[newFile] = `// Kiwi!`;
+
+          // Set up node crawler mock
+          mockNodeCrawler.mockImplementationOnce(
+            async (options: $FlowFixMe) => {
+              const {rootDir} = options;
+              const changedFiles: Map<string, FileMetadata> = new Map();
+              const removedFiles: Set<string> = new Set();
+
+              // Add new file
+              changedFiles.set(path.relative(rootDir, newFile), [
+                200,
+                70,
+                0,
+                null,
+                0,
+                null,
+              ]);
+
+              // Remove existing file
+              removedFiles.add(path.relative(rootDir, existingFile));
+
+              return {
+                changedFiles,
+                removedFiles,
+              };
+            },
+          );
+
+          e.emitFileEvent({
+            event: 'recrawl',
+            relativePath: '',
+          });
+
+          const {changes} = await waitForItToChange(fileMap);
+
+          // Verify both changes were emitted
+          expect(countFileChanges(changes)).toBe(2);
+          expect([...changes.addedFiles]).toHaveLength(1);
+          expect([...changes.removedFiles]).toHaveLength(1);
+
+          // Verify file system state
+          expect(fileSystem.exists(newFile)).toBe(true);
+          expect(fileSystem.exists(existingFile)).toBe(false);
+
+          // Verify haste map state
+          expect(hasteMap.getModule('Kiwi')).toBe(newFile);
+          expect(mockNodeCrawler).toHaveBeenCalled();
+        },
+        {config: {useWatchman: false}},
+      );
+
+      fm_it(
+        'recrawl event with no changes does not emit',
+        async ({fileMap}) => {
+          await fileMap.build();
+          const fruitsRoot = path.join('/', 'project', 'fruits');
+          const e = mockEmitters[fruitsRoot];
+
+          // Set up node crawler mock to return no changes
+          mockNodeCrawler.mockImplementationOnce(async () => ({
+            changedFiles: new Map<string, FileMetadata>(),
+            removedFiles: new Set<string>(),
+          }));
+
+          const changeListener = jest.fn();
+          fileMap.on('change', changeListener);
+
+          e.emitFileEvent({
+            event: 'recrawl',
+            relativePath: 'nonexistent',
+          });
+
+          // Wait for processing
+          await new Promise(resolve => setTimeout(resolve, 100));
+
+          // Verify crawl was called
+          expect(mockNodeCrawler).toHaveBeenCalled();
+
+          // Verify no change event was emitted (since no changes)
+          expect(changeListener).not.toHaveBeenCalled();
+        },
+        {config: {useWatchman: false}},
+      );
+    });
   });
 });

packages/metro-file-map/src/crawlers/node/index.js

@@ -184,6 +184,7 @@ export default async function nodeCrawl(
     perfLogger,
     roots,
     abortSignal,
+    subpath,
   } = options;
 
   abortSignal?.throwIfAborted();
@@ -198,7 +199,9 @@ export default async function nodeCrawl(
 
   return new Promise((resolve, reject) => {
     const callback: Callback = fileData => {
-      const difference = previousState.fileSystem.getDifference(fileData);
+      const difference = previousState.fileSystem.getDifference(fileData, {
+        subpath,
+      });
 
       perfLogger?.point('nodeCrawl_end');