fix(file-entry-cache): restore v8 reconcile and change-detection semantics (#1648)

packages/file-entry-cache/README.md

@@ -61,8 +61,12 @@ let fileDescriptor = cache.getFileDescriptor('./src/file.txt');
 console.log(fileDescriptor.changed); // true as it is the first time
 console.log(fileDescriptor.key); // './src/file.txt' (stored as provided)
 
+// Repeated calls keep reporting `changed: true` until you persist the state
+// with reconcile(); only then does the file become the cached baseline.
+cache.reconcile();
+
 fileDescriptor = cache.getFileDescriptor('./src/file.txt');
-console.log(fileDescriptor.changed); // false as it has not changed
+console.log(fileDescriptor.changed); // false as it has not changed since reconcile()
 
 // do something to change the file
 fs.writeFileSync('./src/file.txt', 'new data foo bar');

packages/file-entry-cache/src/index.ts

@@ -128,7 +128,23 @@ export function create(
 	if (cacheDirectory) {
 		const cachePath = `${cacheDirectory}/${cacheId}`;
 		if (fs.existsSync(cachePath)) {
-			fileEntryCache.cache = createFlatCacheFile(cachePath, opts.cache);
+			try {
+				fileEntryCache.cache = createFlatCacheFile(cachePath, opts.cache);
+			} catch (error) {
+				// If the cache file content cannot be parsed (e.g. corrupted,
+				// non-JSON, or a legacy/foreign format the parser rejects), start
+				// with an empty cache. The existing file is overwritten on the next
+				// reconcile() rather than throwing. Parse failures surface as a
+				// SyntaxError (malformed JSON) or a TypeError (valid JSON whose shape
+				// the flatted parser rejects, e.g. a top-level object instead of the
+				// expected array). Genuine IO/permission failures (e.g. EISDIR/EACCES)
+				// are re-thrown so valid cache data is not silently discarded.
+				if (error instanceof SyntaxError || error instanceof TypeError) {
+					fileEntryCache.cache = new FlatCache(opts.cache);
+				} else {
+					throw error;
+				}
+			}
 		}
 	}
 
@@ -150,6 +166,15 @@ export class FileEntryCache {
 	private _logger?: ILogger;
 	private _useAbsolutePathAsKey = false;
 	private _useModifiedTime = true;
+	/**
+	 * Snapshot of the persisted meta for each key as of the last load/reconcile.
+	 * Change detection compares against this baseline (not the working cache) so
+	 * that repeated `getFileDescriptor()` calls keep reporting a file as changed
+	 * until the cache is reconciled. The set of keys also tracks which files were
+	 * visited during the current session so that `reconcile()` only updates those.
+	 */
+	private _originalMeta: Map<string, FileDescriptorMeta | undefined> =
+		new Map();
 
 	/**
 	 * Create a new FileEntryCache instance
@@ -203,6 +228,9 @@ export class FileEntryCache {
 	 */
 	public set cache(cache: FlatCache) {
 		this._cache = cache;
+		// The baseline is derived from the cache, so reset it when the cache is
+		// replaced. It will be re-snapshotted lazily on the next getFileDescriptor.
+		this._originalMeta = new Map();
 	}
 
 	/**
@@ -263,6 +291,13 @@ export class FileEntryCache {
 
 	/**
 	 * Set the current working directory
+	 *
+	 * Note: when relative paths are used as cache keys (the default), `cwd` must
+	 * stay stable across a `getFileDescriptor()` / `reconcile()` cycle. Relative
+	 * keys are resolved against the *current* `cwd` each time, so changing it
+	 * mid-run can cause `reconcile()` to resolve a key to a different (missing)
+	 * path and drop the entry. Use absolute keys (`useAbsolutePathAsKey: true`)
+	 * if `cwd` must change during a run.
 	 * @param {string} value - The value to set
 	 */
 	public set cwd(value: string) {
@@ -368,6 +403,7 @@ export class FileEntryCache {
 	 */
 	public destroy() {
 		this._cache.destroy();
+		this._originalMeta = new Map();
 	}
 
 	/**
@@ -378,18 +414,55 @@ export class FileEntryCache {
 	public removeEntry(filePath: string): void {
 		const key = this.createFileKey(filePath);
 		this._cache.removeKey(key);
+		this._originalMeta.delete(key);
 	}
 
 	/**
 	 * Reconcile the cache
 	 * @method reconcile
 	 */
 	public reconcile(): void {
-		const { items } = this._cache;
-		for (const item of items) {
-			const fileDescriptor = this.getFileDescriptor(item.key);
-			if (fileDescriptor.notFound) {
-				this._cache.removeKey(item.key);
+		// Prune entries for files that have been deleted from disk. This mirrors
+		// v8's removeNotFoundFiles() and sweeps ALL cache keys (not only the ones
+		// visited this session) so stale entries for deleted files do not
+		// accumulate over time. It only REMOVES missing files; it never refreshes
+		// the meta of an existing entry, so it does not reintroduce the
+		// "reconcile() revalidates every file" bug (#1648).
+		for (const key of [...this._cache.keys()]) {
+			try {
+				fs.statSync(this.getAbsolutePath(key));
+			} catch (error) {
+				if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+					// The file no longer exists; drop it from the cache.
+					this._cache.removeKey(key);
+					this._originalMeta.delete(key);
+				} else {
+					// Any other failure (e.g. EACCES/EIO, or the path-traversal guard
+					// firing when restrictAccessToCwd is toggled) means we could not
+					// confirm the file's state. Keep the entry rather than discarding
+					// valid cached data, and surface the error instead of silently
+					// dropping it.
+					this._logger?.error(
+						{ key, error },
+						"reconcile: unable to stat file; keeping cached entry",
+					);
+				}
+			}
+		}
+
+		// Promote the inspected meta to the baseline for each file visited via
+		// getFileDescriptor() this session, so that subsequent getFileDescriptor()
+		// calls compare against the freshly reconciled state. Only visited entries
+		// are touched. The inspected meta already holds a consistent size/mtime/hash
+		// snapshot, so it is promoted as-is rather than re-stat'ing (which would
+		// refresh size/mtime but not hash, leaving the baseline inconsistent).
+		for (const key of [...this._originalMeta.keys()]) {
+			const meta = this._cache.getKey<FileDescriptorMeta>(key);
+			if (meta) {
+				this._originalMeta.set(key, { ...meta });
+			} else {
+				// The entry was removed during the session or by the prune above.
+				this._originalMeta.delete(key);
 			}
 		}
 
@@ -499,35 +572,49 @@ export class FileEntryCache {
 			};
 		}
 
-		// If the file is not in the cache, add it
-		if (!metaCache) {
+		// Snapshot the baseline (the persisted state as of the last load/reconcile)
+		// the first time this key is seen in the current session. Change detection
+		// compares against this baseline rather than the working cache so that a
+		// file reported as changed keeps reporting as changed until reconcile().
+		if (!this._originalMeta.has(result.key)) {
+			this._originalMeta.set(
+				result.key,
+				metaCache ? { ...metaCache } : undefined,
+			);
+		}
+
+		const baseline = this._originalMeta.get(result.key);
+
+		// If there is no baseline, the file is new (or has not been reconciled yet)
+		// and is therefore considered changed.
+		if (baseline === undefined) {
 			result.changed = true;
 			this._cache.setKey(result.key, result.meta);
 			this._logger?.debug({ filePath }, "File not in cache, marked as changed");
 			return result;
 		}
 
 		// If the file is in the cache, check if the file has changed
-		if (useModifiedTimeValue && metaCache?.mtime !== result.meta?.mtime) {
+		if (useModifiedTimeValue && baseline.mtime !== result.meta?.mtime) {
 			result.changed = true;
 			this._logger?.debug(
-				{ filePath, oldMtime: metaCache.mtime, newMtime: result.meta.mtime },
+				{ filePath, oldMtime: baseline.mtime, newMtime: result.meta.mtime },
 				"File changed: mtime differs",
 			);
 		}
 
-		if (metaCache?.size !== result.meta?.size) {
+		if (baseline.size !== result.meta?.size) {
 			result.changed = true;
 			this._logger?.debug(
-				{ filePath, oldSize: metaCache.size, newSize: result.meta.size },
+				{ filePath, oldSize: baseline.size, newSize: result.meta.size },
 				"File changed: size differs",
 			);
 		}
 
-		if (useCheckSumValue && metaCache?.hash !== result.meta?.hash) {
+		if (useCheckSumValue && baseline.hash !== result.meta?.hash) {
 			result.changed = true;
 			this._logger?.debug(
-				{ filePath, oldHash: metaCache.hash, newHash: result.meta.hash },
+				{ filePath, oldHash: baseline.hash, newHash: result.meta.hash },
 				"File changed: hash differs",
 			);
 		}
@@ -733,6 +820,11 @@ export class FileEntryCache {
 				const meta = this._cache.getKey(key);
 				this._cache.removeKey(key);
 				this._cache.setKey(newKey, meta);
+				// Keep the change-detection baseline aligned with the renamed key.
+				if (this._originalMeta.has(key)) {
+					this._originalMeta.set(newKey, this._originalMeta.get(key));
+					this._originalMeta.delete(key);
+				}
 			}
 		}
 	}