feat(perf): optimize materialization and sync caching (#14)

* perf(verify): optimize file existence checks

* perf: normalization

* perf(sync): cache docs presence checks

* ci: add Node.js 18,20,22 matrix

* ci: run full OS matrix only on PRs

* fix(cache): improve verifyCache error handling

Author: fbosch

.github/workflows/audit.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Setup Node
         uses: actions/setup-node@v4
         with:
-          node-version: 20
+          node-version: 22
           cache: pnpm
 
       - name: Install dependencies

.github/workflows/ci.yml

@@ -13,7 +13,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: ${{ fromJSON(github.event_name == 'pull_request' && '["ubuntu-latest","macos-latest","windows-latest"]' || '["ubuntu-latest"]') }}
+        node-version: [18, 20, 22]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -26,7 +27,7 @@ jobs:
       - name: Setup Node
         uses: actions/setup-node@v4
         with:
-          node-version: 20
+          node-version: ${{ matrix.node-version }}
           cache: pnpm
 
       - name: Install dependencies

.github/workflows/pkg-pr-new-preview.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Setup Node
         uses: actions/setup-node@v4
         with:
-          node-version: 20
+          node-version: 22
           cache: pnpm
 
       - name: Install dependencies

.gitignore

@@ -11,3 +11,4 @@ docs.lock
 coverage
 TODO.md
 .docs/
+benchmarks/

src/materialize.ts

@@ -133,11 +133,14 @@ export const materializeSource = async (params: MaterializeParams) => {
 			onlyFiles: true,
 			followSymbolicLinks: false,
 		});
-		files.sort((left, right) =>
-			normalizePath(left).localeCompare(normalizePath(right)),
-		);
+		const entries = files
+			.map((relativePath) => ({
+				relativePath,
+				normalized: normalizePath(relativePath),
+			}))
+			.sort((left, right) => left.normalized.localeCompare(right.normalized));
 		const targetDirs = new Set<string>();
-		for (const relativePath of files) {
+		for (const { relativePath } of entries) {
 			targetDirs.add(path.dirname(relativePath));
 		}
 		await Promise.all(
@@ -149,7 +152,10 @@ export const materializeSource = async (params: MaterializeParams) => {
 		let fileCount = 0;
 		const concurrency = Math.max(
 			1,
-			Math.min(files.length, Math.max(8, Math.min(128, os.cpus().length * 8))),
+			Math.min(
+				entries.length,
+				Math.max(8, Math.min(128, os.cpus().length * 8)),
+			),
 		);
 		const manifestPath = path.join(tempDir, MANIFEST_FILENAME);
 		const manifestStream = createWriteStream(manifestPath, {
@@ -177,12 +183,11 @@ export const materializeSource = async (params: MaterializeParams) => {
 			});
 		};
 
-		for (let i = 0; i < files.length; i += concurrency) {
-			const batch = files.slice(i, i + concurrency);
+		for (let i = 0; i < entries.length; i += concurrency) {
+			const batch = entries.slice(i, i + concurrency);
 			const results = await Promise.all(
-				batch.map(async (relativePath) => {
-					const relNormalized = normalizePath(relativePath);
-					const filePath = path.join(params.repoDir, relativePath);
+				batch.map(async (entry) => {
+					const filePath = path.join(params.repoDir, entry.relativePath);
 					const fileHandle = await openFileNoFollow(filePath);
 					if (!fileHandle) {
 						return null;
@@ -192,7 +197,7 @@ export const materializeSource = async (params: MaterializeParams) => {
 						if (!stats.isFile()) {
 							return null;
 						}
-						const targetPath = path.join(tempDir, relativePath);
+						const targetPath = path.join(tempDir, entry.relativePath);
 						ensureSafePath(tempDir, targetPath);
 						if (stats.size >= STREAM_COPY_THRESHOLD_BYTES) {
 							const reader = createReadStream(filePath, {
@@ -206,7 +211,7 @@ export const materializeSource = async (params: MaterializeParams) => {
 							await writeFile(targetPath, data);
 						}
 						return {
-							path: relNormalized,
+							path: entry.normalized,
 							size: stats.size,
 						};
 					} finally {

src/sync.ts

@@ -275,6 +275,7 @@ export const runSync = async (options: SyncOptions, deps: SyncDeps = {}) => {
 		const defaults = plan.defaults;
 		const runFetch = deps.fetchSource ?? fetchSource;
 		const runMaterialize = deps.materializeSource ?? materializeSource;
+		const docsPresence = new Map<string, boolean>();
 		const buildJobs = async (ids?: string[], force?: boolean) => {
 			const pick = ids?.length
 				? plan.results.filter((result) => ids.includes(result.id))
@@ -285,9 +286,16 @@ export const runSync = async (options: SyncOptions, deps: SyncDeps = {}) => {
 					if (!source) {
 						return null;
 					}
-					const docsPresent = await hasDocs(plan.cacheDir, result.id);
+					if (force) {
+						return { result, source };
+					}
+					let docsPresent = docsPresence.get(result.id);
+					if (docsPresent === undefined) {
+						docsPresent = await hasDocs(plan.cacheDir, result.id);
+						docsPresence.set(result.id, docsPresent);
+					}
 					const needsMaterialize =
-						force || result.status !== "up-to-date" || !docsPresent;
+						result.status !== "up-to-date" || !docsPresent;
 					return needsMaterialize ? { result, source } : null;
 				}),
 			);

src/verify.ts

@@ -46,13 +46,18 @@ export const verifyCache = async (options: VerifyOptions) => {
 			let sizeMismatchCount = 0;
 			for await (const entry of streamManifestEntries(directory)) {
 				const filePath = path.join(directory, entry.path);
-				if (!(await exists(filePath))) {
-					missingCount += 1;
-					continue;
-				}
-				const info = await stat(filePath);
-				if (info.size !== entry.size) {
-					sizeMismatchCount += 1;
+				try {
+					const info = await stat(filePath);
+					if (info.size !== entry.size) {
+						sizeMismatchCount += 1;
+					}
+				} catch (error) {
+					const code = (error as NodeJS.ErrnoException).code;
+					if (code === "ENOENT" || code === "ENOTDIR") {
+						missingCount += 1;
+						continue;
+					}
+					throw error;
 				}
 			}
 			const issues: string[] = [];
@@ -74,13 +79,17 @@ export const verifyCache = async (options: VerifyOptions) => {
 				ok: issues.length === 0,
 				issues,
 			};
-		} catch (_error) {
-			return {
-				ok: false,
-				issues: [
-					label === "source" ? "missing manifest" : "missing target manifest",
-				],
-			};
+		} catch (error) {
+			const code = (error as NodeJS.ErrnoException).code;
+			if (code === "ENOENT" || code === "ENOTDIR") {
+				return {
+					ok: false,
+					issues: [
+						label === "source" ? "missing manifest" : "missing target manifest",
+					],
+				};
+			}
+			throw error;
 		}
 	};