perf(build): skip cache→dist copy via content hash, drop marker file (#5422)

adrians5j · claude · web-flow · commit 8e71446231fd · 2026-07-16T15:19:15.000+02:00
Co-authored-by: Claude Opus 4.8 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/scripts/buildPackages/src/buildPackages.ts b/scripts/buildPackages/src/buildPackages.ts
@@ -15,7 +15,6 @@ import path from "path";
 import { hideBin } from "yargs/helpers";
 import { PackageBuildError } from "./PackageBuildError";
 import { queueMetaWrite } from "./writeMetaQueue";
-import { writeDistBuildHash, isExperimentalBuildCacheEnabled } from "./distBuildHash";
 
 const argv = yargs(hideBin(process.argv)).parse();
 
@@ -90,16 +89,13 @@ export const buildPackages = async () => {
         try {
             await buildPackage(pkg, options.buildOverrides, "inherit", options.safeReplace);
 
-            // EXPERIMENTAL (opt-in): record the source hash (in dist as a marker
-            // and in build meta) so a later build treats this package as a cache
-            // hit and skips the copy. Off by default — see distBuildHash.ts.
-            if (isExperimentalBuildCacheEnabled()) {
-                const sourceHash = await getPackageSourceHash(pkg);
-                writeDistBuildHash(pkg, sourceHash);
-                const meta = getBuildMeta();
-                meta.packages[pkg.packageJson.name] = { sourceHash };
-                writeJsonFileSync(META_FILE_PATH, meta);
-            }
+            // Record the source hash in build meta so a later build treats this
+            // package as a cache hit (the cache→dist copy is then skipped when
+            // dist already matches — content hash).
+            const sourceHash = await getPackageSourceHash(pkg);
+            const meta = getBuildMeta();
+            meta.packages[pkg.packageJson.name] = { sourceHash };
+            writeJsonFileSync(META_FILE_PATH, meta);
 
             sendNotification(`Webiny Build (${projectFolder})`, "Build completed successfully");
         } catch (err) {
@@ -143,12 +139,6 @@ export const buildPackages = async () => {
 
                                         // Store package hash
                                         const sourceHash = await getPackageSourceHash(pkg);
-                                        // EXPERIMENTAL (opt-in): stamp dist so a
-                                        // later no-op build can skip the
-                                        // cache→dist copy for this package.
-                                        if (isExperimentalBuildCacheEnabled()) {
-                                            writeDistBuildHash(pkg, sourceHash);
-                                        }
                                         await queueMetaWrite(async () => {
                                             const currentMeta = getBuildMeta();
                                             currentMeta.packages[pkg.packageJson.name] = {
diff --git a/scripts/buildPackages/src/buildSinglePackage.ts b/scripts/buildPackages/src/buildSinglePackage.ts
@@ -5,6 +5,7 @@ import { CACHE_FOLDER_PATH } from "./constants";
 import { fork, type StdioOptions } from "child_process";
 import path from "path";
 import { deserializeError } from "serialize-error";
+import { recordCacheHash } from "./distContentHash";
 
 export const buildPackage = async (
     pkg: Package,
@@ -42,4 +43,8 @@ export const buildPackage = async (
     // Delete previous cache!
     await fs.emptyDir(cacheFolderPath);
     await fs.copy(buildFolder, cacheFolderPath);
+
+    // Record the cache's content hash so a later build can skip the cache→dist
+    // copy by hashing dist alone.
+    await recordCacheHash(pkg);
 };
diff --git a/scripts/buildPackages/src/distBuildHash.ts b/scripts/buildPackages/src/distBuildHash.ts
diff --git a/scripts/buildPackages/src/distContentHash.ts b/scripts/buildPackages/src/distContentHash.ts
@@ -0,0 +1,67 @@
+import fs from "fs-extra";
+import path from "path";
+import { hashFolderAsync } from "@webiny/stdlib/node";
+import { getBuildOutputFolder } from "./getBuildOutputFolder";
+import { CACHE_FOLDER_PATH } from "./constants";
+import type { Package } from "./types";
+
+// Sidecar file holding the content hash of a package's cached build, stored
+// next to its cache folder: `.webiny/cached-packages/<name>.hash`. Written when
+// the cache is generated, read at runtime to decide whether dist already
+// matches — so only `dist` is hashed at build time, never the cache twice.
+function storedHashPath(pkg: Package): string {
+    return path.join(CACHE_FOLDER_PATH, `${pkg.packageJson.name}.hash`);
+}
+
+export async function hashBuildOutput(dir: string): Promise<string | null> {
+    if (!fs.existsSync(dir)) {
+        return null;
+    }
+    try {
+        const { hash } = await hashFolderAsync(dir, { excludeFiles: [".webiny-build-hash"] });
+        return hash;
+    } catch {
+        return null;
+    }
+}
+
+export function readStoredCacheHash(pkg: Package): string | null {
+    const file = storedHashPath(pkg);
+    try {
+        return fs.existsSync(file) ? fs.readFileSync(file, "utf8").trim() : null;
+    } catch {
+        return null;
+    }
+}
+
+export function writeStoredCacheHash(pkg: Package, hash: string): void {
+    const file = storedHashPath(pkg);
+    fs.ensureDirSync(path.dirname(file));
+    fs.writeFileSync(file, hash);
+}
+
+/**
+ * Hashes a package's dist and records it as the cache's content hash. Call
+ * right after (re)generating the cache from dist, so a later build can verify
+ * freshness by hashing dist alone.
+ */
+export async function recordCacheHash(pkg: Package): Promise<void> {
+    const hash = await hashBuildOutput(getBuildOutputFolder(pkg));
+    if (hash) {
+        writeStoredCacheHash(pkg, hash);
+    }
+}
+
+/**
+ * Whether the package's current dist already matches the cached build, using
+ * the stored cache hash (no second hashing of the cache). Returns false when
+ * no hash was stored yet (forces a copy, which then records it).
+ */
+export async function distMatchesCache(pkg: Package): Promise<boolean> {
+    const stored = readStoredCacheHash(pkg);
+    if (!stored) {
+        return false;
+    }
+    const distHash = await hashBuildOutput(getBuildOutputFolder(pkg));
+    return distHash !== null && distHash === stored;
+}
diff --git a/scripts/buildPackages/src/getBatches.ts b/scripts/buildPackages/src/getBatches.ts
@@ -9,11 +9,7 @@ import { getBuildOutputFolder } from "./getBuildOutputFolder";
 import { getPackageSourceHash } from "./getPackageSourceHash";
 import { getBuildMeta } from "./getBuildMeta";
 import { getPackageCacheFolderPath } from "./getPackageCacheFolderPath";
-import {
-    distBuildHashMatches,
-    writeDistBuildHash,
-    isExperimentalBuildCacheEnabled
-} from "./distBuildHash";
+import { distMatchesCache, recordCacheHash } from "./distContentHash";
 
 const { green } = chalk;
 
@@ -28,9 +24,6 @@ export async function getBatches(options: GetBatchesOptions = {}) {
 
     const packagesNoCache: Package[] = [];
     const packagesUseCache: Package[] = [];
-    // Source hash of each cache-hit package, reused below to skip the cache→dist
-    // copy when the existing dist was already built from that same hash.
-    const cacheHitHashes = new Map<string, string>();
 
     let workspacesPackages = (
         getPackages({
@@ -70,7 +63,6 @@ export async function getBatches(options: GetBatchesOptions = {}) {
 
             if (packageMeta.sourceHash === sourceHash) {
                 packagesUseCache.push(workspacePackage);
-                cacheHitHashes.set(workspacePackage.name, sourceHash);
             } else {
                 packagesNoCache.push(workspacePackage);
             }
@@ -123,31 +115,30 @@ export async function getBatches(options: GetBatchesOptions = {}) {
             }
         }
 
-        const experimentalCache = isExperimentalBuildCacheEnabled();
+        // Skip the cache→dist copy for packages whose dist already matches the
+        // cache byte-for-byte (content hash). Reads actual bytes in parallel, so
+        // it can't go stale from out-of-band dist writes (`webiny watch`, manual
+        // edits) — those change the hash and force a copy.
+        const fresh = await Promise.all(packagesUseCache.map(pkg => distMatchesCache(pkg)));
 
-        let copied = 0;
+        const restored: Package[] = [];
         for (let i = 0; i < packagesUseCache.length; i++) {
             const workspacePackage = packagesUseCache[i];
-            const sourceHash = cacheHitHashes.get(workspacePackage.name)!;
 
-            // EXPERIMENTAL (opt-in): skip the copy when dist was already
-            // built/restored from this exact source hash — the bytes on disk are
-            // already identical. Off by default; the marker can go stale if
-            // something writes dist out of band (e.g. `webiny watch`).
-            if (experimentalCache && distBuildHashMatches(workspacePackage, sourceHash)) {
+            if (fresh[i]) {
                 continue;
             }
 
             const cacheFolderPath = path.join(CACHE_FOLDER_PATH, workspacePackage.packageJson.name);
             fs.copySync(cacheFolderPath, getBuildOutputFolder(workspacePackage));
-            if (experimentalCache) {
-                writeDistBuildHash(workspacePackage, sourceHash);
-            }
-            copied++;
+            restored.push(workspacePackage);
         }
 
-        if (experimentalCache && copied > 0) {
-            console.log(`Restored ${green(copied)} package(s) from cache into dist.`);
+        if (restored.length) {
+            // dist now equals the cache — record the hash so the next build can
+            // verify freshness by hashing dist alone.
+            await Promise.all(restored.map(pkg => recordCacheHash(pkg)));
+            console.log(`Restored ${green(restored.length)} package(s) from cache into dist.`);
         }
     } else {
         if (useCache) {
