From 547b270b7f4c22539aeaff7584ee87921e343002 Mon Sep 17 00:00:00 2001 From: adrians5j Date: Wed, 15 Jul 2026 07:40:01 +0200 Subject: [PATCH] perf(build): skip redundant cache copy when dist is already fresh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On every `yarn build`, the orchestrator restored each cache-hit package by copying its cached output into `dist` — even when `dist` already held the identical bytes from a previous build. For the full package set this is ~13s of pure syscall-bound file copying on every build, dominating the local edit->rebuild loop. Stamp each package's `dist` with the source hash it was built from (`.webiny-build-hash`). On a cache hit, skip the copy when the marker already matches the current source hash; still copy (and re-stamp) when `dist` is missing or stale. A no-op full build drops from ~15s to ~2s. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/buildPackages/src/buildPackages.ts | 13 +++++++++ scripts/buildPackages/src/distBuildHash.ts | 33 ++++++++++++++++++++++ scripts/buildPackages/src/getBatches.ts | 21 ++++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 scripts/buildPackages/src/distBuildHash.ts diff --git a/scripts/buildPackages/src/buildPackages.ts b/scripts/buildPackages/src/buildPackages.ts index 46d88059ccc..f3091dc8b17 100644 --- a/scripts/buildPackages/src/buildPackages.ts +++ b/scripts/buildPackages/src/buildPackages.ts @@ -15,6 +15,7 @@ import path from "path"; import { hideBin } from "yargs/helpers"; import { PackageBuildError } from "./PackageBuildError"; import { queueMetaWrite } from "./writeMetaQueue"; +import { writeDistBuildHash } from "./distBuildHash"; const argv = yargs(hideBin(process.argv)).parse(); @@ -88,6 +89,15 @@ export const buildPackages = async () => { const [pkg] = allPackages; try { await buildPackage(pkg, options.buildOverrides, "inherit", options.safeReplace); + + // Record the source hash (in dist as a marker and in build meta) so + // a later build treats this package as a cache hit and skips the copy. + const sourceHash = await getPackageSourceHash(pkg); + writeDistBuildHash(pkg, sourceHash); + const meta = getBuildMeta(); + meta.packages[pkg.packageJson.name] = { sourceHash }; + writeJsonFileSync(META_FILE_PATH, meta); + sendNotification(`Webiny Build (${projectFolder})`, "Build completed successfully"); } catch (err) { sendNotification(`Webiny Build (${projectFolder})`, "Build failed"); @@ -130,6 +140,9 @@ export const buildPackages = async () => { // Store package hash const sourceHash = await getPackageSourceHash(pkg); + // Stamp dist so a later no-op build can + // skip the cache→dist copy for this package. + writeDistBuildHash(pkg, sourceHash); await queueMetaWrite(async () => { const currentMeta = getBuildMeta(); currentMeta.packages[pkg.packageJson.name] = { diff --git a/scripts/buildPackages/src/distBuildHash.ts b/scripts/buildPackages/src/distBuildHash.ts new file mode 100644 index 00000000000..ec73ab8b711 --- /dev/null +++ b/scripts/buildPackages/src/distBuildHash.ts @@ -0,0 +1,33 @@ +import fs from "fs-extra"; +import path from "path"; +import { getBuildOutputFolder } from "./getBuildOutputFolder"; +import type { Package } from "./types"; + +// Marker file written into a package's build output recording the source hash +// the output was produced from. Lets the orchestrator skip re-copying a cached +// dist into place when the existing dist is already up to date. +const MARKER = ".webiny-build-hash"; + +const markerPath = (pkg: Package) => path.join(getBuildOutputFolder(pkg), MARKER); + +export function readDistBuildHash(pkg: Package): string | null { + const file = markerPath(pkg); + if (!fs.existsSync(file)) { + return null; + } + try { + return fs.readFileSync(file, "utf8").trim(); + } catch { + return null; + } +} + +export function writeDistBuildHash(pkg: Package, hash: string) { + const dir = getBuildOutputFolder(pkg); + fs.ensureDirSync(dir); + fs.writeFileSync(markerPath(pkg), hash); +} + +export function distBuildHashMatches(pkg: Package, hash: string): boolean { + return readDistBuildHash(pkg) === hash; +} diff --git a/scripts/buildPackages/src/getBatches.ts b/scripts/buildPackages/src/getBatches.ts index ad26bfd0697..22ef9396be3 100644 --- a/scripts/buildPackages/src/getBatches.ts +++ b/scripts/buildPackages/src/getBatches.ts @@ -9,6 +9,7 @@ import { getBuildOutputFolder } from "./getBuildOutputFolder"; import { getPackageSourceHash } from "./getPackageSourceHash"; import { getBuildMeta } from "./getBuildMeta"; import { getPackageCacheFolderPath } from "./getPackageCacheFolderPath"; +import { distBuildHashMatches, writeDistBuildHash } from "./distBuildHash"; const { green } = chalk; @@ -23,6 +24,9 @@ export async function getBatches(options: GetBatchesOptions = {}) { const packagesNoCache: Package[] = []; const packagesUseCache: Package[] = []; + // Source hash of each cache-hit package, reused below to skip the cache→dist + // copy when the existing dist was already built from that same hash. + const cacheHitHashes = new Map(); let workspacesPackages = ( getPackages({ @@ -62,6 +66,7 @@ export async function getBatches(options: GetBatchesOptions = {}) { if (packageMeta.sourceHash === sourceHash) { packagesUseCache.push(workspacePackage); + cacheHitHashes.set(workspacePackage.name, sourceHash); } else { packagesNoCache.push(workspacePackage); } @@ -114,10 +119,26 @@ export async function getBatches(options: GetBatchesOptions = {}) { } } + let copied = 0; for (let i = 0; i < packagesUseCache.length; i++) { const workspacePackage = packagesUseCache[i]; + const sourceHash = cacheHitHashes.get(workspacePackage.name)!; + + // Skip the copy when dist was already built/restored from this exact + // source hash — the bytes on disk are already identical. This is the + // common local-dev case (dist persists between builds). + if (distBuildHashMatches(workspacePackage, sourceHash)) { + continue; + } + const cacheFolderPath = path.join(CACHE_FOLDER_PATH, workspacePackage.packageJson.name); fs.copySync(cacheFolderPath, getBuildOutputFolder(workspacePackage)); + writeDistBuildHash(workspacePackage, sourceHash); + copied++; + } + + if (copied > 0) { + console.log(`Restored ${green(copied)} package(s) from cache into dist.`); } } else { if (useCache) {