diff --git a/packages/vinext/src/build/precompress.ts b/packages/vinext/src/build/precompress.ts index aa0a21aadb..e9df5e5907 100644 --- a/packages/vinext/src/build/precompress.ts +++ b/packages/vinext/src/build/precompress.ts @@ -13,8 +13,13 @@ import fsp from "node:fs/promises"; import os from "node:os"; import path from "pathslash"; import zlib from "node:zlib"; +import { randomUUID } from "node:crypto"; import { promisify } from "node:util"; import { ASSET_PREFIX_URL_DIR } from "../utils/asset-prefix.js"; +import { + isPrecompressedVariantBeneficial, + type PrecompressedEncoding, +} from "../utils/precompressed-variant.js"; const brotliCompress = promisify(zlib.brotliCompress); const gzip = promisify(zlib.gzip); @@ -47,7 +52,10 @@ const CONCURRENCY = Math.min(os.availableParallelism(), 8); type PrecompressResult = { filesCompressed: number; totalOriginalBytes: number; - /** Sum of brotli-compressed sizes (used for compression ratio reporting). */ + /** + * Sum of the representation sizes Brotli negotiation would serve. Falls + * back to the original size when Brotli is not beneficial for a file. + */ totalBrotliBytes: number; }; @@ -126,7 +134,10 @@ export async function precompressAssets( // readFile already done before this check — stat()-first would save // the read for tiny files but costs an extra syscall per file; // sub-1KB hashed assets are rare enough that read-first is cheaper. - if (content.length < MIN_SIZE) return; + if (content.length < MIN_SIZE) { + await removeCompressedVariants(fullPath); + return; + } // Compress all variants concurrently within each file const compressions: Promise[] = [ @@ -146,20 +157,21 @@ export async function precompressAssets( const results = await Promise.all(compressions); const [brContent, gzContent, zstdContent] = results; - const writes = [ - fsp.writeFile(fullPath + ".br", brContent), - fsp.writeFile(fullPath + ".gz", gzContent), - ]; - if (zstdContent) { - writes.push(fsp.writeFile(fullPath + ".zst", zstdContent)); - } - await Promise.all(writes); + const outcomes = await Promise.all([ + writeBeneficialVariant(fullPath + ".br", brContent, content.length, "br"), + writeBeneficialVariant(fullPath + ".gz", gzContent, content.length, "gzip"), + zstdContent + ? writeBeneficialVariant(fullPath + ".zst", zstdContent, content.length, "zstd") + : removeFileIfPresent(fullPath + ".zst").then(() => false), + ]); // Increment counters only after all writes succeed, so partial // failures (e.g. ENOSPC mid-write) don't inflate the reported totals. - result.filesCompressed++; - result.totalOriginalBytes += content.length; - result.totalBrotliBytes += brContent.length; + if (outcomes.some(Boolean)) { + result.filesCompressed++; + result.totalOriginalBytes += content.length; + result.totalBrotliBytes += outcomes[0] ? brContent.length : content.length; + } }), ); // Report progress once per chunk to avoid non-deterministic ordering @@ -172,3 +184,37 @@ export async function precompressAssets( return result; } + +async function writeBeneficialVariant( + destination: string, + compressed: Buffer, + originalSize: number, + encoding: PrecompressedEncoding, +): Promise { + if (!isPrecompressedVariantBeneficial(compressed.length, originalSize, encoding)) { + await removeFileIfPresent(destination); + return false; + } + + const temporary = `${destination}.${process.pid}.${randomUUID()}.tmp`; + try { + await fsp.writeFile(temporary, compressed); + await fsp.rename(temporary, destination); + } catch (error) { + await removeFileIfPresent(temporary); + throw error; + } + return true; +} + +async function removeCompressedVariants(fullPath: string): Promise { + await Promise.all([ + removeFileIfPresent(fullPath + ".br"), + removeFileIfPresent(fullPath + ".gz"), + removeFileIfPresent(fullPath + ".zst"), + ]); +} + +async function removeFileIfPresent(filePath: string): Promise { + await fsp.rm(filePath, { force: true }); +} diff --git a/packages/vinext/src/server/static-file-cache.ts b/packages/vinext/src/server/static-file-cache.ts index 88366e9321..e407dbe475 100644 --- a/packages/vinext/src/server/static-file-cache.ts +++ b/packages/vinext/src/server/static-file-cache.ts @@ -14,6 +14,7 @@ import fsp from "node:fs/promises"; import path, { toSlash } from "pathslash"; import { ASSET_PREFIX_URL_DIR } from "../utils/asset-prefix.js"; +import { isPrecompressedVariantBeneficial } from "../utils/precompressed-variant.js"; /** Content-type lookup for static assets. Shared with prod-server.ts. */ export const CONTENT_TYPES: Record = { @@ -64,6 +65,10 @@ export function contentTypeForPath(filePath: string): string { */ const BUFFER_THRESHOLD = 64 * 1024; +/** Temp files left by an interrupted atomic precompression write. */ +const PRECOMPRESSION_TEMP_FILE_RE = + /\.(?:br|gz|zst)\.\d+\.[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\.tmp$/i; + /** A servable file variant with pre-computed response headers. */ type FileVariant = { /** Absolute file path (used for streaming large files). */ @@ -136,8 +141,6 @@ export class StaticFileCache { // Skip .vite/ internal directory if (relativePath.startsWith(".vite/") || relativePath === ".vite") continue; - const ext = path.extname(relativePath); - const contentType = contentTypeForPath(relativePath); // Files under Vite's `assetsDir` are content-hashed. The default // layout writes to `/` (Next.js's canonical // convention); when `assetPrefix` is a path prefix the layout @@ -154,6 +157,14 @@ export class StaticFileCache { const isHashed = relativePath.startsWith(`${ASSET_PREFIX_URL_DIR}/`) || relativePath.includes(`/${ASSET_PREFIX_URL_DIR}/`); + + // A hard process exit can strand the temporary side of an atomic + // precompression write. Never register that internal artifact as a URL, + // but preserve matching user files outside the precompression target. + if (isHashed && PRECOMPRESSION_TEMP_FILE_RE.test(relativePath)) continue; + + const ext = path.extname(relativePath); + const contentType = contentTypeForPath(relativePath); const cacheControl = isHashed ? "public, max-age=31536000, immutable" : "public, max-age=3600"; @@ -190,17 +201,17 @@ export class StaticFileCache { // Pre-compute compressed variant headers (with Content-Encoding, Vary, correct Content-Length) const brInfo = allFiles.get(relativePath + ".br"); - if (brInfo) { + if (brInfo && isPrecompressedVariantBeneficial(brInfo.size, fileInfo.size, "br")) { entry.br = buildVariant(brInfo, baseHeaders, "br"); } const gzInfo = allFiles.get(relativePath + ".gz"); - if (gzInfo) { + if (gzInfo && isPrecompressedVariantBeneficial(gzInfo.size, fileInfo.size, "gzip")) { entry.gz = buildVariant(gzInfo, baseHeaders, "gzip"); } const zstInfo = allFiles.get(relativePath + ".zst"); - if (zstInfo) { + if (zstInfo && isPrecompressedVariantBeneficial(zstInfo.size, fileInfo.size, "zstd")) { entry.zst = buildVariant(zstInfo, baseHeaders, "zstd"); } diff --git a/packages/vinext/src/utils/precompressed-variant.ts b/packages/vinext/src/utils/precompressed-variant.ts new file mode 100644 index 0000000000..df5856e7d6 --- /dev/null +++ b/packages/vinext/src/utils/precompressed-variant.ts @@ -0,0 +1,20 @@ +export type PrecompressedEncoding = "br" | "gzip" | "zstd"; + +const VARY_HEADER_BYTES = "Vary: Accept-Encoding\r\n".length; +const RESPONSE_HEADER_OVERHEAD: Record = { + br: "Content-Encoding: br\r\n".length + VARY_HEADER_BYTES, + gzip: "Content-Encoding: gzip\r\n".length + VARY_HEADER_BYTES, + zstd: "Content-Encoding: zstd\r\n".length + VARY_HEADER_BYTES, +}; + +/** + * Return whether an encoded representation reduces conservative HTTP/1 wire + * bytes after accounting for the response fields needed to negotiate it. + */ +export function isPrecompressedVariantBeneficial( + compressedSize: number, + originalSize: number, + encoding: PrecompressedEncoding, +): boolean { + return originalSize - compressedSize > RESPONSE_HEADER_OVERHEAD[encoding]; +} diff --git a/tests/precompress.test.ts b/tests/precompress.test.ts index 4aa2d3b135..ae31d46bd1 100644 --- a/tests/precompress.test.ts +++ b/tests/precompress.test.ts @@ -11,15 +11,32 @@ import fs from "node:fs"; import path from "node:path"; import os from "node:os"; import zlib from "node:zlib"; +import { createHash } from "node:crypto"; import { precompressAssets } from "../packages/vinext/src/build/precompress.js"; +import { isPrecompressedVariantBeneficial } from "../packages/vinext/src/utils/precompressed-variant.js"; /** Write a file with repeated content to ensure it exceeds compression threshold. */ -async function writeAsset(clientDir: string, relativePath: string, content: string): Promise { +async function writeAsset( + clientDir: string, + relativePath: string, + content: string | Uint8Array, +): Promise { const fullPath = path.join(clientDir, relativePath); await fsp.mkdir(path.dirname(fullPath), { recursive: true }); await fsp.writeFile(fullPath, content); } +function deterministicHighEntropyBytes(size: number): Buffer { + const chunks: Buffer[] = []; + let bytes = 0; + for (let index = 0; bytes < size; index++) { + const chunk = createHash("sha256").update(`vinext-precompress-${index}`).digest(); + chunks.push(chunk); + bytes += chunk.length; + } + return Buffer.concat(chunks).subarray(0, size); +} + describe("precompressAssets", () => { let clientDir: string; @@ -82,6 +99,64 @@ describe("precompressAssets", () => { expect(result.filesCompressed).toBe(0); }); + it("does not emit compressed representations that are not smaller", async () => { + const content = deterministicHighEntropyBytes(4096); + await writeAsset(clientDir, "_next/static/random-aaa111.wasm", content); + + const result = await precompressAssets(clientDir); + + expect(result.filesCompressed).toBe(0); + expect(fs.existsSync(path.join(clientDir, "_next/static/random-aaa111.wasm.br"))).toBe(false); + expect(fs.existsSync(path.join(clientDir, "_next/static/random-aaa111.wasm.gz"))).toBe(false); + expect(fs.existsSync(path.join(clientDir, "_next/static/random-aaa111.wasm.zst"))).toBe(false); + }); + + it("does not emit a marginally smaller representation with larger wire overhead", async () => { + const content = deterministicHighEntropyBytes(4096); + content.copy(content, content.length - 48, 0, 48); + const brotli = zlib.brotliCompressSync(content, { + params: { [zlib.constants.BROTLI_PARAM_QUALITY]: 5 }, + }); + expect(brotli.length).toBeLessThan(content.length); + expect(isPrecompressedVariantBeneficial(brotli.length, content.length, "br")).toBe(false); + + const relativePath = "_next/static/marginal-aaa111.wasm"; + await writeAsset(clientDir, relativePath, content); + await precompressAssets(clientDir); + + expect(fs.existsSync(path.join(clientDir, relativePath + ".br"))).toBe(false); + }); + + it("removes obsolete compressed representations on a later run", async () => { + const relativePath = "_next/static/changing-aaa111.wasm"; + const fullPath = path.join(clientDir, relativePath); + await writeAsset(clientDir, relativePath, "compressible\n".repeat(500)); + await precompressAssets(clientDir); + expect(fs.existsSync(fullPath + ".br")).toBe(true); + + await writeAsset(clientDir, relativePath, deterministicHighEntropyBytes(4096)); + await precompressAssets(clientDir); + + expect(fs.existsSync(fullPath + ".br")).toBe(false); + expect(fs.existsSync(fullPath + ".gz")).toBe(false); + expect(fs.existsSync(fullPath + ".zst")).toBe(false); + }); + + it("removes obsolete variants when an asset falls below the size threshold", async () => { + const relativePath = "_next/static/shrinking-aaa111.js"; + const fullPath = path.join(clientDir, relativePath); + await writeAsset(clientDir, relativePath, "const value = 1;\n".repeat(500)); + await precompressAssets(clientDir); + expect(fs.existsSync(fullPath + ".br")).toBe(true); + + await writeAsset(clientDir, relativePath, "const value = 1;"); + await precompressAssets(clientDir); + + expect(fs.existsSync(fullPath + ".br")).toBe(false); + expect(fs.existsSync(fullPath + ".gz")).toBe(false); + expect(fs.existsSync(fullPath + ".zst")).toBe(false); + }); + it("skips non-compressible file types (images, fonts)", async () => { // PNG file (binary, already compressed) const pngHeader = Buffer.alloc(2048, 0x89); // fake PNG data, above threshold @@ -229,3 +304,17 @@ describe("precompressAssets", () => { expect(result.totalBrotliBytes).toBeLessThan(result.totalOriginalBytes); }); }); + +describe("isPrecompressedVariantBeneficial", () => { + it.each([ + ["br", 45], + ["gzip", 47], + ["zstd", 47], + ] as const)( + "requires %s to save more than its response-header overhead", + (encoding, overhead) => { + expect(isPrecompressedVariantBeneficial(1000 - overhead, 1000, encoding)).toBe(false); + expect(isPrecompressedVariantBeneficial(999 - overhead, 1000, encoding)).toBe(true); + }, + ); +}); diff --git a/tests/serve-static.test.ts b/tests/serve-static.test.ts index 6881c87f5f..1038192c7d 100644 --- a/tests/serve-static.test.ts +++ b/tests/serve-static.test.ts @@ -832,7 +832,9 @@ describe("tryServeStatic (with StaticFileCache)", () => { it("serves cached ranges with conditional precedence and lossless large integers", async () => { const relativePath = "_next/static/range-aaa111.js"; - const content = "0123456789"; + // Keep the compressed sidecar wire-beneficial so this still exercises + // range negotiation against an entry that varies by Accept-Encoding. + const content = "0123456789".repeat(100); await writeFile(clientDir, relativePath, content); await writeFile(clientDir, `${relativePath}.br`, zlib.brotliCompressSync(content)); const cache = await StaticFileCache.create(clientDir); @@ -864,11 +866,11 @@ describe("tryServeStatic (with StaticFileCache)", () => { await tryServeStatic(rangeReq, rangeRes, clientDir, `/${relativePath}`, true, cache); await range.ended; expect(range.status).toBe(206); - expect(range.headers["Content-Range"]).toBe("bytes 2-9/10"); - expect(range.headers["Content-Length"]).toBe("8"); + expect(range.headers["Content-Range"]).toBe("bytes 2-999/1000"); + expect(range.headers["Content-Length"]).toBe("998"); expect(range.headers["Content-Encoding"]).toBeUndefined(); expect(range.headers.Vary).toBe("Accept-Encoding"); - expect(range.body.toString()).toBe("23456789"); + expect(range.body.toString()).toBe(content.slice(2)); const unsatisfiableReq = mockReq(undefined, { range: "bytes=9007199254740992-" }); const { res: unsatisfiableRes, captured: unsatisfiable } = mockRes(); @@ -883,7 +885,7 @@ describe("tryServeStatic (with StaticFileCache)", () => { await unsatisfiable.ended; expect(unsatisfiable.status).toBe(416); expect(unsatisfiable.headers["Content-Type"]).toBe("application/javascript; charset=utf-8"); - expect(unsatisfiable.headers["Content-Range"]).toBe("bytes */10"); + expect(unsatisfiable.headers["Content-Range"]).toBe("bytes */1000"); expect(unsatisfiable.body).toHaveLength(0); const headReq = mockReq(undefined, { range: "bytes=2-5" }, "HEAD"); @@ -891,7 +893,7 @@ describe("tryServeStatic (with StaticFileCache)", () => { await tryServeStatic(headReq, headRes, clientDir, `/${relativePath}`, true, cache); await head.ended; expect(head.status).toBe(206); - expect(head.headers["Content-Range"]).toBe("bytes 2-5/10"); + expect(head.headers["Content-Range"]).toBe("bytes 2-5/1000"); expect(head.headers["Content-Length"]).toBe("4"); expect(head.body).toHaveLength(0); }); diff --git a/tests/static-file-cache.test.ts b/tests/static-file-cache.test.ts index fc62eaa390..ef46e7c557 100644 --- a/tests/static-file-cache.test.ts +++ b/tests/static-file-cache.test.ts @@ -90,6 +90,44 @@ describe("StaticFileCache", () => { expect(cache.lookup("/_next/static/missing-xyz789.js")).toBeUndefined(); }); + it("ignores precompressed variants that are not smaller than the original", async () => { + await writeFile(clientDir, "_next/static/app-abc123.js", "small original"); + await writeFile(clientDir, "_next/static/app-abc123.js.br", "larger brotli representation"); + + const cache = await StaticFileCache.create(clientDir); + const entry = cache.lookup("/_next/static/app-abc123.js"); + + expect(entry?.br).toBeUndefined(); + expect(entry?.original.headers.Vary).toBeUndefined(); + }); + + it("ignores variants whose savings do not cover response-header overhead", async () => { + await writeFile(clientDir, "_next/static/app-abc123.js", "x".repeat(1000)); + await writeFile(clientDir, "_next/static/app-abc123.js.br", "y".repeat(955)); + await writeFile(clientDir, "_next/static/app-abc123.js.gz", "z".repeat(953)); + + const cache = await StaticFileCache.create(clientDir); + const entry = cache.lookup("/_next/static/app-abc123.js"); + + expect(entry?.br).toBeUndefined(); + expect(entry?.gz).toBeUndefined(); + expect(entry?.original.headers.Vary).toBeUndefined(); + }); + + it("does not serve temporary files left by interrupted precompression", async () => { + const basename = "app-abc123.js.br.123.123e4567-e89b-42d3-a456-426614174000.tmp"; + const temporaryPath = `_next/static/${basename}`; + await writeFile(clientDir, temporaryPath, "partial compressed content"); + await writeFile(clientDir, `downloads/${basename}`, "legitimate public content"); + + const cache = await StaticFileCache.create(clientDir); + + expect(cache.lookup("/" + temporaryPath)).toBeUndefined(); + expect(cache.lookup(`/downloads/${basename}`)?.original.buffer?.toString()).toBe( + "legitimate public content", + ); + }); + it("sets immutable cache-control for hashed assets under /assets/", async () => { await writeFile(clientDir, "_next/static/bundle-abc123.js", "x".repeat(100));