From f3cc71a68835bd60b1e817cd9fd6ec5518b8dd57 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Thu, 23 Jul 2026 09:10:54 -0400 Subject: [PATCH] fix: stream large staged Playground files --- .../src/mount-materialization.ts | 138 +++++++++++++++--- tests/mount-materialization.test.ts | 56 ++++++- 2 files changed, 171 insertions(+), 23 deletions(-) diff --git a/packages/runtime-playground/src/mount-materialization.ts b/packages/runtime-playground/src/mount-materialization.ts index 7d12fd4a..3d9d3f92 100644 --- a/packages/runtime-playground/src/mount-materialization.ts +++ b/packages/runtime-playground/src/mount-materialization.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto" -import { cp, lstat, mkdir, mkdtemp, readdir, readFile, realpath, rm, stat, writeFile } from "node:fs/promises" +import { cp, lstat, mkdir, mkdtemp, open, readdir, readFile, realpath, rm, stat, writeFile } from "node:fs/promises" import { tmpdir } from "node:os" import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path" import { materializationPhaseResult, namedFileTreeSkipPolicyNames, phpStringArrayLiteral, type MaterializationDiagnostic, type MaterializationPhaseResult, type MountSpec } from "@automattic/wp-codebox-core" @@ -43,7 +43,14 @@ export interface ReadonlyMountStaging { interface HostMountFilePayload { target: string - contentsBase64: string + source: string + size: number +} + +interface HostMountFileVerificationPayload { + target: string + contentsBase64?: string + sha256?: string } interface HostMountDirectoryMaterializationResponse { @@ -70,6 +77,8 @@ interface HostMountFileVerificationResponse { const HOST_MOUNT_FILE_BATCH_SIZE = 100 const HOST_MOUNT_DIRECTORY_BATCH_SIZE = 500 +const HOST_MOUNT_CHUNKED_WRITE_THRESHOLD = 1024 * 1024 +const HOST_MOUNT_WRITE_CHUNK_SIZE = 256 * 1024 /** * Playground's Node filesystem mount handler is writable. Snapshot readonly @@ -256,8 +265,8 @@ async function materializeHostMountToVfs(server: PlaygroundCliServer, mount: Mou let created = 0 let skipped = 0 const directoryBatch: string[] = [] - const fileBatch: HostMountFilePayload[] = [] - const verificationBatch: HostMountFilePayload[] = [] + const fileBatch: Array = [] + const verificationBatch: HostMountFileVerificationPayload[] = [] const flushDirectories = async () => { if (directoryBatch.length === 0) { @@ -290,22 +299,32 @@ async function materializeHostMountToVfs(server: PlaygroundCliServer, mount: Mou skipped += result.skipped } const writeFilePayload = async (payload: HostMountFilePayload) => { + const target = payload.target.trim() + if (!target || target.includes("\0")) { + skipped++ + return + } + if (payload.size > HOST_MOUNT_CHUNKED_WRITE_THRESHOLD) { + const verification = await materializeHostMountFileInChunks(server, { ...payload, target }) + materialized++ + verificationBatch.push(verification) + if (verificationBatch.length >= HOST_MOUNT_FILE_BATCH_SIZE) { + await flushVerificationBatch() + } + return + } + const contents = await readFile(payload.source) + const contentsBase64 = contents.toString("base64") if (!server.playground.writeFile) { - fileBatch.push(payload) + fileBatch.push({ ...payload, contentsBase64 }) if (fileBatch.length >= HOST_MOUNT_FILE_BATCH_SIZE) { await flushFileBatch() } return } - const target = payload.target.trim() - if (!target || target.includes("\0")) { - skipped++ - return - } - const contents = Buffer.from(payload.contentsBase64, "base64") const text = contents.toString("utf8") if (!Buffer.from(text, "utf8").equals(contents)) { - fileBatch.push(payload) + fileBatch.push({ ...payload, contentsBase64 }) if (fileBatch.length >= HOST_MOUNT_FILE_BATCH_SIZE) { await flushFileBatch() } @@ -314,12 +333,12 @@ async function materializeHostMountToVfs(server: PlaygroundCliServer, mount: Mou try { await server.playground.writeFile(target, text) materialized++ - verificationBatch.push(payload) + verificationBatch.push({ target, contentsBase64 }) if (verificationBatch.length >= HOST_MOUNT_FILE_BATCH_SIZE) { await flushVerificationBatch() } } catch { - const fallback = await materializeHostMountFilesWithPhp(server, [payload], []) + const fallback = await materializeHostMountFilesWithPhp(server, [{ ...payload, contentsBase64 }], []) materialized += fallback.materialized created += fallback.created skipped += fallback.skipped @@ -384,7 +403,7 @@ async function* hostMountEntriesForVfs(mount: MountSpec): AsyncGenerator { +async function materializeHostMountFilesWithPhp(server: PlaygroundCliServer, files: Array, directories: string[]): Promise<{ materialized: number; created: number; skipped: number }> { const response = await server.playground.run({ code: hostMountWritePhp(files, directories) }) assertPlaygroundResponseOk("playground-staged-input-write", response) const parsed = parseMaterializationJson(response.text, "wp-codebox/host-mount-materialization/v1", "playground-staged-input-write") @@ -436,7 +456,47 @@ async function materializeHostMountFilesWithPhp(server: PlaygroundCliServer, fil } } -async function verifyHostMountFilesWithPhp(server: PlaygroundCliServer, files: HostMountFilePayload[]): Promise<{ repaired: number; skipped: number }> { +async function materializeHostMountFileInChunks(server: PlaygroundCliServer, file: HostMountFilePayload): Promise { + const truncateResponse = await server.playground.run({ code: hostMountChunkWritePhp(file.target) }) + assertPlaygroundResponseOk("playground-staged-input-chunked-write", truncateResponse) + assertChunkWriteResult(truncateResponse.text, file.target, 0) + + const hash = createHash("sha256") + const handle = await open(file.source, "r") + try { + const buffer = Buffer.allocUnsafe(HOST_MOUNT_WRITE_CHUNK_SIZE) + let position = 0 + let chunk = 0 + while (position < file.size) { + const { bytesRead } = await handle.read(buffer, 0, Math.min(buffer.length, file.size - position), position) + if (bytesRead === 0) { + throw new Error(`playground-staged-input-chunked-write could not read ${boundedTarget(file.target)} after ${position} byte(s)`) + } + const contents = buffer.subarray(0, bytesRead) + hash.update(contents) + const response = await server.playground.run({ code: hostMountChunkWritePhp(file.target, contents.toString("base64")) }) + assertPlaygroundResponseOk("playground-staged-input-chunked-write", response) + assertChunkWriteResult(response.text, file.target, ++chunk) + position += bytesRead + } + } finally { + await handle.close() + } + return { target: file.target, sha256: hash.digest("hex") } +} + +function assertChunkWriteResult(text: string, target: string, chunk: number): void { + const parsed = parseMaterializationJson(text, "wp-codebox/host-mount-chunk-materialization/v1", "playground-staged-input-chunked-write") + if ((parsed.skipped ?? 0) > 0 || (chunk > 0 && (parsed.materialized ?? 0) !== 1)) { + throw new Error(`playground-staged-input-chunked-write could not write ${boundedTarget(target)} at chunk ${chunk}`) + } +} + +function boundedTarget(target: string): string { + return target.length <= 200 ? target : `${target.slice(0, 197)}...` +} + +async function verifyHostMountFilesWithPhp(server: PlaygroundCliServer, files: HostMountFileVerificationPayload[]): Promise<{ repaired: number; skipped: number }> { const response = await server.playground.run({ code: hostMountVerifyPhp(files) }) assertPlaygroundResponseOk("playground-staged-input-verify", response) const parsed = parseMaterializationJson(response.text, "wp-codebox/host-mount-verification/v1", "playground-staged-input-verify") @@ -557,7 +617,7 @@ async function hostFileHashes(directory: string, relativeDirectory = ""): Promis return files } -function hostMountWritePhp(files: HostMountFilePayload[], directories: string[]): string { +function hostMountWritePhp(files: Array, directories: string[]): string { const payload = JSON.stringify(JSON.stringify({ files, directories })) return ` 'wp-codebox/host-mount-materialization/v1', ' ` } -function hostMountVerifyPhp(files: HostMountFilePayload[]): string { +function hostMountVerifyPhp(files: HostMountFileVerificationPayload[]): string { const payload = JSON.stringify(JSON.stringify({ files })) return ` 'wp-codebox/host-mount-verification/v1', 'rep ` } +function hostMountChunkWritePhp(target: string, contentsBase64?: string): string { + const payload = JSON.stringify(JSON.stringify({ target, contentsBase64 })) + return ` 'wp-codebox/host-mount-chunk-materialization/v1', 'materialized' => $materialized, 'skipped' => $skipped), JSON_UNESCAPED_SLASHES); +` +} + function hostMountMkdirPhp(directories: string[]): string { const payload = JSON.stringify(JSON.stringify({ directories })) return ` } { +const chunkedSource = await mkdtemp(join(tmpdir(), "wp-codebox-chunked-materialization-")) + +try { + const sourcePath = join(chunkedSource, "artifact.bin") + const contents = Buffer.alloc(1024 * 1024 + 77) + for (let offset = 0; offset < contents.length; offset++) { + contents[offset] = (offset * 31 + 255) % 256 + } + await writeFile(sourcePath, contents) + + const target = "/wordpress/wp-content/uploads/artifact.bin" + const sandboxFiles = new Map([[target, Buffer.from("stale target contents")]]) + const chunkPayloadLengths: number[] = [] + let directWrites = 0 + let verificationPayload: { sha256?: string; contentsBase64?: string } | undefined + const result = await materializePlaygroundStagedInputs({ + playground: { + async run({ code }: { code: string }) { + const payload = materializationPayload(code) + if (code.includes("wp-codebox/host-mount-chunk-materialization/v1")) { + const chunk = payload as { target: string; contentsBase64?: string } + if (chunk.contentsBase64 === undefined) { + sandboxFiles.set(chunk.target, Buffer.alloc(0)) + return { text: JSON.stringify({ schema: "wp-codebox/host-mount-chunk-materialization/v1", materialized: 0, skipped: 0 }) } + } + chunkPayloadLengths.push(chunk.contentsBase64.length) + sandboxFiles.set(chunk.target, Buffer.concat([sandboxFiles.get(chunk.target) ?? Buffer.alloc(0), Buffer.from(chunk.contentsBase64, "base64")])) + return { text: JSON.stringify({ schema: "wp-codebox/host-mount-chunk-materialization/v1", materialized: 1, skipped: 0 }) } + } + if (code.includes("wp-codebox/host-mount-verification/v1")) { + verificationPayload = payload.files?.[0] + const expected = verificationPayload?.sha256 + const actual = sandboxFiles.has(target) ? createHash("sha256").update(sandboxFiles.get(target)!).digest("hex") : undefined + return { text: JSON.stringify({ schema: "wp-codebox/host-mount-verification/v1", repaired: 0, skipped: expected === actual ? 0 : 1 }) } + } + return { text: JSON.stringify({ schema: "wp-codebox/host-mount-directory-materialization/v1", created: payload.directories?.length ?? 0, skipped: 0 }) } + }, + async writeFile() { + directWrites++ + }, + }, + } as never, [{ type: "file", source: sourcePath, target, mode: "readwrite" }]) + + assert.equal(result.materialized, 1) + assert.equal(directWrites, 0, "large files never use the whole-file direct writer") + assert.deepEqual(sandboxFiles.get(target), contents, "chunked writes preserve exact binary bytes and replace existing targets") + assert.deepEqual(chunkPayloadLengths, [349528, 349528, 349528, 349528, 104]) + assert.equal(verificationPayload?.contentsBase64, undefined, "large-file verification does not embed the file contents") + assert.match(verificationPayload?.sha256 ?? "", /^[a-f0-9]{64}$/) +} finally { + await rm(chunkedSource, { recursive: true, force: true }) +} + +function materializationPayload(code: string): { target?: string; contentsBase64?: string; directories?: string[]; files?: Array<{ target: string; contentsBase64?: string; sha256?: string }> } { const match = code.match(/\$payload = json_decode\((.*), true\);/) assert.ok(match, "materialization PHP includes a JSON payload") return JSON.parse(JSON.parse(match[1]))