diff --git a/.github/workflows/agent-task-contracts.yml b/.github/workflows/agent-task-contracts.yml index 3c7cabac1..8825fb5e5 100644 --- a/.github/workflows/agent-task-contracts.yml +++ b/.github/workflows/agent-task-contracts.yml @@ -96,6 +96,7 @@ jobs: - run: npm run test:native-agent-task-playground-e2e - run: npm run test:native-agent-task-interruption - run: npm run test:trusted-apply-artifact-channel + - run: npm run test:runtime-command-artifact-bounds - run: npm run test:redaction - run: npm run test:production-boundary-enforcement - run: npm run test:runtime-tool-policy diff --git a/package.json b/package.json index be0f4c7cc..76c6e3d89 100644 --- a/package.json +++ b/package.json @@ -154,6 +154,7 @@ "test:workspace-preload-artifacts": "tsx tests/workspace-preload-artifacts.test.ts", "test:runner-workspace-apply": "tsx tests/runner-workspace-apply.test.ts", "test:trusted-apply-artifact-channel": "tsx tests/trusted-apply-artifact-channel.integration.test.ts", + "test:runtime-command-artifact-bounds": "tsx tests/runtime-command-artifact-bounds.test.ts", "test:runner-workspace-publisher": "node tests/runner-workspace-publisher.test.mjs", "test:native-agent-task-lifecycle": "node tests/execute-native-agent-task-lifecycle.test.mjs", "test:native-agent-task-interruption": "node tests/execute-native-agent-task-interruption.test.mjs", diff --git a/packages/cli/src/commands/recipe-run-output.ts b/packages/cli/src/commands/recipe-run-output.ts index 4f0e222ee..8cdd50e91 100644 --- a/packages/cli/src/commands/recipe-run-output.ts +++ b/packages/cli/src/commands/recipe-run-output.ts @@ -1,5 +1,6 @@ import { setTimeout as delay } from "node:timers/promises" -import type { RecipeRunSummary, RuntimeRunRecord } from "@automattic/wp-codebox-core" +import type { ExecutionResult, RecipeRunSummary, RuntimeRunRecord } from "@automattic/wp-codebox-core" +import { boundedExecutionResultsForArtifacts } from "@automattic/wp-codebox-core/internals" import { serializeError } from "../output.js" import type { RunOutput } from "../runtime-command-wrappers.js" import { RecipePhaseError } from "./recipe-run-phases.js" @@ -95,7 +96,14 @@ function hasTerminalRecipePhaseFailure(output: RecipeRunOutput): boolean { } export async function writeRecipeJsonOutput(output: unknown): Promise { - await writeStdout(`${JSON.stringify(output, null, 2)}\n`) + await writeStdout(`${JSON.stringify(boundedRecipeJsonOutput(output), null, 2)}\n`) +} + +export function boundedRecipeJsonOutput(output: unknown): unknown { + if (!output || typeof output !== "object" || Array.isArray(output)) return output + const record = output as Record + if (!Array.isArray(record.executions)) return output + return { ...record, executions: boundedExecutionResultsForArtifacts(record.executions as ExecutionResult[]) } } export async function writeRecipeSummaryHumanOutput(summary: RecipeRunSummary): Promise { diff --git a/packages/cli/src/commands/recipe-run.ts b/packages/cli/src/commands/recipe-run.ts index ef1412b08..c219072b1 100644 --- a/packages/cli/src/commands/recipe-run.ts +++ b/packages/cli/src/commands/recipe-run.ts @@ -327,7 +327,7 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe await awaitRecipe("runtime.observe:runtime-info", runtime!.observe({ type: "runtime-info" })) await awaitRecipe("runtime.observe:mounts", runtime!.observe({ type: "mounts" })) runRecord = await runRegistry.update(runRecord.runId, { status: "collecting_artifacts", runtime: await runtime!.info() }) - artifacts = await awaitRecipe("runtime.collect-artifacts", collectRecipeRuntimeArtifacts(runtime!, { includeLogs: true, includeObservations: true }, { snapshotTimeoutMs: SUCCESSFUL_RECIPE_RUNTIME_SNAPSHOT_TIMEOUT_MS })) + artifacts = await awaitRecipe("runtime.collect-artifacts", collectRecipeRuntimeArtifacts(runtime!, { includeLogs: true, includeObservations: true }, { snapshotTimeoutMs: SUCCESSFUL_RECIPE_RUNTIME_SNAPSHOT_TIMEOUT_MS, activeExecution: executions.at(-1) })) browserEvidence = await recipeBrowserEvidence(artifacts, executions, recipe) await artifactPointer.update({ runtime: await runtime!.info(), artifacts, phases: phaseTracker.list(), browserEvidence }) await materializeTypedRecipeDeclaredArtifacts(artifacts, declaredArtifacts) @@ -350,7 +350,7 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe const recipeFailure = strictFailure ?? agentFailure ?? verifyFailure const successfulRecipe = !recipeFailure if (successfulRecipe && options.previewHoldSeconds) { - artifacts = await awaitRecipe("runtime.collect-artifacts.preview-hold", collectRecipeRuntimeArtifacts(runtime, { includeLogs: true, includeObservations: true, previewHoldSeconds: options.previewHoldSeconds }, { snapshotTimeoutMs: SUCCESSFUL_RECIPE_RUNTIME_SNAPSHOT_TIMEOUT_MS })) + artifacts = await awaitRecipe("runtime.collect-artifacts.preview-hold", collectRecipeRuntimeArtifacts(runtime, { includeLogs: true, includeObservations: true, previewHoldSeconds: options.previewHoldSeconds }, { snapshotTimeoutMs: SUCCESSFUL_RECIPE_RUNTIME_SNAPSHOT_TIMEOUT_MS, activeExecution: executions.at(-1) })) browserEvidence = await recipeBrowserEvidence(artifacts, executions, recipe) await artifactPointer.update({ runtime: await runtime.info(), artifacts, phases: phaseTracker.list(), browserEvidence }) declaredArtifacts = await collectRecipeDeclaredArtifacts(recipe, runtime) diff --git a/packages/cli/src/recipe-evidence.ts b/packages/cli/src/recipe-evidence.ts index 92aba304f..c9b59f3ab 100644 --- a/packages/cli/src/recipe-evidence.ts +++ b/packages/cli/src/recipe-evidence.ts @@ -301,6 +301,31 @@ export interface RecipeArtifactsFinalizationController { interface RecipeRuntimeArtifactCollectionOptions { timeoutMs?: number snapshotTimeoutMs?: number + activeExecution?: RecipeEvidenceExecutionResult +} + +export class RecipeRuntimeArtifactCollectionError extends Error { + readonly code = "recipe-artifact-collection-failed" + readonly operation = "runtime.collect-artifacts" + readonly subsystem = "artifact-bundle" + readonly workflowStepIndex?: number + readonly command?: string + readonly artifactPath?: string + readonly payload?: unknown + readonly limits?: unknown + readonly causeStack?: string + + constructor(activeExecution: RecipeEvidenceExecutionResult | undefined, cause: Error) { + const causeRecord = cause as Error & Record + super(`Recipe artifact collection failed${activeExecution?.recipeStepIndex === undefined ? "" : ` at workflow step ${activeExecution.recipeStepIndex}`}: ${cause.message}`, { cause }) + this.name = "RecipeRuntimeArtifactCollectionError" + this.workflowStepIndex = activeExecution?.recipeStepIndex + this.command = activeExecution?.recipeCommand ?? activeExecution?.command + this.artifactPath = typeof causeRecord.artifactPath === "string" ? causeRecord.artifactPath : undefined + this.payload = causeRecord.payload + this.limits = causeRecord.limits + this.causeStack = typeof causeRecord.causeStack === "string" ? causeRecord.causeStack : cause.stack?.slice(0, 16 * 1024) + } } const execFileAsync = promisify(execFile) @@ -322,7 +347,7 @@ export async function collectAndFinalizeFailedRecipeArtifacts(args: { if (!artifacts) { try { - artifacts = await collectRecipeRuntimeArtifacts(args.runtime, { includeLogs: true, includeObservations: true }, { snapshotTimeoutMs: 20_000, timeoutMs: 30_000 }) + artifacts = await collectRecipeRuntimeArtifacts(args.runtime, { includeLogs: true, includeObservations: true }, { snapshotTimeoutMs: 20_000, timeoutMs: 30_000, activeExecution: args.executions.at(-1) }) } catch { return undefined } @@ -354,11 +379,19 @@ export async function collectRecipeRuntimeArtifacts(runtime: Runtime, spec: Arti } } - const artifacts = runtime.collectArtifacts(spec) - if (options.timeoutMs && options.timeoutMs > 0) { - return timeoutOrReject(artifacts, options.timeoutMs, `Runtime artifact collection exceeded ${options.timeoutMs}ms`) + try { + const artifacts = runtime.collectArtifacts(spec) + if (options.timeoutMs && options.timeoutMs > 0) { + return await timeoutOrReject(artifacts, options.timeoutMs, `Runtime artifact collection exceeded ${options.timeoutMs}ms`) + } + return await artifacts + } catch (error) { + const candidate = error instanceof Error ? error as Error & Record : undefined + if (candidate?.code === "command-artifact-collection-failed" || error instanceof RangeError) { + throw new RecipeRuntimeArtifactCollectionError(options.activeExecution, candidate ?? Object.assign(new Error(String(error)), { code: "runtime-allocation-failed" })) + } + throw error } - return artifacts } async function timeoutOrUndefined(promise: Promise, timeoutMs: number): Promise { diff --git a/packages/runtime-core/src/bounded-execution-results.ts b/packages/runtime-core/src/bounded-execution-results.ts new file mode 100644 index 000000000..b2bc0ffc3 --- /dev/null +++ b/packages/runtime-core/src/bounded-execution-results.ts @@ -0,0 +1,168 @@ +import type { ExecutionResult } from "./runtime-contracts.js" +import { normalizeJsonValue } from "./object-utils.js" + +export const COMMAND_ARTIFACT_STRING_MAX_BYTES = 1024 * 1024 +export const COMMAND_ARTIFACT_COMMAND_STRING_MAX_BYTES = 2 * 1024 * 1024 +export const COMMAND_ARTIFACT_TOTAL_STRING_MAX_BYTES = 16 * 1024 * 1024 +export const COMMAND_ARTIFACT_MAX_NODES = 25_000 +export const COMMAND_ARTIFACT_MAX_RECORDS = 1_000 +const COMMAND_ARTIFACT_MAX_TRUNCATIONS = 1_000 +const COMMAND_ARTIFACT_KEY_MAX_BYTES = 256 +const COMMAND_ARTIFACT_IDENTITY_MAX_BYTES = 1024 +const COMMAND_ARTIFACT_PATH_MAX_BYTES = 4096 +const EXECUTION_RESULT_KEYS = new Set(["id", "command", "args", "exitCode", "stdout", "stderr", "result", "diagnostics", "artifactRefs", "startedAt", "finishedAt", "artifactCapture"]) +const EXECUTION_IDENTITY_EXTENSION_KEYS = new Set(["recipePhase", "recipeCommand", "fuzzCaseId", "fuzzPhase"]) + +export interface CommandArtifactTruncation { + path: string + reason: "string-byte-limit" | "command-string-byte-limit" | "total-string-byte-limit" | "node-limit" + observedBytes?: number + capturedBytes?: number + configuredLimitBytes?: number +} + +export interface CommandArtifactCapture { + schema: "wp-codebox/command-artifact-capture/v1" + truncated: true + limits: { + capturedStringBytesPerValue: number + capturedStringBytesPerCommand: number + capturedStringBytesTotal: number + nodes: number + records: number + } + fields: CommandArtifactTruncation[] + omittedFieldCount?: number + omittedCommandCount?: number +} + +export type BoundedExecutionResult = T & { artifactCapture?: CommandArtifactCapture } + +export function boundedExecutionResultsForArtifacts(commands: T[]): Array> { + const budget = { + remainingBytes: COMMAND_ARTIFACT_TOTAL_STRING_MAX_BYTES, + remainingCommandBytes: COMMAND_ARTIFACT_COMMAND_STRING_MAX_BYTES, + remainingNodes: COMMAND_ARTIFACT_MAX_NODES, + remainingTruncations: COMMAND_ARTIFACT_MAX_TRUNCATIONS, + } + const selectedCommands = commands.length <= COMMAND_ARTIFACT_MAX_RECORDS + ? commands + : [...commands.slice(0, COMMAND_ARTIFACT_MAX_RECORDS - 1), commands.at(-1)!] + + return selectedCommands.map((command, selectedIndex) => { + budget.remainingCommandBytes = COMMAND_ARTIFACT_COMMAND_STRING_MAX_BYTES + const fields: CommandArtifactTruncation[] = [] + let omittedFieldCount = 0 + const capture = (value: unknown, path: string): unknown => boundedArtifactValue(value, path, budget, (field) => { + if (fields.length < 100 && budget.remainingTruncations > 0) { + fields.push(field) + budget.remainingTruncations -= 1 + } else { + omittedFieldCount += 1 + } + }) + const extensions = Object.fromEntries( + Object.entries(command) + .filter(([key]) => !EXECUTION_RESULT_KEYS.has(key)) + .map(([key, value]) => [key, typeof value === "string" && EXECUTION_IDENTITY_EXTENSION_KEYS.has(key) ? boundedIdentityString(value) : capture(normalizeJsonValue(value), key)]), + ) + const projected = { + ...extensions, + id: boundedIdentityString(command.id), + command: boundedIdentityString(command.command), + args: capture(normalizeJsonValue(command.args), "args") as string[], + exitCode: command.exitCode, + stdout: capture(command.stdout, "stdout") as string, + stderr: capture(command.stderr, "stderr") as string, + ...(command.result === undefined ? {} : { result: capture(normalizeJsonValue(command.result), "result") as ExecutionResult["result"] }), + ...(command.diagnostics === undefined ? {} : { diagnostics: capture(normalizeJsonValue(command.diagnostics), "diagnostics") }), + ...(command.artifactRefs === undefined ? {} : { artifactRefs: capture(normalizeJsonValue(command.artifactRefs), "artifactRefs") as ExecutionResult["artifactRefs"] }), + startedAt: boundedIdentityString(command.startedAt), + finishedAt: boundedIdentityString(command.finishedAt), + } as BoundedExecutionResult + const omittedCommandCount = commands.length - selectedCommands.length + if (fields.length > 0 || omittedFieldCount > 0 || (omittedCommandCount > 0 && selectedIndex === selectedCommands.length - 1)) { + projected.artifactCapture = { + schema: "wp-codebox/command-artifact-capture/v1", + truncated: true, + limits: { + capturedStringBytesPerValue: COMMAND_ARTIFACT_STRING_MAX_BYTES, + capturedStringBytesPerCommand: COMMAND_ARTIFACT_COMMAND_STRING_MAX_BYTES, + capturedStringBytesTotal: COMMAND_ARTIFACT_TOTAL_STRING_MAX_BYTES, + nodes: COMMAND_ARTIFACT_MAX_NODES, + records: COMMAND_ARTIFACT_MAX_RECORDS, + }, + fields, + ...(omittedFieldCount > 0 ? { omittedFieldCount } : {}), + ...(omittedCommandCount > 0 && selectedIndex === selectedCommands.length - 1 ? { omittedCommandCount } : {}), + } + } + return projected + }) +} + +function boundedArtifactValue( + value: unknown, + path: string, + budget: { remainingBytes: number; remainingCommandBytes: number; remainingNodes: number; remainingTruncations: number }, + recordTruncation: (field: CommandArtifactTruncation) => void, +): unknown { + if (value === null || value === undefined || (typeof value !== "string" && typeof value !== "object")) { + return value + } + if (budget.remainingNodes <= 0) { + recordTruncation({ path: truncateUtf8(path, COMMAND_ARTIFACT_PATH_MAX_BYTES), reason: "node-limit" }) + if (typeof value === "string") return "" + return Array.isArray(value) ? [] : {} + } + budget.remainingNodes -= 1 + if (typeof value === "string") { + const observedBytes = Buffer.byteLength(value, "utf8") + const totalBudgetLimited = budget.remainingBytes < Math.min(observedBytes, COMMAND_ARTIFACT_STRING_MAX_BYTES) + const commandBudgetLimited = budget.remainingCommandBytes < Math.min(observedBytes, COMMAND_ARTIFACT_STRING_MAX_BYTES) + const configuredLimitBytes = Math.min(COMMAND_ARTIFACT_STRING_MAX_BYTES, budget.remainingCommandBytes, budget.remainingBytes) + const captured = truncateUtf8(value, configuredLimitBytes) + const capturedBytes = Buffer.byteLength(captured, "utf8") + budget.remainingBytes -= capturedBytes + budget.remainingCommandBytes -= capturedBytes + if (capturedBytes < observedBytes) { + recordTruncation({ + path: truncateUtf8(path, COMMAND_ARTIFACT_PATH_MAX_BYTES), + reason: totalBudgetLimited ? "total-string-byte-limit" : commandBudgetLimited ? "command-string-byte-limit" : "string-byte-limit", + observedBytes, + capturedBytes, + configuredLimitBytes, + }) + } + return captured + } + if (Array.isArray(value)) { + return value.map((item, index) => boundedArtifactValue(item, `${path}[${index}]`, budget, recordTruncation)) + } + if (value && typeof value === "object") { + return Object.fromEntries(Object.entries(value).map(([key, item]) => { + const capturedKey = truncateUtf8(key, COMMAND_ARTIFACT_KEY_MAX_BYTES) + return [capturedKey, boundedArtifactValue(item, `${path}.${capturedKey}`, budget, recordTruncation)] + })) + } + return value +} + +function boundedIdentityString(value: string): string { + return truncateUtf8(value, COMMAND_ARTIFACT_IDENTITY_MAX_BYTES) +} + +export function truncateUtf8(value: string, maxBytes: number): string { + if (maxBytes <= 0) return "" + if (Buffer.byteLength(value, "utf8") <= maxBytes) return value + + let low = 0 + let high = Math.min(value.length, maxBytes) + while (low < high) { + const middle = Math.ceil((low + high) / 2) + if (Buffer.byteLength(value.slice(0, middle), "utf8") <= maxBytes) low = middle + else high = middle - 1 + } + const captured = value.slice(0, low) + return /[\uD800-\uDBFF]$/.test(captured) ? captured.slice(0, -1) : captured +} diff --git a/packages/runtime-core/src/internals.ts b/packages/runtime-core/src/internals.ts index cbe906671..d6b12326f 100644 --- a/packages/runtime-core/src/internals.ts +++ b/packages/runtime-core/src/internals.ts @@ -4,6 +4,7 @@ * subpath instead. */ export * from "./benchmark-substrate.js" +export * from "./bounded-execution-results.js" export * from "./fanout-aggregation.js" export * from "./fanout-execution.js" export * from "./object-utils.js" diff --git a/packages/runtime-playground/src/artifact-bundle-builder.ts b/packages/runtime-playground/src/artifact-bundle-builder.ts index bb3bcd7d4..a6f51cfb1 100644 --- a/packages/runtime-playground/src/artifact-bundle-builder.ts +++ b/packages/runtime-playground/src/artifact-bundle-builder.ts @@ -42,7 +42,7 @@ import { previewLeaseSummary, type Snapshot, } from "@automattic/wp-codebox-core" -import { normalizeJsonValue, stripUndefined } from "@automattic/wp-codebox-core/internals" +import { COMMAND_ARTIFACT_COMMAND_STRING_MAX_BYTES, COMMAND_ARTIFACT_MAX_NODES, COMMAND_ARTIFACT_MAX_RECORDS, COMMAND_ARTIFACT_STRING_MAX_BYTES, COMMAND_ARTIFACT_TOTAL_STRING_MAX_BYTES, boundedExecutionResultsForArtifacts, normalizeJsonValue, stripUndefined, truncateUtf8, type BoundedExecutionResult } from "@automattic/wp-codebox-core/internals" import type { BrowserArtifact } from "./browser-artifacts.js" import { firstCommandWordPressAdminAuthRequirement } from "./command-auth-requirements.js" import { writeTrustedApplyArtifacts } from "./trusted-apply-artifact-channel.js" @@ -92,7 +92,7 @@ export interface ArtifactBundleBuilderSource { pluginCheckManifestFiles(): ArtifactManifestFile[] themeCheckManifestFiles(): ArtifactManifestFile[] formatRuntimeLog(): string - formatCommandsLog(): string + formatCommandsLog(commands: ExecutionResult[]): string recordArtifactsCollected(bundleId: string, createdAt: string, spec: ArtifactSpec): void } @@ -366,11 +366,25 @@ export class ArtifactBundleBuilder { if (replaySnapshot) { await writeRedactedArtifact(redactor, partialBlueprintAfterPath, source.artifactRoot, artifactJson(partialBlueprintAfter)) } + let artifactCommands: BoundedExecutionResult[] + try { + artifactCommands = boundedExecutionResultsForArtifacts(source.commands) + } catch (error) { + throw commandArtifactCollectionError(source.commands, "commands.jsonl", error) + } await writeJsonLines(eventsPath, source.events, redactor, source.artifactRoot) - await writeJsonLines(commandsPath, source.commands, redactor, source.artifactRoot) + try { + await writeJsonLines(commandsPath, artifactCommands, redactor, source.artifactRoot) + } catch (error) { + throw commandArtifactCollectionError(source.commands, "commands.jsonl", error) + } await writeJsonLines(observationsPath, source.observations, redactor, source.artifactRoot) await writeRedactedArtifact(redactor, runtimeLogPath, source.artifactRoot, source.formatRuntimeLog()) - await writeRedactedArtifact(redactor, commandsLogPath, source.artifactRoot, source.formatCommandsLog()) + try { + await writeRedactedArtifact(redactor, commandsLogPath, source.artifactRoot, source.formatCommandsLog(artifactCommands)) + } catch (error) { + throw commandArtifactCollectionError(source.commands, "logs/commands.log", error) + } await writeRedactedArtifact(redactor, mountsPath, source.artifactRoot, artifactJson(source.mounts)) await writeRedactedArtifact(redactor, capturedMountsPath, source.artifactRoot, artifactJson(serializeCapturedMountFiles(capturedMounts))) await writeRedactedArtifact(redactor, diffsPath, source.artifactRoot, artifactJson(mountDiffs)) @@ -758,6 +772,31 @@ async function writeJsonLines(path: string, records: unknown[], redactor: Artifa await writeRedactedArtifact(redactor, path, artifactRoot, artifactJsonLines(records.map((record) => normalizeJsonValue(record)))) } +function commandArtifactCollectionError(commands: ExecutionResult[], artifactPath: string, cause: unknown): Error { + const activeCommand = commands.at(-1) + return Object.assign(new Error(`Command artifact collection failed while writing ${artifactPath}`, { cause }), { + name: "CommandArtifactCollectionError", + code: "command-artifact-collection-failed", + operation: "serialize-command-evidence", + subsystem: "artifact-bundle", + artifactPath, + payload: { + commandCount: commands.length, + stdoutBytes: commands.reduce((total, command) => total + Buffer.byteLength(command.stdout, "utf8"), 0), + stderrBytes: commands.reduce((total, command) => total + Buffer.byteLength(command.stderr, "utf8"), 0), + }, + limits: { + capturedStringBytesPerValue: COMMAND_ARTIFACT_STRING_MAX_BYTES, + capturedStringBytesPerCommand: COMMAND_ARTIFACT_COMMAND_STRING_MAX_BYTES, + capturedStringBytesTotal: COMMAND_ARTIFACT_TOTAL_STRING_MAX_BYTES, + nodes: COMMAND_ARTIFACT_MAX_NODES, + records: COMMAND_ARTIFACT_MAX_RECORDS, + }, + ...(activeCommand ? { commandIndex: commands.length - 1, command: activeCommand.command, commandId: activeCommand.id } : {}), + causeStack: cause instanceof Error ? truncateUtf8(cause.stack ?? `${cause.name}: ${cause.message}`, 16 * 1024) : undefined, + }) +} + function artifactJson(value: unknown): string { return `${JSON.stringify(normalizeJsonValue(value), null, 2)}\n` } diff --git a/packages/runtime-playground/src/runtime-artifact-helpers.ts b/packages/runtime-playground/src/runtime-artifact-helpers.ts index eb963d0a4..b4a14090f 100644 --- a/packages/runtime-playground/src/runtime-artifact-helpers.ts +++ b/packages/runtime-playground/src/runtime-artifact-helpers.ts @@ -81,7 +81,7 @@ export async function collectPlaygroundArtifacts({ pluginCheckManifestFiles: () => pluginCheckManifestFiles(artifactRoot, pluginChecks), themeCheckManifestFiles: () => themeCheckManifestFiles(artifactRoot, themeChecks), formatRuntimeLog: () => formatRuntimeLog(events), - formatCommandsLog: () => formatCommandsLog(commands), + formatCommandsLog: (artifactCommands) => formatCommandsLog(artifactCommands), recordArtifactsCollected, }).build(artifactSpec) } diff --git a/tests/runtime-command-artifact-bounds.test.ts b/tests/runtime-command-artifact-bounds.test.ts new file mode 100644 index 000000000..a63b9dc83 --- /dev/null +++ b/tests/runtime-command-artifact-bounds.test.ts @@ -0,0 +1,192 @@ +import assert from "node:assert/strict" +import { Buffer } from "node:buffer" +import { mkdtemp, readFile, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { + COMMAND_ARTIFACT_STRING_MAX_BYTES, + COMMAND_ARTIFACT_COMMAND_STRING_MAX_BYTES, + COMMAND_ARTIFACT_MAX_NODES, + COMMAND_ARTIFACT_MAX_RECORDS, + COMMAND_ARTIFACT_TOTAL_STRING_MAX_BYTES, + boundedExecutionResultsForArtifacts, +} from "../packages/runtime-core/src/bounded-execution-results.js" +import { collectPlaygroundArtifacts, formatCommandsLog } from "../packages/runtime-playground/src/runtime-artifact-helpers.js" +import { collectRecipeRuntimeArtifacts } from "../packages/cli/src/recipe-evidence.js" +import { boundedRecipeJsonOutput, serializeRecipeRunError } from "../packages/cli/src/commands/recipe-run-output.js" +import type { ExecutionResult } from "../packages/runtime-core/src/runtime-contracts.js" + +const oversized = "x".repeat(COMMAND_ARTIFACT_STRING_MAX_BYTES + 1024) +const command: ExecutionResult = { + id: "command-7", + command: "wordpress.apply-site-plan", + args: ["--plan", "fixture.json"], + exitCode: 0, + stdout: oversized, + stderr: "", + result: { + schema: "wp-codebox/runtime-command-result/v1", + status: "ok", + stdout: oversized, + json: { artifact: oversized }, + }, + diagnostics: { retained: oversized }, + startedAt: "2026-07-21T20:00:00.000Z", + finishedAt: "2026-07-21T20:01:00.000Z", +} + +const projected = boundedExecutionResultsForArtifacts([command]) +const record = projected[0] +assert(record, "the command remains represented") +assert.equal(record.id, command.id) +assert.equal(record.command, command.command) +assert.equal(record.startedAt, command.startedAt) +assert.equal(Buffer.byteLength(record.stdout), COMMAND_ARTIFACT_STRING_MAX_BYTES) +assert.equal(Buffer.byteLength(record.result?.stdout ?? ""), 0) +assert(record.artifactCapture?.truncated) +assert.deepEqual(record.artifactCapture?.fields.map(({ path }) => path), [ + "stdout", + "result.json.artifact", + "result.schema", + "result.status", + "result.stdout", + "diagnostics.retained", +]) +assert.deepEqual(record.artifactCapture?.fields.map(({ reason }) => reason), [ + "string-byte-limit", + "command-string-byte-limit", + "command-string-byte-limit", + "command-string-byte-limit", + "command-string-byte-limit", + "command-string-byte-limit", +]) +assert.equal(record.artifactCapture?.fields.find((field) => field.path === "stdout")?.observedBytes, Buffer.byteLength(oversized)) +assert(Buffer.byteLength(JSON.stringify(projected)) < COMMAND_ARTIFACT_TOTAL_STRING_MAX_BYTES) + +const commandsLog = formatCommandsLog(projected) +assert.match(commandsLog, /wordpress\.apply-site-plan --plan fixture\.json/) +assert(Buffer.byteLength(commandsLog) <= COMMAND_ARTIFACT_STRING_MAX_BYTES + 1024) +assert(!commandsLog.includes(oversized)) + +const unicode = "😀".repeat(COMMAND_ARTIFACT_STRING_MAX_BYTES) +const unicodeRecord = boundedExecutionResultsForArtifacts([{ ...command, stdout: unicode, result: undefined, diagnostics: undefined }])[0] +assert(unicodeRecord) +assert(Buffer.byteLength(unicodeRecord.stdout) <= COMMAND_ARTIFACT_STRING_MAX_BYTES) +assert(!/[\uD800-\uDBFF]$/.test(unicodeRecord.stdout), "UTF-8 truncation does not split a surrogate pair") + +const commandBudgetRecord = boundedExecutionResultsForArtifacts([{ + ...command, + stdout: "", + result: undefined, + diagnostics: Array.from({ length: 20 }, () => oversized), +}])[0] +assert(commandBudgetRecord?.artifactCapture?.fields.some((field) => field.reason === "command-string-byte-limit")) +assert.equal(commandBudgetRecord?.artifactCapture?.limits.capturedStringBytesPerCommand, COMMAND_ARTIFACT_COMMAND_STRING_MAX_BYTES) +assert(Buffer.byteLength(JSON.stringify(commandBudgetRecord)) < COMMAND_ARTIFACT_COMMAND_STRING_MAX_BYTES + 64 * 1024) + +const totalBudgetRecords = boundedExecutionResultsForArtifacts(Array.from({ length: 10 }, (_, index) => ({ + ...command, + id: `total-budget-${index}`, + stdout: "", + result: undefined, + diagnostics: [oversized, oversized], +}))) +assert(totalBudgetRecords.some((record) => record.artifactCapture?.fields.some((field) => field.reason === "total-string-byte-limit"))) + +const nodeBudgetRecord = boundedExecutionResultsForArtifacts([{ + ...command, + stdout: "", + result: undefined, + diagnostics: Array.from({ length: 20 }, () => Array.from({ length: 2_000 }, () => "x")), +}])[0] +assert(nodeBudgetRecord?.artifactCapture?.fields.some((field) => field.reason === "node-limit")) +assert.equal(nodeBudgetRecord?.artifactCapture?.limits.nodes, COMMAND_ARTIFACT_MAX_NODES) + +const manyCommands = Array.from({ length: COMMAND_ARTIFACT_MAX_RECORDS + 2 }, (_, index) => ({ + ...command, + id: `command-${index}`, + stdout: "ok", + result: undefined, + diagnostics: undefined, +})) +const boundedCommands = boundedExecutionResultsForArtifacts(manyCommands) +assert.equal(boundedCommands.length, COMMAND_ARTIFACT_MAX_RECORDS) +assert.equal(boundedCommands.at(-1)?.id, `command-${COMMAND_ARTIFACT_MAX_RECORDS + 1}`) +assert.equal(boundedCommands.at(-1)?.artifactCapture?.omittedCommandCount, 2) + +const boundedRecipeOutput = boundedRecipeJsonOutput({ + schema: "wp-codebox/recipe-run/v1", + executions: [{ ...command, recipePhase: "workflow", recipeStepIndex: 4, recipeCommand: "wordpress.apply-site-plan" }], +}) as { executions: Array } +assert.equal(boundedRecipeOutput.executions[0]?.recipePhase, "workflow") +assert.equal(boundedRecipeOutput.executions[0]?.recipeStepIndex, 4) +assert.equal(boundedRecipeOutput.executions[0]?.recipeCommand, "wordpress.apply-site-plan") +assert(boundedRecipeOutput.executions[0]?.artifactCapture) +assert(Buffer.byteLength(JSON.stringify(boundedRecipeOutput)) < COMMAND_ARTIFACT_TOTAL_STRING_MAX_BYTES) + +const boundedRecipeSequence = boundedRecipeJsonOutput({ + executions: [ + { ...command, result: undefined, diagnostics: [oversized, oversized, oversized], recipePhase: "steps", recipeStepIndex: 1, recipeCommand: "wordpress.import" }, + { ...command, id: "later-command", stdout: "later evidence", result: undefined, diagnostics: undefined, recipePhase: "steps", recipeStepIndex: 2, recipeCommand: "wordpress.visual-compare" }, + ], +}) as { executions: Array } +assert.equal(boundedRecipeSequence.executions[1]?.stdout, "later evidence") +assert.equal(boundedRecipeSequence.executions[1]?.recipePhase, "steps") +assert.equal(boundedRecipeSequence.executions[1]?.recipeCommand, "wordpress.visual-compare") + +const artifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-command-artifact-bounds-")) +try { + await collectPlaygroundArtifacts({ + artifactRoot, + browserProbes: [], + commands: [command], + createdAt: "2026-07-21T19:59:00.000Z", + events: [], + info: async () => ({ id: "runtime-1", backend: "playground", status: "ready", environment: { version: "latest" } }), + mounts: [], + observations: [], + pluginChecks: [], + previewInfo: async () => undefined, + recordArtifactsCollected: () => {}, + runtimeId: "runtime-1", + snapshots: [], + spec: { environment: { blueprint: {} } }, + themeChecks: [], + }) + const persistedRecord = JSON.parse((await readFile(join(artifactRoot, "commands.jsonl"), "utf8")).trim()) + assert.equal(persistedRecord.command, command.command) + assert.equal(persistedRecord.artifactCapture.schema, "wp-codebox/command-artifact-capture/v1") + assert(Buffer.byteLength(await readFile(join(artifactRoot, "logs", "commands.log"), "utf8")) <= COMMAND_ARTIFACT_STRING_MAX_BYTES + 1024) +} finally { + await rm(artifactRoot, { recursive: true, force: true }) +} + +const collectionCause = Object.assign(new RangeError("Invalid string length"), { + code: "command-artifact-collection-failed", + artifactPath: "commands.jsonl", + payload: { commandCount: 7, stdoutBytes: 65_683_096 }, + limits: { totalStringBytes: COMMAND_ARTIFACT_TOTAL_STRING_MAX_BYTES }, + causeStack: "RangeError: Invalid string length\n at ArtifactBundleBuilder.build", +}) +let collectionError: unknown +try { + await collectRecipeRuntimeArtifacts({ + collectArtifacts: async () => { throw collectionCause }, + } as any, {}, { + activeExecution: { ...command, recipePhase: "workflow", recipeStepIndex: 4, recipeCommand: "wordpress.apply-site-plan" }, + }) +} catch (error) { + collectionError = error +} +const serializedError = serializeRecipeRunError(collectionError) +assert.equal(serializedError.code, "recipe-artifact-collection-failed") +assert.equal(serializedError.operation, "runtime.collect-artifacts") +assert.equal(serializedError.workflowStepIndex, 4) +assert.equal(serializedError.command, "wordpress.apply-site-plan") +assert.equal(serializedError.artifactPath, "commands.jsonl") +assert.deepEqual(serializedError.payload, collectionCause.payload) +assert.deepEqual(serializedError.limits, collectionCause.limits) +assert.equal(serializedError.causeStack, collectionCause.causeStack) +assert.equal((serializedError.cause as { code?: string } | undefined)?.code, "command-artifact-collection-failed") + +console.log("runtime command artifact bounds ok")