|
| 1 | +import type { ExecutionResult } from "./runtime-contracts.js" |
| 2 | +import { normalizeJsonValue } from "./object-utils.js" |
| 3 | + |
| 4 | +export const COMMAND_ARTIFACT_STRING_MAX_BYTES = 1024 * 1024 |
| 5 | +export const COMMAND_ARTIFACT_COMMAND_STRING_MAX_BYTES = 2 * 1024 * 1024 |
| 6 | +export const COMMAND_ARTIFACT_TOTAL_STRING_MAX_BYTES = 16 * 1024 * 1024 |
| 7 | +export const COMMAND_ARTIFACT_MAX_NODES = 25_000 |
| 8 | +export const COMMAND_ARTIFACT_MAX_RECORDS = 1_000 |
| 9 | +const COMMAND_ARTIFACT_MAX_TRUNCATIONS = 1_000 |
| 10 | +const COMMAND_ARTIFACT_KEY_MAX_BYTES = 256 |
| 11 | +const COMMAND_ARTIFACT_IDENTITY_MAX_BYTES = 1024 |
| 12 | +const COMMAND_ARTIFACT_PATH_MAX_BYTES = 4096 |
| 13 | +const EXECUTION_RESULT_KEYS = new Set(["id", "command", "args", "exitCode", "stdout", "stderr", "result", "diagnostics", "artifactRefs", "startedAt", "finishedAt", "artifactCapture"]) |
| 14 | +const EXECUTION_IDENTITY_EXTENSION_KEYS = new Set(["recipePhase", "recipeCommand", "fuzzCaseId", "fuzzPhase"]) |
| 15 | + |
| 16 | +export interface CommandArtifactTruncation { |
| 17 | + path: string |
| 18 | + reason: "string-byte-limit" | "command-string-byte-limit" | "total-string-byte-limit" | "node-limit" |
| 19 | + observedBytes?: number |
| 20 | + capturedBytes?: number |
| 21 | + configuredLimitBytes?: number |
| 22 | +} |
| 23 | + |
| 24 | +export interface CommandArtifactCapture { |
| 25 | + schema: "wp-codebox/command-artifact-capture/v1" |
| 26 | + truncated: true |
| 27 | + limits: { |
| 28 | + capturedStringBytesPerValue: number |
| 29 | + capturedStringBytesPerCommand: number |
| 30 | + capturedStringBytesTotal: number |
| 31 | + nodes: number |
| 32 | + records: number |
| 33 | + } |
| 34 | + fields: CommandArtifactTruncation[] |
| 35 | + omittedFieldCount?: number |
| 36 | + omittedCommandCount?: number |
| 37 | +} |
| 38 | + |
| 39 | +export type BoundedExecutionResult<T extends ExecutionResult = ExecutionResult> = T & { artifactCapture?: CommandArtifactCapture } |
| 40 | + |
| 41 | +export function boundedExecutionResultsForArtifacts<T extends ExecutionResult>(commands: T[]): Array<BoundedExecutionResult<T>> { |
| 42 | + const budget = { |
| 43 | + remainingBytes: COMMAND_ARTIFACT_TOTAL_STRING_MAX_BYTES, |
| 44 | + remainingCommandBytes: COMMAND_ARTIFACT_COMMAND_STRING_MAX_BYTES, |
| 45 | + remainingNodes: COMMAND_ARTIFACT_MAX_NODES, |
| 46 | + remainingTruncations: COMMAND_ARTIFACT_MAX_TRUNCATIONS, |
| 47 | + } |
| 48 | + const selectedCommands = commands.length <= COMMAND_ARTIFACT_MAX_RECORDS |
| 49 | + ? commands |
| 50 | + : [...commands.slice(0, COMMAND_ARTIFACT_MAX_RECORDS - 1), commands.at(-1)!] |
| 51 | + |
| 52 | + return selectedCommands.map((command, selectedIndex) => { |
| 53 | + budget.remainingCommandBytes = COMMAND_ARTIFACT_COMMAND_STRING_MAX_BYTES |
| 54 | + const fields: CommandArtifactTruncation[] = [] |
| 55 | + let omittedFieldCount = 0 |
| 56 | + const capture = (value: unknown, path: string): unknown => boundedArtifactValue(value, path, budget, (field) => { |
| 57 | + if (fields.length < 100 && budget.remainingTruncations > 0) { |
| 58 | + fields.push(field) |
| 59 | + budget.remainingTruncations -= 1 |
| 60 | + } else { |
| 61 | + omittedFieldCount += 1 |
| 62 | + } |
| 63 | + }) |
| 64 | + const extensions = Object.fromEntries( |
| 65 | + Object.entries(command) |
| 66 | + .filter(([key]) => !EXECUTION_RESULT_KEYS.has(key)) |
| 67 | + .map(([key, value]) => [key, typeof value === "string" && EXECUTION_IDENTITY_EXTENSION_KEYS.has(key) ? boundedIdentityString(value) : capture(normalizeJsonValue(value), key)]), |
| 68 | + ) |
| 69 | + const projected = { |
| 70 | + ...extensions, |
| 71 | + id: boundedIdentityString(command.id), |
| 72 | + command: boundedIdentityString(command.command), |
| 73 | + args: capture(normalizeJsonValue(command.args), "args") as string[], |
| 74 | + exitCode: command.exitCode, |
| 75 | + stdout: capture(command.stdout, "stdout") as string, |
| 76 | + stderr: capture(command.stderr, "stderr") as string, |
| 77 | + ...(command.result === undefined ? {} : { result: capture(normalizeJsonValue(command.result), "result") as ExecutionResult["result"] }), |
| 78 | + ...(command.diagnostics === undefined ? {} : { diagnostics: capture(normalizeJsonValue(command.diagnostics), "diagnostics") }), |
| 79 | + ...(command.artifactRefs === undefined ? {} : { artifactRefs: capture(normalizeJsonValue(command.artifactRefs), "artifactRefs") as ExecutionResult["artifactRefs"] }), |
| 80 | + startedAt: boundedIdentityString(command.startedAt), |
| 81 | + finishedAt: boundedIdentityString(command.finishedAt), |
| 82 | + } as BoundedExecutionResult<T> |
| 83 | + const omittedCommandCount = commands.length - selectedCommands.length |
| 84 | + if (fields.length > 0 || omittedFieldCount > 0 || (omittedCommandCount > 0 && selectedIndex === selectedCommands.length - 1)) { |
| 85 | + projected.artifactCapture = { |
| 86 | + schema: "wp-codebox/command-artifact-capture/v1", |
| 87 | + truncated: true, |
| 88 | + limits: { |
| 89 | + capturedStringBytesPerValue: COMMAND_ARTIFACT_STRING_MAX_BYTES, |
| 90 | + capturedStringBytesPerCommand: COMMAND_ARTIFACT_COMMAND_STRING_MAX_BYTES, |
| 91 | + capturedStringBytesTotal: COMMAND_ARTIFACT_TOTAL_STRING_MAX_BYTES, |
| 92 | + nodes: COMMAND_ARTIFACT_MAX_NODES, |
| 93 | + records: COMMAND_ARTIFACT_MAX_RECORDS, |
| 94 | + }, |
| 95 | + fields, |
| 96 | + ...(omittedFieldCount > 0 ? { omittedFieldCount } : {}), |
| 97 | + ...(omittedCommandCount > 0 && selectedIndex === selectedCommands.length - 1 ? { omittedCommandCount } : {}), |
| 98 | + } |
| 99 | + } |
| 100 | + return projected |
| 101 | + }) |
| 102 | +} |
| 103 | + |
| 104 | +function boundedArtifactValue( |
| 105 | + value: unknown, |
| 106 | + path: string, |
| 107 | + budget: { remainingBytes: number; remainingCommandBytes: number; remainingNodes: number; remainingTruncations: number }, |
| 108 | + recordTruncation: (field: CommandArtifactTruncation) => void, |
| 109 | +): unknown { |
| 110 | + if (value === null || value === undefined || (typeof value !== "string" && typeof value !== "object")) { |
| 111 | + return value |
| 112 | + } |
| 113 | + if (budget.remainingNodes <= 0) { |
| 114 | + recordTruncation({ path: truncateUtf8(path, COMMAND_ARTIFACT_PATH_MAX_BYTES), reason: "node-limit" }) |
| 115 | + if (typeof value === "string") return "" |
| 116 | + return Array.isArray(value) ? [] : {} |
| 117 | + } |
| 118 | + budget.remainingNodes -= 1 |
| 119 | + if (typeof value === "string") { |
| 120 | + const observedBytes = Buffer.byteLength(value, "utf8") |
| 121 | + const totalBudgetLimited = budget.remainingBytes < Math.min(observedBytes, COMMAND_ARTIFACT_STRING_MAX_BYTES) |
| 122 | + const commandBudgetLimited = budget.remainingCommandBytes < Math.min(observedBytes, COMMAND_ARTIFACT_STRING_MAX_BYTES) |
| 123 | + const configuredLimitBytes = Math.min(COMMAND_ARTIFACT_STRING_MAX_BYTES, budget.remainingCommandBytes, budget.remainingBytes) |
| 124 | + const captured = truncateUtf8(value, configuredLimitBytes) |
| 125 | + const capturedBytes = Buffer.byteLength(captured, "utf8") |
| 126 | + budget.remainingBytes -= capturedBytes |
| 127 | + budget.remainingCommandBytes -= capturedBytes |
| 128 | + if (capturedBytes < observedBytes) { |
| 129 | + recordTruncation({ |
| 130 | + path: truncateUtf8(path, COMMAND_ARTIFACT_PATH_MAX_BYTES), |
| 131 | + reason: totalBudgetLimited ? "total-string-byte-limit" : commandBudgetLimited ? "command-string-byte-limit" : "string-byte-limit", |
| 132 | + observedBytes, |
| 133 | + capturedBytes, |
| 134 | + configuredLimitBytes, |
| 135 | + }) |
| 136 | + } |
| 137 | + return captured |
| 138 | + } |
| 139 | + if (Array.isArray(value)) { |
| 140 | + return value.map((item, index) => boundedArtifactValue(item, `${path}[${index}]`, budget, recordTruncation)) |
| 141 | + } |
| 142 | + if (value && typeof value === "object") { |
| 143 | + return Object.fromEntries(Object.entries(value).map(([key, item]) => { |
| 144 | + const capturedKey = truncateUtf8(key, COMMAND_ARTIFACT_KEY_MAX_BYTES) |
| 145 | + return [capturedKey, boundedArtifactValue(item, `${path}.${capturedKey}`, budget, recordTruncation)] |
| 146 | + })) |
| 147 | + } |
| 148 | + return value |
| 149 | +} |
| 150 | + |
| 151 | +function boundedIdentityString(value: string): string { |
| 152 | + return truncateUtf8(value, COMMAND_ARTIFACT_IDENTITY_MAX_BYTES) |
| 153 | +} |
| 154 | + |
| 155 | +export function truncateUtf8(value: string, maxBytes: number): string { |
| 156 | + if (maxBytes <= 0) return "" |
| 157 | + if (Buffer.byteLength(value, "utf8") <= maxBytes) return value |
| 158 | + |
| 159 | + let low = 0 |
| 160 | + let high = Math.min(value.length, maxBytes) |
| 161 | + while (low < high) { |
| 162 | + const middle = Math.ceil((low + high) / 2) |
| 163 | + if (Buffer.byteLength(value.slice(0, middle), "utf8") <= maxBytes) low = middle |
| 164 | + else high = middle - 1 |
| 165 | + } |
| 166 | + const captured = value.slice(0, low) |
| 167 | + return /[\uD800-\uDBFF]$/.test(captured) ? captured.slice(0, -1) : captured |
| 168 | +} |
0 commit comments