From 16a42fe0ec4afd770964bfe7f438033439a06485 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Fri, 24 Jul 2026 18:44:37 +0000 Subject: [PATCH] feat: add adversarial fuzz runtime contracts --- README.md | 4 + docs/adversarial-runtime.md | 202 +++++++ package.json | 1 + packages/cli/src/recipe-validation.ts | 10 +- packages/cli/src/runtime-services.ts | 180 ++++++- .../runtime-core/src/adversarial-artifacts.ts | 81 +++ .../runtime-core/src/adversarial-browser.ts | 136 +++++ .../runtime-core/src/adversarial-campaign.ts | 496 ++++++++++++++++++ packages/runtime-core/src/contracts.ts | 4 + packages/runtime-core/src/index.ts | 4 + packages/runtime-core/src/public.ts | 4 + packages/runtime-core/src/recipe-schema.ts | 5 +- .../src/runtime-contract-manifest.ts | 18 + .../runtime-core/src/runtime-contracts.ts | 5 +- packages/runtime-core/src/transport-faults.ts | 296 +++++++++++ .../src/browser-clock-control.ts | 39 ++ .../src/browser-transport-faults.ts | 127 +++++ packages/runtime-playground/src/index.ts | 2 + tests/adversarial-browser.test.ts | 36 ++ tests/adversarial-campaign.test.ts | 99 ++++ tests/browser-clock-control.test.ts | 23 + tests/public-api-contract.test.ts | 8 + tests/runtime-services.test.ts | 31 ++ tests/transport-faults.test.ts | 68 +++ 24 files changed, 1871 insertions(+), 8 deletions(-) create mode 100644 docs/adversarial-runtime.md create mode 100644 packages/runtime-core/src/adversarial-artifacts.ts create mode 100644 packages/runtime-core/src/adversarial-browser.ts create mode 100644 packages/runtime-core/src/adversarial-campaign.ts create mode 100644 packages/runtime-core/src/transport-faults.ts create mode 100644 packages/runtime-playground/src/browser-clock-control.ts create mode 100644 packages/runtime-playground/src/browser-transport-faults.ts create mode 100644 tests/adversarial-browser.test.ts create mode 100644 tests/adversarial-campaign.test.ts create mode 100644 tests/browser-clock-control.test.ts create mode 100644 tests/transport-faults.test.ts diff --git a/README.md b/README.md index c3dff3ba7..17863a4e6 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # WP Codebox +Adversarial campaign, transport fault, service disruption, browser oracle, and +sealed replay contracts are documented in +[`docs/adversarial-runtime.md`](docs/adversarial-runtime.md). + **WP Codebox unlocks secure WordPress code execution from anywhere.** Run agents, accept untrusted patches, evaluate plugins, reproduce bugs, or experiment freely - every sandbox is a disposable contained WordPress runtime that can't touch its caller. Your host can be a CLI, CI job, mobile app, Node service, WordPress plugin, or anything else that can shell out or hit an API. WordPress has historically lacked a clean scratch space for code execution. Modern dev workflows assume one - Node has `npm install` per project, Python has venvs, containers have ephemeral filesystems. WP Codebox provides that primitive as a usable runtime contract: real WordPress, fully ephemeral, no host filesystem access except via declared mounts. Any product - WordPress or not - can offer code execution against a real WordPress instance without risking the caller. diff --git a/docs/adversarial-runtime.md b/docs/adversarial-runtime.md new file mode 100644 index 000000000..175bb2729 --- /dev/null +++ b/docs/adversarial-runtime.md @@ -0,0 +1,202 @@ +# Adversarial runtime + +WP Codebox exposes backend-neutral adversarial campaign and transport-fault +contracts from `@automattic/wp-codebox-core/public`. The contracts compose with +the existing fuzz suite, runtime episode, checkpoint, browser, service, artifact, +and replay surfaces; they do not replace those surfaces. + +## Architecture + +- `adversarial-campaign.ts` owns deterministic corpus scheduling, generic value + and action-sequence mutation, novelty retention, bounded concurrent execution, + minimization, stable finding fingerprints, replay schedules, and differential + result classification. +- `transport-faults.ts` owns request matching, deterministic stateful outcome + sequences, capability negotiation, redacted evidence, and deduplication. It + does not assume a browser, HTTP library, application, or transport. +- `adversarial-browser.ts` owns DOM-derived journey planning, hostile generic + input generation, journey minimization, and user-facing oracle contracts. +- `adversarial-artifacts.ts` writes bounded manifested finding and replay bundles, + removes declared secrets and machine-specific paths, and seals corpus/finding + identity with a deterministic content digest. +- `runtime-playground` maps supported fault outcomes and browser clock controls + onto Playwright. Socket behavior that Playwright cannot faithfully produce is + reported as unsupported. +- `runtime-services.ts` provisions loopback-only, ephemeral MySQL/MariaDB, Redis, + SMTP sink, and HTTP fixture services. Every service has idempotent teardown and + typed disruption controls. + +Generic layers contain no component, plugin, vendor integration, or product +policy. WordPress grammars and attack packs remain extension responsibilities. + +## Campaign contract + +```ts +const campaign = adversarialCampaign({ + id: "component-campaign", + seed: "ci-seed-1", + corpus: [{ + id: "create-update-read", + actions: [ + { type: "create", input: { title: "seed" } }, + { type: "update", input: { title: "candidate" } }, + { type: "read" }, + ], + }], + budgets: { + maxCases: 1000, + workers: 4, + maxCaseTimeMs: 30000, + maxWallTimeMs: 300000, + }, + oracles: [{ + schema: "wp-codebox/adversarial-oracle/v1", + id: "state-integrity", + severity: "high", + }], +}) + +const result = await runAdversarialCampaign(campaign, { + execute: async (plan, signal) => adapter.execute(plan, signal), + evaluate: async (plan, observation, oracles) => + adapter.evaluate(plan, observation, oracles), +}) +``` + +Workers execute a deterministic round concurrently. Results are committed to the +corpus in case-index order after every round, so completion timing cannot change +the retained corpus or later mutations. Replay metadata records the seed, +worker, iteration, matrix cell, minimized actions/input, fault schedule, exact +provenance, normalized schedule, and a one-command replay instruction. + +## Mutation and novelty + +Core mutation supports generic scalar, structured, binary, and stateful sequence +inputs. Mutation targets are selected from stable JSON paths using a seeded +SHA-256 schedule. Extensions can represent multipart, serialized, markup, or +domain-specific inputs as structured actions and register richer mutators at the +adapter boundary without adding those grammars to runtime-core. + +Adapters return bounded novelty signals such as coverage edges, route ids, state +digests, hook ids, or query fingerprints. New signals retain a case in the +interesting corpus. Minimization removes action chunks and shrinks input values, +replaying each candidate through the same adapter and oracle set. + +## Fault model + +```json +{ + "schema": "wp-codebox/transport-fault-model/v1", + "seed": "fault-seed-1", + "rules": [{ + "id": "verification-outage", + "match": { + "host": "service.example", + "method": "POST", + "path": "/verify" + }, + "sequence": [ + { "status": 500 }, + { "delayMs": 30000, "timeoutMs": 1000 }, + { "status": 200, "body": "malformed", "truncateAfterBytes": 4 } + ], + "repeat": "last" + }] +} +``` + +Each adapter publishes `wp-codebox/transport-fault-capabilities/v1`. Campaigns +must negotiate before execution and fail closed when a required semantic is +unsupported. + +### Playwright fidelity + +| Semantic | Fidelity | +| --- | --- | +| Response status/header/body substitution | Exact | +| Delay, deterministic jitter, host remap | Exact | +| Malformed/truncated payload, bandwidth, timeout, refusal/reset | Emulated and labeled | +| Chunk framing, half-close, disconnect-after-N-bytes | Unsupported | + +Playwright owns HTTP framing and sockets, so payload truncation must not be +reported as a transport disconnect. A lower-level proxy provider is required for +those exact semantics. + +## Services + +Recipe `inputs.services` accepts: + +- `mysql`: MySQL 8.4 or MariaDB 11.4, preserving existing output and readiness + behavior. +- `redis`: ephemeral Redis with persistence disabled. +- `smtp`: SMTP message sink plus its loopback inspection port. +- `http`: deterministic loopback response fixture. + +The provisioned service set exposes `control(serviceId, action, options)`. +Container lifecycle actions (`stop`, `start`, `pause`, `resume`, `restart`, +`disconnect`, and `reconnect`) have exact Docker fidelity. Provider-specific +`flush` and read-only controls are exact where implemented. Network latency is +explicitly unsupported by this provider and should use the transport-fault +adapter instead. + +All containers bind ephemeral loopback ports, use no persistent volumes, and are +removed in reverse order after success, failure, cancellation, or timeout. + +## Clock control + +Playwright browser time supports exact freeze, advance, skew, and resume through +its clock API. The same capability response explicitly marks server process, +scheduler, and database clocks unsupported. A WordPress runtime extension is +required to control those surfaces without faking server behavior in browser +JavaScript. + +## Browser oracles + +The generic browser planner derives actions from visible control descriptors and +generates empty, oversized, hostile punctuation, bidi, and Unicode inputs. Submit +controls are repeated to expose duplicate side effects. Generic oracles cover: + +- page errors, console errors, and unhandled rejections; +- controls that accept input without an observable action; +- loading indicators that exceed a declared threshold; +- clipping, viewport overflow, and invisible focus; +- adapter-provided accessibility violations; +- duplicate observable effects. + +Journey minimization uses deterministic subset replay to retain the shortest +sequence that preserves the oracle failure. + +## Evidence and safety + +`writeAdversarialEvidenceBundle()` writes a manifest, campaign result, one +finding and replay document per stable fingerprint, and a secret-scan report. +The writer enforces an artifact byte ceiling, redacts sensitive fields and +caller-declared values, removes machine-specific paths, and records SHA-256 file +digests plus a stable bundle content digest. + +Campaign budgets independently bound cases, actions, input bytes, per-case and +wall time, worker count, and artifact bytes. Adapters remain responsible for the +existing Codebox CPU, memory, disk, process, network-deny, mount, and disposable +sandbox boundaries. + +## Compatibility + +Existing recipes and `fuzzRun` cases are unchanged. Existing MySQL defaults, +outputs, readiness, and teardown behavior remain compatible. The new service +kinds and public TypeScript contracts are additive. + +## Current boundaries + +The following capabilities are intentionally not claimed by this change: + +- exact socket framing faults require a lower-level proxy provider; +- server-side WordPress HTTP fault interception requires a WordPress extension + adapter using the generic fault contract; +- PHP/WordPress, cron, and database clock control require a WordPress extension; +- WordPress-specific mutation grammars, security policies, and instrumentation + remain extension-owned; +- live vulnerable plugin/theme discovery campaigns require disposable runtime + fixtures and are not replaced by the neutral contract tests. + +These are capability gaps, not silent skips. Consumers can inspect negotiation +results before running a campaign. diff --git a/package.json b/package.json index cc9ff7805..558ce3a27 100644 --- a/package.json +++ b/package.json @@ -230,6 +230,7 @@ "test:recipe-run-provenance": "tsx tests/recipe-run-provenance.test.ts", "test:fuzz-run-recipe": "tsx tests/fuzz-run-recipe.test.ts", "test:fuzz-suite-runner": "tsx tests/fuzz-suite-runner.test.ts", + "test:adversarial-runtime": "tsx --test tests/transport-faults.test.ts tests/adversarial-campaign.test.ts tests/adversarial-browser.test.ts tests/browser-clock-control.test.ts", "test:playground-fuzz-suite-public": "tsx tests/playground-fuzz-suite-public.test.ts", "test:nested-fuzz-suite-recipe-command": "tsx tests/nested-fuzz-suite-recipe-command.test.ts", "test:wordpress-fuzz-suite-builders": "tsx tests/wordpress-fuzz-suite-builders.test.ts", diff --git a/packages/cli/src/recipe-validation.ts b/packages/cli/src/recipe-validation.ts index 74411ff47..8abe2f699 100644 --- a/packages/cli/src/recipe-validation.ts +++ b/packages/cli/src/recipe-validation.ts @@ -742,9 +742,15 @@ function validateRecipeRuntimeServices(recipe: WorkspaceRecipe, addIssue: (code: if (!/^[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(service.id)) addIssue("invalid-runtime-service-id", `${path}.id`, "Runtime service ids must be stable identifiers.") if (ids.has(service.id)) addIssue("duplicate-runtime-service-id", `${path}.id`, `Runtime service ids must be unique: ${service.id}`) ids.add(service.id) - if (service.kind !== "mysql") addIssue("unsupported-runtime-service-kind", `${path}.kind`, `Unsupported managed runtime service kind: ${service.kind}`) + if (!["mysql", "redis", "smtp", "http"].includes(service.kind)) addIssue("unsupported-runtime-service-kind", `${path}.kind`, `Unsupported managed runtime service kind: ${service.kind}`) + const supportedOutputs: Record = { + mysql: /^(host|port|username|password|database)$/, + redis: /^(host|port|url)$/, + smtp: /^(host|port|httpPort|url)$/, + http: /^(host|port|url)$/, + } for (const [output, name] of Object.entries(service.outputs)) { - if (!/^(host|port|username|password|database)$/.test(output)) addIssue("unknown-runtime-service-output", `${path}.outputs.${output}`, `Unsupported ${service.kind} service output: ${output}`) + if (!(supportedOutputs[service.kind] ?? /^$/).test(output)) addIssue("unknown-runtime-service-output", `${path}.outputs.${output}`, `Unsupported ${service.kind} service output: ${output}`) if (!/^[A-Z_][A-Z0-9_]*$/.test(name)) addIssue("invalid-runtime-service-env", `${path}.outputs.${output}`, "Runtime service environment variable names must match /^[A-Z_][A-Z0-9_]*$/.") if (environment.has(name)) addIssue("duplicate-runtime-service-env", `${path}.outputs.${output}`, `Runtime service output environment variable is already declared: ${name}`) environment.add(name) diff --git a/packages/cli/src/runtime-services.ts b/packages/cli/src/runtime-services.ts index 90ed27580..811205d5b 100644 --- a/packages/cli/src/runtime-services.ts +++ b/packages/cli/src/runtime-services.ts @@ -6,6 +6,17 @@ import type { WorkspaceRecipeRuntimeService } from "@automattic/wp-codebox-core" const execFileAsync = promisify(execFile) const MYSQL_IMAGES = { mysql: "mysql:8.4", mariadb: "mariadb:11.4" } as const +const SERVICE_IMAGES = { redis: "redis:7.4-alpine", smtp: "axllent/mailpit:v1.27", http: "hashicorp/http-echo:1.0" } as const + +export type RuntimeServiceControlAction = "stop" | "start" | "pause" | "resume" | "restart" | "disconnect" | "reconnect" | "flush" | "read-only" | "read-write" | "latency" + +export interface RuntimeServiceControlResult { + serviceId: string + action: RuntimeServiceControlAction + status: "applied" | "unsupported" | "failed" + fidelity: "exact" | "emulated" | "unsupported" + reason?: string +} export interface RuntimeServiceEvidence { id: string @@ -16,6 +27,7 @@ export interface RuntimeServiceEvidence { lifecycle: "provisioning" | "provisioned" | "released" | "failed" teardown?: "completed" | "failed" diagnostic?: { code: "readiness-failed" | "provision-failed" | "teardown-failed" | "interrupted" } + controls?: RuntimeServiceControlResult[] } export class RuntimeServiceProvisionError extends Error { @@ -40,6 +52,7 @@ interface ManagedRuntimeService { env: Record evidence: RuntimeServiceEvidence release(): Promise + control(action: RuntimeServiceControlAction, options?: Record): Promise } export interface RuntimeServiceDependencies { @@ -57,7 +70,7 @@ export interface RuntimeServiceProvider { const defaultDependencies: RuntimeServiceDependencies = { execute: async (command, args, options) => await execFileAsync(command, args, options), - waitForReady: waitForMysqlProtocol, + waitForReady: waitForTcpProtocol, randomBytes, } @@ -68,7 +81,7 @@ export function runtimeServicePlan(services: WorkspaceRecipeRuntimeService[]): A }) } -export async function provisionRuntimeServices(services: WorkspaceRecipeRuntimeService[], options: { signal?: AbortSignal; dependencies?: RuntimeServiceDependencies } = {}): Promise<{ env: Record; evidence: RuntimeServiceEvidence[]; release(): Promise }> { +export async function provisionRuntimeServices(services: WorkspaceRecipeRuntimeService[], options: { signal?: AbortSignal; dependencies?: RuntimeServiceDependencies } = {}): Promise<{ env: Record; evidence: RuntimeServiceEvidence[]; control(serviceId: string, action: RuntimeServiceControlAction, controlOptions?: Record): Promise; release(): Promise }> { const dependencies = options.dependencies ?? defaultDependencies const provisioned: ManagedRuntimeService[] = [] const evidence: RuntimeServiceEvidence[] = [] @@ -91,6 +104,11 @@ export async function provisionRuntimeServices(services: WorkspaceRecipeRuntimeS return { env: Object.assign({}, ...provisioned.map((service) => service.env)), evidence, + async control(serviceId, action, controlOptions) { + const service = provisioned.find((candidate) => candidate.evidence.id === serviceId) + if (!service) throw new Error(`Managed runtime service does not exist: ${serviceId}`) + return await service.control(action, controlOptions) + }, async release() { try { await releaseServices(provisioned) @@ -139,12 +157,19 @@ const mysqlDockerProvider: RuntimeServiceProvider = { provision: provisionMysqlDockerService, } +const redisDockerProvider: RuntimeServiceProvider = { name: "docker", kind: "redis", version: (service) => service.configuration?.image ?? SERVICE_IMAGES.redis, provision: provisionRedisDockerService } +const smtpDockerProvider: RuntimeServiceProvider = { name: "docker", kind: "smtp", version: (service) => service.configuration?.image ?? SERVICE_IMAGES.smtp, provision: provisionSmtpDockerService } +const httpDockerProvider: RuntimeServiceProvider = { name: "docker", kind: "http", version: (service) => service.configuration?.image ?? SERVICE_IMAGES.http, provision: provisionHttpDockerService } + function mysqlDockerImage(service: WorkspaceRecipeRuntimeService): string { return MYSQL_IMAGES[service.configuration?.engine ?? "mysql"] } function runtimeServiceProvider(kind: string): RuntimeServiceProvider { if (kind === mysqlDockerProvider.kind) return mysqlDockerProvider + if (kind === redisDockerProvider.kind) return redisDockerProvider + if (kind === smtpDockerProvider.kind) return smtpDockerProvider + if (kind === httpDockerProvider.kind) return httpDockerProvider throw new Error(`Unsupported managed runtime service kind: ${kind}`) } @@ -162,7 +187,7 @@ async function provisionMysqlDockerService(service: WorkspaceRecipeRuntimeServic const childEnvironment = { ...process.env, [`${environmentPrefix}_DATABASE`]: "runtime", [`${environmentPrefix}_USER`]: "runtime", [`${environmentPrefix}_PASSWORD`]: password, ...rootEnvironment } const foreignKeyTargetPolicy = service.configuration?.foreignKeyTargetPolicy const mysqlArguments = engine === "mysql" && foreignKeyTargetPolicy ? [`--restrict-fk-on-non-standard-key=${foreignKeyTargetPolicy === "indexed" ? "OFF" : "ON"}`] : [] - const runArgs = ["run", "--detach", "--rm", "--name", container, "--publish", "127.0.0.1::3306", "--tmpfs", "/var/lib/mysql", "--env", `${environmentPrefix}_DATABASE`, "--env", `${environmentPrefix}_USER`, "--env", `${environmentPrefix}_PASSWORD`, "--env", rootEnvironmentName, image, ...mysqlArguments] + const runArgs = ["run", "--detach", "--name", container, "--label", "wp-codebox.managed=true", "--publish", "127.0.0.1::3306", "--tmpfs", "/var/lib/mysql", "--env", `${environmentPrefix}_DATABASE`, "--env", `${environmentPrefix}_USER`, "--env", `${environmentPrefix}_PASSWORD`, "--env", rootEnvironmentName, image, ...mysqlArguments] let started = false try { throwIfAborted(signal) @@ -177,7 +202,23 @@ async function provisionMysqlDockerService(service: WorkspaceRecipeRuntimeServic evidence.readiness = "ready" evidence.lifecycle = "provisioned" const values: Record = { host: "127.0.0.1", port: String(port), username: "runtime", password, database: "runtime" } - return { env: Object.fromEntries(Object.entries(service.outputs).map(([output, name]) => [name, values[output] ?? ""])), evidence, async release() { await releaseService(container, evidence, dependencies) } } + return { + env: Object.fromEntries(Object.entries(service.outputs).map(([output, name]) => [name, values[output] ?? ""])), + evidence, + async control(action, options) { return await controlDockerService(container, evidence, dependencies, action, options, async (customAction) => { + if (customAction === "flush") { + await dependencies.execute("docker", ["exec", "--env", "MYSQL_PWD", container, engine === "mariadb" ? "mariadb" : "mysql", "--user=root", "--execute=RESET MASTER"], { env: { ...process.env, MYSQL_PWD: emptyRootPassword ? "" : password }, timeout: 10_000 }) + return true + } + if (customAction === "read-only" || customAction === "read-write") { + const enabled = customAction === "read-only" ? "ON" : "OFF" + await dependencies.execute("docker", ["exec", "--env", "MYSQL_PWD", container, engine === "mariadb" ? "mariadb" : "mysql", "--user=root", `--execute=SET GLOBAL read_only=${enabled}`], { env: { ...process.env, MYSQL_PWD: emptyRootPassword ? "" : password }, timeout: 10_000 }) + return true + } + return false + }, async () => { await dependencies.waitForReady("127.0.0.1", port, 30_000); await waitForMysqlDatabase(container, engine, password, dependencies, 30_000) }) }, + async release() { await releaseService(container, evidence, dependencies) }, + } } catch (error) { evidence.readiness = "failed" evidence.lifecycle = "failed" @@ -187,6 +228,110 @@ async function provisionMysqlDockerService(service: WorkspaceRecipeRuntimeServic } } +async function provisionRedisDockerService(service: WorkspaceRecipeRuntimeService, dependencies: RuntimeServiceDependencies, signal: AbortSignal | undefined, evidenceList: RuntimeServiceEvidence[]): Promise { + return await provisionSimpleDockerService(service, dependencies, signal, evidenceList, { + image: service.configuration?.image ?? SERVICE_IMAGES.redis, + ports: [6379], + runArgs: ["--save", "", "--appendonly", "no"], + values: (ports) => ({ host: "127.0.0.1", port: String(ports[0]), url: `redis://127.0.0.1:${ports[0]}` }), + customControl: async (container, action) => { + if (action !== "flush") return false + await dependencies.execute("docker", ["exec", container, "redis-cli", "FLUSHALL"], { timeout: 10_000 }) + return true + }, + }) +} + +async function provisionSmtpDockerService(service: WorkspaceRecipeRuntimeService, dependencies: RuntimeServiceDependencies, signal: AbortSignal | undefined, evidenceList: RuntimeServiceEvidence[]): Promise { + return await provisionSimpleDockerService(service, dependencies, signal, evidenceList, { + image: service.configuration?.image ?? SERVICE_IMAGES.smtp, + ports: [1025, 8025], + runArgs: [], + values: (ports) => ({ host: "127.0.0.1", port: String(ports[0]), httpPort: String(ports[1]), url: `smtp://127.0.0.1:${ports[0]}` }), + }) +} + +async function provisionHttpDockerService(service: WorkspaceRecipeRuntimeService, dependencies: RuntimeServiceDependencies, signal: AbortSignal | undefined, evidenceList: RuntimeServiceEvidence[]): Promise { + return await provisionSimpleDockerService(service, dependencies, signal, evidenceList, { + image: service.configuration?.image ?? SERVICE_IMAGES.http, + ports: [5678], + runArgs: ["-listen=:5678", `-status-code=${service.configuration?.responseStatus ?? 200}`, `-text=${service.configuration?.responseBody ?? "ok"}`], + values: (ports) => ({ host: "127.0.0.1", port: String(ports[0]), url: `http://127.0.0.1:${ports[0]}` }), + }) +} + +async function provisionSimpleDockerService( + service: WorkspaceRecipeRuntimeService, + dependencies: RuntimeServiceDependencies, + signal: AbortSignal | undefined, + evidenceList: RuntimeServiceEvidence[], + spec: { image: string; ports: number[]; runArgs: string[]; values(ports: number[]): Record; customControl?(container: string, action: RuntimeServiceControlAction, options?: Record): Promise }, +): Promise { + const evidence: RuntimeServiceEvidence = { id: service.id, kind: service.kind, provider: "docker", version: spec.image, readiness: "pending", lifecycle: "provisioning", controls: [] } + evidenceList.push(evidence) + const container = `wp-codebox-${service.id}-${dependencies.randomBytes(6).toString("hex")}` + let started = false + try { + throwIfAborted(signal) + await ensureDockerImage(spec.image, dependencies, signal) + const publishArgs = spec.ports.flatMap((port) => ["--publish", `127.0.0.1::${port}`]) + await dependencies.execute("docker", ["run", "--detach", "--name", container, "--label", "wp-codebox.managed=true", ...publishArgs, "--tmpfs", "/tmp", spec.image, ...spec.runArgs], { signal, timeout: 30_000 }) + started = true + const ports: number[] = [] + for (const containerPort of spec.ports) { + const { stdout } = await dependencies.execute("docker", ["port", container, `${containerPort}/tcp`], { signal, timeout: 10_000 }) + ports.push(parseLoopbackPort(stdout)) + } + await dependencies.waitForReady("127.0.0.1", ports[0] as number, 30_000, signal) + evidence.readiness = "ready" + evidence.lifecycle = "provisioned" + const values = spec.values(ports) + return { + env: Object.fromEntries(Object.entries(service.outputs).map(([output, name]) => [name, values[output] ?? ""])), + evidence, + async control(action, options) { return await controlDockerService(container, evidence, dependencies, action, options, spec.customControl ? async (candidate, candidateOptions) => await spec.customControl?.(container, candidate, candidateOptions) ?? false : undefined, async () => await dependencies.waitForReady("127.0.0.1", ports[0] as number, 30_000)) }, + async release() { await releaseService(container, evidence, dependencies) }, + } + } catch (error) { + evidence.readiness = "failed" + evidence.lifecycle = "failed" + evidence.diagnostic = { code: signal?.aborted ? "interrupted" : started ? "readiness-failed" : "provision-failed" } + if (started) await releaseService(container, evidence, dependencies).catch(() => undefined) + throw new RuntimeServiceProvisionError(`Managed runtime service failed: ${service.id}`, evidenceList) + } +} + +async function controlDockerService(container: string, evidence: RuntimeServiceEvidence, dependencies: RuntimeServiceDependencies, action: RuntimeServiceControlAction, options?: Record, custom?: (action: RuntimeServiceControlAction, options?: Record) => Promise, recover?: () => Promise): Promise { + const common: Partial> = { + stop: ["stop", container], + start: ["start", container], + pause: ["pause", container], + resume: ["unpause", container], + restart: ["restart", container], + disconnect: ["network", "disconnect", "bridge", container], + reconnect: ["network", "connect", "bridge", container], + } + let result: RuntimeServiceControlResult + try { + const args = common[action] + if (args) { + await dependencies.execute("docker", args, { timeout: 30_000 }) + if (["start", "resume", "restart", "reconnect"].includes(action)) await recover?.() + result = { serviceId: evidence.id, action, status: "applied", fidelity: "exact" } + } else if (await custom?.(action, options)) { + result = { serviceId: evidence.id, action, status: "applied", fidelity: "exact" } + } else if (action === "latency") { + result = { serviceId: evidence.id, action, status: "unsupported", fidelity: "unsupported", reason: "The Docker provider does not inject host network shaping; use a declared transport fault adapter." } + } else { + result = { serviceId: evidence.id, action, status: "unsupported", fidelity: "unsupported", reason: `The ${evidence.kind} provider does not support ${action}.` } + } + } catch (error) { + result = { serviceId: evidence.id, action, status: "failed", fidelity: "exact", reason: error instanceof Error ? error.message : String(error) } + } + ;(evidence.controls ??= []).push(result) + return result +} + async function waitForMysqlDatabase(container: string, engine: keyof typeof MYSQL_IMAGES, password: string, dependencies: RuntimeServiceDependencies, timeoutMs: number, signal?: AbortSignal): Promise { const deadline = Date.now() + timeoutMs const client = engine === "mariadb" ? "mariadb" : "mysql" @@ -266,6 +411,33 @@ export async function waitForMysqlProtocol(host: string, port: number, timeoutMs throw new Error(`MySQL protocol readiness timed out after ${timeoutMs}ms`) } +export async function waitForTcpProtocol(host: string, port: number, timeoutMs: number, signal?: AbortSignal): Promise { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + throwIfAborted(signal) + try { + await tcpConnect(host, port, signal) + return + } catch (error) { + if (signal?.aborted) throw error + await abortableDelay(100, signal) + } + } + throw new Error(`TCP readiness timed out after ${timeoutMs}ms`) +} + +function tcpConnect(host: string, port: number, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + const socket = createConnection({ host, port }) + const timer = setTimeout(() => socket.destroy(new Error("connection timeout")), 1_000) + const abort = () => socket.destroy(new Error("aborted")) + const cleanup = () => { clearTimeout(timer); signal?.removeEventListener("abort", abort) } + signal?.addEventListener("abort", abort, { once: true }) + socket.once("connect", () => { cleanup(); socket.destroy(); resolve() }) + socket.once("error", (error) => { cleanup(); reject(error) }) + }) +} + function mysqlHandshake(host: string, port: number, signal?: AbortSignal): Promise { return new Promise((resolve, reject) => { const socket = createConnection({ host, port }) diff --git a/packages/runtime-core/src/adversarial-artifacts.ts b/packages/runtime-core/src/adversarial-artifacts.ts new file mode 100644 index 000000000..904b80152 --- /dev/null +++ b/packages/runtime-core/src/adversarial-artifacts.ts @@ -0,0 +1,81 @@ +import { createHash } from "node:crypto" + +import { ArtifactBundleWriter } from "./artifact-layout.js" +import type { ArtifactManifest } from "./artifact-manifest.js" +import type { AdversarialCampaignResult } from "./adversarial-campaign.js" + +export const ADVERSARIAL_EVIDENCE_BUNDLE_SCHEMA = "wp-codebox/adversarial-evidence-bundle/v1" as const + +export interface AdversarialEvidenceBundle { + schema: typeof ADVERSARIAL_EVIDENCE_BUNDLE_SCHEMA + path: string + manifestPath: string + resultPath: string + findingPaths: string[] + replayPaths: string[] + secretScanPath: string + contentDigest: string + bytes: number + redactions: number +} + +export async function writeAdversarialEvidenceBundle(directory: string, result: AdversarialCampaignResult, options: { maxBytes?: number; sensitiveValues?: string[]; createdAt?: string } = {}): Promise { + const maximum = options.maxBytes ?? 100 * 1_048_576 + const writer = new ArtifactBundleWriter(directory) + const findingPaths: string[] = [] + const replayPaths: string[] = [] + let bytes = 0 + let redactions = 0 + const writeJson = async (path: string, value: unknown, kind: string): Promise => { + const redacted = redactAdversarialEvidence(value, options.sensitiveValues ?? []) + redactions += redacted.redactions + bytes += Buffer.byteLength(redacted.contents) + if (bytes > maximum) throw new Error(`Adversarial evidence bundle exceeds ${maximum} bytes.`) + await writer.write(path, redacted.contents, { kind, contentType: "application/json", redaction: { policy: "applied", sensitive: true, reason: "Adversarial evidence is secret-scanned and path-normalized before publication." } }) + } + + await writeJson("result/adversarial-campaign-result.json", result, "adversarial-campaign-result") + for (const finding of result.findings) { + const findingPath = `findings/${finding.fingerprint}.json` + const replayPath = `replay/${finding.fingerprint}.json` + await writeJson(findingPath, finding, "adversarial-finding") + await writeJson(replayPath, finding.replay, "adversarial-replay") + findingPaths.push(findingPath) + replayPaths.push(replayPath) + } + const secretScanPath = "evidence/secret-scan.json" + await writeJson(secretScanPath, { schema: "wp-codebox/adversarial-secret-scan/v1", status: redactions > 0 ? "redacted" : "passed", redactions, scannedFiles: 1 + findingPaths.length + replayPaths.length }, "secret-scan") + + const contentDigest = createHash("sha256").update("wp-codebox/adversarial-evidence-bundle/v1\n").update(JSON.stringify({ campaignId: result.campaignId, seed: result.seed, fingerprints: result.findings.map((finding) => finding.fingerprint).sort() })).digest("hex") + const createdAt = options.createdAt ?? new Date().toISOString() + const manifest: ArtifactManifest = { + id: `${result.campaignId}-adversarial-evidence`, + contentDigest: { algorithm: "sha256", value: contentDigest, inputs: ["result/adversarial-campaign-result.json", ...findingPaths, ...replayPaths, secretScanPath] }, + createdAt, + runtime: { id: "adversarial-runtime", backend: "declared-adapter", environment: { kind: "runtime", name: "Adversarial campaign" }, createdAt, status: "created" }, + files: [], + } + await writer.writeManifest(manifest) + return { schema: ADVERSARIAL_EVIDENCE_BUNDLE_SCHEMA, path: directory, manifestPath: "manifest.json", resultPath: "result/adversarial-campaign-result.json", findingPaths, replayPaths, secretScanPath, contentDigest, bytes, redactions } +} + +export function redactAdversarialEvidence(value: unknown, sensitiveValues: readonly string[] = []): { contents: string; redactions: number } { + let contents = `${JSON.stringify(value, null, 2)}\n` + let redactions = 0 + const replace = (pattern: RegExp, replacement: string): void => { + const matches = contents.match(pattern) + redactions += matches?.length ?? 0 + contents = contents.replace(pattern, replacement) + } + replace(/("(?:authorization|cookie|password|passwd|secret|token|apiKey|api_key)"\s*:\s*")[^"]*(")/gi, "$1[redacted]$2") + replace(/("(?:path|sourcePath|workspace|cwd)"\s*:\s*")\/(?:home|Users|var|tmp)\/[^"\n]*(")/g, "$1[redacted-path]$2") + for (const secret of sensitiveValues.filter(Boolean)) { + const pattern = new RegExp(escapeRegExp(secret), "g") + replace(pattern, "[redacted]") + } + return { contents, redactions } +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") +} diff --git a/packages/runtime-core/src/adversarial-browser.ts b/packages/runtime-core/src/adversarial-browser.ts new file mode 100644 index 000000000..e90656317 --- /dev/null +++ b/packages/runtime-core/src/adversarial-browser.ts @@ -0,0 +1,136 @@ +import { createHash } from "node:crypto" + +import type { BrowserActionCorpusDescriptor, BrowserInteractionStep } from "./browser-interaction.js" + +export const ADVERSARIAL_BROWSER_PLAN_SCHEMA = "wp-codebox/adversarial-browser-plan/v1" as const +export const ADVERSARIAL_BROWSER_ORACLE_RESULT_SCHEMA = "wp-codebox/adversarial-browser-oracle-result/v1" as const +export const CLOCK_CONTROL_CAPABILITIES_SCHEMA = "wp-codebox/clock-control-capabilities/v1" as const + +export type AdversarialBrowserOracle = "crash" | "dead-control" | "stuck-interaction" | "layout" | "accessibility" | "duplicate-effect" + +export interface AdversarialBrowserPlan { + schema: typeof ADVERSARIAL_BROWSER_PLAN_SCHEMA + seed: string + startUrl: string + steps: BrowserInteractionStep[] + descriptorIds: string[] + matrix: { viewport: "desktop" | "mobile"; locale: string; timezone: string; slowDevice: boolean } + replay: { seed: string; startUrl: string; steps: BrowserInteractionStep[] } +} + +export interface AdversarialBrowserObservation { + pageErrors?: string[] + consoleErrors?: string[] + unhandledRejections?: string[] + controls?: Array<{ id: string; expectedAction: boolean; actionObserved: boolean; disabled?: boolean }> + loadingIndicators?: Array<{ id: string; visibleForMs: number }> + boxes?: Array<{ id: string; x: number; y: number; width: number; height: number; viewportWidth: number; viewportHeight: number; focused?: boolean; visible?: boolean; clipped?: boolean }> + accessibilityViolations?: Array<{ rule: string; target: string; impact?: string }> + effects?: Array<{ id: string; count: number; expectedMaximum?: number }> +} + +export interface AdversarialBrowserOracleResult { + schema: typeof ADVERSARIAL_BROWSER_ORACLE_RESULT_SCHEMA + failed: boolean + failures: Array<{ oracle: AdversarialBrowserOracle; code: string; message: string; target?: string }> +} + +export interface ClockControlCapability { + surface: "runtime" | "browser" | "scheduler" | "database" + freeze: boolean + advance: boolean + skew: boolean + restore: boolean + fidelity: "exact" | "emulated" | "unsupported" + reason?: string +} + +export interface ClockControlCapabilities { + schema: typeof CLOCK_CONTROL_CAPABILITIES_SCHEMA + adapter: string + capabilities: ClockControlCapability[] +} + +export function planAdversarialBrowserJourney(input: { seed: string; startUrl: string; descriptors: BrowserActionCorpusDescriptor[]; maxSteps?: number; locale?: string; timezone?: string }): AdversarialBrowserPlan { + const maximum = Math.max(1, Math.min(input.maxSteps ?? 12, 100)) + const descriptors = input.descriptors.filter((item) => !item.disabled && !item.readonly && item.selector).sort((left, right) => seededOrder(input.seed, left.id) - seededOrder(input.seed, right.id) || left.id.localeCompare(right.id)) + const steps: BrowserInteractionStep[] = [{ kind: "navigate", url: input.startUrl, waitFor: "load" }] + const descriptorIds: string[] = [] + for (const descriptor of descriptors) { + if (steps.length >= maximum) break + const generated = adversarialDescriptorSteps(descriptor, input.seed) + for (const step of generated) { + if (steps.length >= maximum) break + steps.push(step) + descriptorIds.push(descriptor.id) + } + } + if (steps.length < maximum) steps.push({ kind: "capture" }) + const matrix = { + viewport: seededOrder(input.seed, "viewport") % 2 === 0 ? "desktop" as const : "mobile" as const, + locale: input.locale ?? "en-US", + timezone: input.timezone ?? "UTC", + slowDevice: seededOrder(input.seed, "device") % 2 === 0, + } + return { schema: ADVERSARIAL_BROWSER_PLAN_SCHEMA, seed: input.seed, startUrl: input.startUrl, steps: steps.slice(0, maximum), descriptorIds, matrix, replay: { seed: input.seed, startUrl: input.startUrl, steps: steps.slice(0, maximum) } } +} + +export function evaluateAdversarialBrowserOracles(observation: AdversarialBrowserObservation, stuckThresholdMs = 10_000): AdversarialBrowserOracleResult { + const failures: AdversarialBrowserOracleResult["failures"] = [] + for (const message of [...(observation.pageErrors ?? []), ...(observation.unhandledRejections ?? [])]) failures.push({ oracle: "crash", code: "browser-runtime-error", message }) + for (const message of observation.consoleErrors ?? []) failures.push({ oracle: "crash", code: "browser-console-error", message }) + for (const control of observation.controls ?? []) { + if (control.expectedAction && !control.actionObserved && !control.disabled) failures.push({ oracle: "dead-control", code: "browser-dead-control", message: `Control ${control.id} accepted input without an observable action.`, target: control.id }) + } + for (const indicator of observation.loadingIndicators ?? []) { + if (indicator.visibleForMs >= stuckThresholdMs) failures.push({ oracle: "stuck-interaction", code: "browser-stuck-loading", message: `Loading indicator remained visible for ${indicator.visibleForMs}ms.`, target: indicator.id }) + } + for (const box of observation.boxes ?? []) { + const overflow = box.x < 0 || box.y < 0 || box.x + box.width > box.viewportWidth || box.y + box.height > box.viewportHeight + if (overflow || box.clipped) failures.push({ oracle: "layout", code: "browser-layout-overflow", message: `Element ${box.id} is clipped or outside the viewport.`, target: box.id }) + if (box.focused && box.visible === false) failures.push({ oracle: "layout", code: "browser-invisible-focus", message: `Focused element ${box.id} is not visible.`, target: box.id }) + } + for (const violation of observation.accessibilityViolations ?? []) failures.push({ oracle: "accessibility", code: `browser-a11y-${violation.rule}`, message: `Accessibility rule ${violation.rule} failed.`, target: violation.target }) + for (const effect of observation.effects ?? []) { + if (effect.count > (effect.expectedMaximum ?? 1)) failures.push({ oracle: "duplicate-effect", code: "browser-duplicate-effect", message: `Effect ${effect.id} occurred ${effect.count} times.`, target: effect.id }) + } + return { schema: ADVERSARIAL_BROWSER_ORACLE_RESULT_SCHEMA, failed: failures.length > 0, failures } +} + +export async function minimizeAdversarialBrowserJourney(steps: BrowserInteractionStep[], preservesFailure: (candidate: BrowserInteractionStep[]) => Promise): Promise { + let current = [...steps] + let chunk = Math.max(1, Math.floor(current.length / 2)) + while (current.length > 1 && chunk >= 1) { + let reduced = false + for (let start = 0; start < current.length; start += chunk) { + const candidate = [...current.slice(0, start), ...current.slice(start + chunk)] + if (candidate.length === 0) continue + if (await preservesFailure(candidate)) { current = candidate; reduced = true; break } + } + if (!reduced) chunk = Math.floor(chunk / 2) + } + return current +} + +export function clockControlCapabilities(adapter: string, capabilities: ClockControlCapability[]): ClockControlCapabilities { + return { schema: CLOCK_CONTROL_CAPABILITIES_SCHEMA, adapter, capabilities } +} + +function adversarialDescriptorSteps(descriptor: BrowserActionCorpusDescriptor, seed: string): BrowserInteractionStep[] { + if (descriptor.kind === "input" || descriptor.kind === "textarea") { + const values = ["", "A".repeat(4096), "\u202e\u2066hostile\u2069", "'\"<>\\", "😀".repeat(64)] + const value = values[seededOrder(seed, descriptor.id) % values.length] as string + return [{ kind: "fill", selector: descriptor.selector, value }] + } + if (descriptor.kind === "select") { + const value = descriptor.optionValues?.[seededOrder(seed, descriptor.id) % Math.max(descriptor.optionValues.length, 1)] + return value === undefined ? [] : [{ kind: "select", selector: descriptor.selector, value }] + } + if (descriptor.kind === "button") return [{ kind: "click", selector: descriptor.selector }, { kind: "click", selector: descriptor.selector }] + if (descriptor.kind === "link") return [{ kind: "click", selector: descriptor.selector }] + return [] +} + +function seededOrder(seed: string, value: string): number { + return createHash("sha256").update(`${seed}:${value}`).digest().readUInt32BE(0) +} diff --git a/packages/runtime-core/src/adversarial-campaign.ts b/packages/runtime-core/src/adversarial-campaign.ts new file mode 100644 index 000000000..6b356bddf --- /dev/null +++ b/packages/runtime-core/src/adversarial-campaign.ts @@ -0,0 +1,496 @@ +import { createHash } from "node:crypto" + +import { stableJson, stripUndefined } from "./object-utils.js" +import type { TransportFaultModel } from "./transport-faults.js" + +export const ADVERSARIAL_CAMPAIGN_SCHEMA = "wp-codebox/adversarial-campaign/v1" as const +export const ADVERSARIAL_CAMPAIGN_RESULT_SCHEMA = "wp-codebox/adversarial-campaign-result/v1" as const +export const ADVERSARIAL_FINDING_SCHEMA = "wp-codebox/adversarial-finding/v1" as const +export const ADVERSARIAL_REPLAY_SCHEMA = "wp-codebox/adversarial-replay/v1" as const +export const ADVERSARIAL_ORACLE_SCHEMA = "wp-codebox/adversarial-oracle/v1" as const +export const DIFFERENTIAL_RESULT_SCHEMA = "wp-codebox/differential-result/v1" as const + +export type AdversarialMutationKind = "scalar" | "structured" | "binary" | "sequence" +export type AdversarialCaseStatus = "passed" | "failed" | "error" | "timed-out" | "resource-exhausted" + +export interface AdversarialAction { + type: string + input?: unknown + metadata?: Record +} + +export interface AdversarialCorpusEntry { + id: string + actions: AdversarialAction[] + input?: unknown + signals?: string[] + metadata?: Record +} + +export interface AdversarialResourceBudget { + maxCases: number + maxActionsPerCase: number + maxInputBytes: number + maxCaseTimeMs: number + maxWallTimeMs: number + maxArtifactBytes: number + workers: number +} + +export interface AdversarialMatrixDimension { + name: string + values: string[] +} + +export interface AdversarialOracleContract { + schema: typeof ADVERSARIAL_ORACLE_SCHEMA + id: string + severity: "low" | "medium" | "high" | "critical" + description?: string + metadata?: Record +} + +export interface AdversarialCampaign { + schema: typeof ADVERSARIAL_CAMPAIGN_SCHEMA + id: string + seed: string + corpus: AdversarialCorpusEntry[] + mutationKinds: AdversarialMutationKind[] + budgets: AdversarialResourceBudget + oracles: AdversarialOracleContract[] + matrix?: AdversarialMatrixDimension[] + faults?: TransportFaultModel + provenance?: Record + replayCommand?: string + metadata?: Record +} + +export interface AdversarialCasePlan extends AdversarialCorpusEntry { + caseId: string + corpusId: string + iteration: number + workerId: number + matrix: Record + mutation: { kind: AdversarialMutationKind; path: string; description: string } +} + +export interface AdversarialExecutionObservation { + status: AdversarialCaseStatus + signals?: string[] + diagnostics?: Array<{ code: string; message: string; severity?: string; metadata?: Record }> + artifacts?: Array<{ path: string; kind: string; bytes?: number; sha256?: string }> + stateDigest?: string + metrics?: { durationMs?: number; memoryBytes?: number; cpuMs?: number; [name: string]: number | undefined } + metadata?: Record +} + +export interface AdversarialOracleResult { + oracleId: string + failed: boolean + code?: string + message?: string + evidence?: Record +} + +export interface AdversarialFinding { + schema: typeof ADVERSARIAL_FINDING_SCHEMA + fingerprint: string + caseId: string + corpusId: string + oracleIds: string[] + status: AdversarialCaseStatus + minimized: AdversarialCorpusEntry + original: AdversarialCorpusEntry + replay: AdversarialReplay + diagnostics: AdversarialExecutionObservation["diagnostics"] + artifactRefs: NonNullable + secretScan: { status: "passed" | "redacted"; redactions: number } + duplicates: number + matrix: Record +} + +export interface AdversarialReplay { + schema: typeof ADVERSARIAL_REPLAY_SCHEMA + campaignId: string + seed: string + caseId: string + corpusId: string + workerId: number + iteration: number + matrix: Record + actions: AdversarialAction[] + input?: unknown + faultSchedule?: TransportFaultModel + schedule: AdversarialScheduleEntry[] + provenance?: Record + command: string +} + +export interface AdversarialScheduleEntry { + round: number + workerId: number + caseId: string + corpusId: string + iteration: number +} + +export interface AdversarialCampaignResult { + schema: typeof ADVERSARIAL_CAMPAIGN_RESULT_SCHEMA + campaignId: string + seed: string + status: "passed" | "findings" | "incomplete" + summary: { generated: number; executed: number; retained: number; findings: number; duplicates: number; timedOut: number } + corpus: AdversarialCorpusEntry[] + findings: AdversarialFinding[] + schedule: AdversarialScheduleEntry[] + noveltySignals: string[] + diagnostics: Array<{ code: string; message: string }> + resourceUsage: { wallTimeMs: number; artifactBytes: number } +} + +export interface AdversarialCampaignRunnerOptions { + execute(plan: AdversarialCasePlan, signal: AbortSignal): Promise + evaluate?(plan: AdversarialCasePlan, observation: AdversarialExecutionObservation, oracles: readonly AdversarialOracleContract[]): Promise | AdversarialOracleResult[] + now?: () => number + replayCommand?: (campaign: AdversarialCampaign, plan: AdversarialCasePlan) => string +} + +export interface DifferentialCell { + id: string + role?: "base" | "candidate" + runtime?: Record + fingerprint?: string + status: AdversarialCaseStatus | "missing" +} + +export interface DifferentialResult { + schema: typeof DIFFERENTIAL_RESULT_SCHEMA + classification: "candidate-regression" | "pre-existing" | "platform-difference" | "nondeterminism" | "equivalent" + cells: DifferentialCell[] + fingerprints: string[] +} + +export function adversarialCampaign(input: Omit & { budgets?: Partial; mutationKinds?: AdversarialMutationKind[]; oracles?: AdversarialOracleContract[] }): AdversarialCampaign { + if (!input.id || !input.seed) throw new Error("Adversarial campaigns require non-empty id and seed values.") + if (input.corpus.length === 0) throw new Error("Adversarial campaigns require at least one corpus entry.") + const ids = new Set() + for (const entry of input.corpus) { + if (!entry.id || ids.has(entry.id)) throw new Error(`Adversarial corpus ids must be non-empty and unique: ${entry.id}`) + ids.add(entry.id) + } + return stripUndefined({ + schema: ADVERSARIAL_CAMPAIGN_SCHEMA, + id: input.id, + seed: input.seed, + corpus: input.corpus.map(normalizeCorpusEntry), + mutationKinds: input.mutationKinds?.length ? [...new Set(input.mutationKinds)] : ["scalar", "structured", "binary", "sequence"] as AdversarialMutationKind[], + budgets: normalizeBudgets(input.budgets), + oracles: input.oracles ?? [], + matrix: input.matrix, + faults: input.faults, + provenance: input.provenance, + replayCommand: input.replayCommand, + metadata: input.metadata, + }) +} + +export async function runAdversarialCampaign(campaignInput: AdversarialCampaign, options: AdversarialCampaignRunnerOptions): Promise { + const campaign = adversarialCampaign(campaignInput) + const started = (options.now ?? Date.now)() + const corpus = [...campaign.corpus] + const novelty = new Set(corpus.flatMap((entry) => entry.signals ?? [])) + const findings = new Map() + const schedule: AdversarialScheduleEntry[] = [] + const diagnostics: AdversarialCampaignResult["diagnostics"] = [] + let executed = 0 + let generated = 0 + let artifactBytes = 0 + let timedOut = 0 + let incomplete = false + + for (let round = 0; generated < campaign.budgets.maxCases; round += 1) { + if ((options.now ?? Date.now)() - started >= campaign.budgets.maxWallTimeMs) { incomplete = true; diagnostics.push({ code: "campaign-wall-time-exhausted", message: "Campaign stopped at its wall-time budget." }); break } + const roundPlans: AdversarialCasePlan[] = [] + for (let workerId = 0; workerId < campaign.budgets.workers && generated < campaign.budgets.maxCases; workerId += 1) { + const corpusEntry = corpus[deterministicInteger(`${campaign.seed}:source:${generated}`, corpus.length)] as AdversarialCorpusEntry + const plan = mutateCorpusEntry(campaign, corpusEntry, generated, workerId) + roundPlans.push(plan) + schedule.push({ round, workerId, caseId: plan.caseId, corpusId: plan.corpusId, iteration: plan.iteration }) + generated += 1 + } + const observations = await Promise.all(roundPlans.map(async (plan) => await executeBoundedCase(campaign, plan, options))) + for (let index = 0; index < roundPlans.length; index += 1) { + const plan = roundPlans[index] as AdversarialCasePlan + const observation = observations[index] as AdversarialExecutionObservation + executed += 1 + if (observation.status === "timed-out") timedOut += 1 + artifactBytes += (observation.artifacts ?? []).reduce((total, artifact) => total + (artifact.bytes ?? 0), 0) + if (artifactBytes > campaign.budgets.maxArtifactBytes) { incomplete = true; diagnostics.push({ code: "campaign-artifact-budget-exhausted", message: "Campaign stopped before artifact evidence exceeded its byte budget." }); break } + + const newSignals = (observation.signals ?? []).filter((signal) => !novelty.has(signal)) + for (const signal of newSignals) novelty.add(signal) + if (newSignals.length > 0) corpus.push({ id: plan.caseId, actions: plan.actions, input: plan.input, signals: [...new Set(observation.signals ?? [])], metadata: { retainedFrom: plan.corpusId, mutation: plan.mutation } }) + + const oracleResults = options.evaluate ? await options.evaluate(plan, observation, campaign.oracles) : defaultOracleResults(observation) + const failedOracles = oracleResults.filter((result) => result.failed) + if (failedOracles.length > 0 || observation.status !== "passed") { + const minimized = await minimizeAdversarialCase(campaign, plan, failedOracles, options) + const fingerprint = adversarialFindingFingerprint({ oracleIds: failedOracles.map((result) => result.oracleId), status: observation.status, diagnosticCodes: (observation.diagnostics ?? []).map((item) => item.code).sort(), matrix: plan.matrix }) + const existing = findings.get(fingerprint) + if (existing) existing.duplicates += 1 + else findings.set(fingerprint, createFinding(campaign, plan, minimized, observation, failedOracles, schedule, options, fingerprint)) + } + } + if (incomplete) break + } + + const duplicateCount = [...findings.values()].reduce((total, finding) => total + finding.duplicates, 0) + return { + schema: ADVERSARIAL_CAMPAIGN_RESULT_SCHEMA, + campaignId: campaign.id, + seed: campaign.seed, + status: incomplete ? "incomplete" : findings.size > 0 ? "findings" : "passed", + summary: { generated, executed, retained: corpus.length, findings: findings.size, duplicates: duplicateCount, timedOut }, + corpus, + findings: [...findings.values()].sort((left, right) => left.fingerprint.localeCompare(right.fingerprint)), + schedule, + noveltySignals: [...novelty].sort(), + diagnostics, + resourceUsage: { wallTimeMs: Math.max(0, (options.now ?? Date.now)() - started), artifactBytes }, + } +} + +export function mutateAdversarialValue(value: unknown, kind: AdversarialMutationKind, seed: string): { value: unknown; path: string; description: string } { + if (kind === "sequence") return { value, path: "$", description: "sequence mutation is applied to actions" } + const leaves = collectValueLeaves(value) + if (leaves.length === 0) return { value: adversarialBoundaryValue(undefined, seed, kind), path: "$", description: `${kind} root boundary` } + const leaf = leaves[deterministicInteger(seed, leaves.length)] as { path: Array; value: unknown } + const replacement = adversarialBoundaryValue(leaf.value, seed, kind) + if (leaf.path.length === 0) return { value: replacement, path: "$", description: `${kind} root boundary mutation` } + const mutated = cloneJsonValue(value) + setValueAtPath(mutated, leaf.path, replacement) + return { value: mutated, path: jsonPath(leaf.path), description: `${kind} boundary mutation` } +} + +export function adversarialFindingFingerprint(value: unknown): string { + return createHash("sha256").update("wp-codebox/adversarial-finding-fingerprint/v1\n").update(stableJson(value)).digest("hex") +} + +export function classifyDifferentialResult(cells: DifferentialCell[]): DifferentialResult { + const fingerprints = [...new Set(cells.map((cell) => cell.fingerprint).filter((value): value is string => Boolean(value)))].sort() + const base = cells.filter((cell) => cell.role === "base") + const candidate = cells.filter((cell) => cell.role === "candidate") + let classification: DifferentialResult["classification"] = "equivalent" + if (cells.some((cell) => cell.status === "missing") || cells.some((cell, index) => cells.findIndex((candidateCell) => candidateCell.id === cell.id) !== index)) classification = "nondeterminism" + else if (candidate.some(isDifferentialFailure) && base.every((cell) => !isDifferentialFailure(cell))) classification = "candidate-regression" + else if (candidate.some(isDifferentialFailure) && base.some(isDifferentialFailure)) classification = "pre-existing" + else if (fingerprints.length > 1 || new Set(cells.map((cell) => cell.status)).size > 1) classification = "platform-difference" + return { schema: DIFFERENTIAL_RESULT_SCHEMA, classification, cells, fingerprints } +} + +async function executeBoundedCase(campaign: AdversarialCampaign, plan: AdversarialCasePlan, options: AdversarialCampaignRunnerOptions): Promise { + const controller = new AbortController() + let timer: NodeJS.Timeout | undefined + try { + return await Promise.race([ + options.execute(plan, controller.signal), + new Promise((resolve) => { timer = setTimeout(() => { controller.abort(); resolve({ status: "timed-out", diagnostics: [{ code: "case-time-budget-exhausted", message: `Case exceeded ${campaign.budgets.maxCaseTimeMs}ms.` }] }) }, campaign.budgets.maxCaseTimeMs) }), + ]) + } catch (error) { + return { status: "error", diagnostics: [{ code: "case-execution-error", message: error instanceof Error ? error.message : String(error) }] } + } finally { + if (timer) clearTimeout(timer) + } +} + +function mutateCorpusEntry(campaign: AdversarialCampaign, source: AdversarialCorpusEntry, iteration: number, workerId: number): AdversarialCasePlan { + const kind = campaign.mutationKinds[deterministicInteger(`${campaign.seed}:kind:${iteration}`, campaign.mutationKinds.length)] as AdversarialMutationKind + let actions: AdversarialAction[] = source.actions.map((action) => ({ ...action, input: cloneJsonValue(action.input) })) + let input = cloneJsonValue(source.input) + let mutation = { kind, path: "$", description: `${kind} mutation` } + if (kind === "sequence") { + actions = mutateActionSequence(actions, `${campaign.seed}:sequence:${iteration}`, campaign.budgets.maxActionsPerCase) + mutation = { kind, path: "$.actions", description: "stateful action schedule mutation" } + } else if (actions.length > 0 && deterministicInteger(`${campaign.seed}:target:${iteration}`, 2) === 0) { + const actionIndex = deterministicInteger(`${campaign.seed}:action:${iteration}`, actions.length) + const result = mutateAdversarialValue(actions[actionIndex]?.input, kind, `${campaign.seed}:value:${iteration}`) + actions[actionIndex] = { ...(actions[actionIndex] as AdversarialAction), input: result.value } + mutation = { kind, path: `$.actions[${actionIndex}].input${result.path.slice(1)}`, description: result.description } + } else { + const result = mutateAdversarialValue(input, kind, `${campaign.seed}:value:${iteration}`) + input = result.value + mutation = { kind, path: `$.input${result.path.slice(1)}`, description: result.description } + } + actions = actions.slice(0, campaign.budgets.maxActionsPerCase) + if (jsonBytes(input) > campaign.budgets.maxInputBytes) input = truncateJsonValue(input, campaign.budgets.maxInputBytes) + const matrix = matrixCell(campaign.matrix, iteration) + return { id: `${source.id}-mutation-${iteration}`, caseId: `${campaign.id}-${iteration}`, corpusId: source.id, iteration, workerId, matrix, actions, input, mutation, metadata: source.metadata } +} + +async function minimizeAdversarialCase(campaign: AdversarialCampaign, plan: AdversarialCasePlan, originalOracles: AdversarialOracleResult[], options: AdversarialCampaignRunnerOptions): Promise { + const oracleIds = new Set(originalOracles.filter((item) => item.failed).map((item) => item.oracleId)) + const preserves = async (candidate: AdversarialCorpusEntry): Promise => { + const candidatePlan = { ...plan, actions: candidate.actions, input: candidate.input } + const observation = await executeBoundedCase(campaign, candidatePlan, options) + const oracleResults = options.evaluate ? await options.evaluate(candidatePlan, observation, campaign.oracles) : defaultOracleResults(observation) + return observation.status !== "passed" || oracleResults.some((item) => item.failed && (oracleIds.size === 0 || oracleIds.has(item.oracleId))) + } + let actions = [...plan.actions] + let chunk = Math.max(1, Math.floor(actions.length / 2)) + while (actions.length > 1 && chunk >= 1) { + let reduced = false + for (let start = 0; start < actions.length; start += chunk) { + const candidate = [...actions.slice(0, start), ...actions.slice(start + chunk)] + if (candidate.length === 0) continue + if (await preserves({ id: plan.caseId, actions: candidate, input: plan.input })) { actions = candidate; reduced = true; break } + } + if (!reduced) chunk = Math.floor(chunk / 2) + } + let input = plan.input + for (const candidateInput of shrinkAdversarialValue(input)) { + if (await preserves({ id: plan.caseId, actions, input: candidateInput })) input = candidateInput + } + return stripUndefined({ id: plan.caseId, actions, input, metadata: { minimizedFrom: plan.corpusId } }) +} + +function createFinding(campaign: AdversarialCampaign, plan: AdversarialCasePlan, minimized: AdversarialCorpusEntry, observation: AdversarialExecutionObservation, oracles: AdversarialOracleResult[], schedule: AdversarialScheduleEntry[], options: AdversarialCampaignRunnerOptions, fingerprint: string): AdversarialFinding { + const command = options.replayCommand?.(campaign, plan) ?? campaign.replayCommand ?? `wp-codebox adversarial replay --campaign ${campaign.id} --case ${plan.caseId}` + return { + schema: ADVERSARIAL_FINDING_SCHEMA, + fingerprint, + caseId: plan.caseId, + corpusId: plan.corpusId, + oracleIds: oracles.filter((item) => item.failed).map((item) => item.oracleId).sort(), + status: observation.status, + minimized, + original: { id: plan.caseId, actions: plan.actions, input: plan.input }, + replay: stripUndefined({ schema: ADVERSARIAL_REPLAY_SCHEMA, campaignId: campaign.id, seed: campaign.seed, caseId: plan.caseId, corpusId: plan.corpusId, workerId: plan.workerId, iteration: plan.iteration, matrix: plan.matrix, actions: minimized.actions, input: minimized.input, faultSchedule: campaign.faults, schedule: [...schedule], provenance: campaign.provenance, command }), + diagnostics: observation.diagnostics ?? [], + artifactRefs: observation.artifacts ?? [], + secretScan: { status: "passed", redactions: 0 }, + duplicates: 0, + matrix: plan.matrix, + } +} + +function defaultOracleResults(observation: AdversarialExecutionObservation): AdversarialOracleResult[] { + return observation.status === "passed" ? [] : [{ oracleId: "runtime-status", failed: true, code: observation.status, message: observation.diagnostics?.[0]?.message ?? `Runtime status was ${observation.status}.` }] +} + +function normalizeBudgets(input: Partial | undefined): AdversarialResourceBudget { + return { + maxCases: boundedInteger(input?.maxCases, 100, 1, 100_000), + maxActionsPerCase: boundedInteger(input?.maxActionsPerCase, 50, 1, 10_000), + maxInputBytes: boundedInteger(input?.maxInputBytes, 1_048_576, 1, 64 * 1_048_576), + maxCaseTimeMs: boundedInteger(input?.maxCaseTimeMs, 30_000, 1, 3_600_000), + maxWallTimeMs: boundedInteger(input?.maxWallTimeMs, 300_000, 1, 86_400_000), + maxArtifactBytes: boundedInteger(input?.maxArtifactBytes, 100 * 1_048_576, 1, 4 * 1024 * 1_048_576), + workers: boundedInteger(input?.workers, 1, 1, 128), + } +} + +function normalizeCorpusEntry(entry: AdversarialCorpusEntry): AdversarialCorpusEntry { + return stripUndefined({ ...entry, actions: entry.actions.map((action) => ({ ...action, input: cloneJsonValue(action.input) })), input: cloneJsonValue(entry.input), signals: entry.signals ? [...new Set(entry.signals)].sort() : undefined }) +} + +function mutateActionSequence(actions: AdversarialAction[], seed: string, maximum: number): AdversarialAction[] { + if (actions.length === 0) return [{ type: "noop", metadata: { generated: true } }] + const operation = deterministicInteger(seed, 4) + const index = deterministicInteger(`${seed}:index`, actions.length) + if (operation === 0 && actions.length > 1) return actions.filter((_, candidate) => candidate !== index) + if (operation === 1 && actions.length < maximum) return [...actions.slice(0, index), actions[index] as AdversarialAction, actions[index] as AdversarialAction, ...actions.slice(index + 1)] + if (operation === 2 && actions.length > 1) { + const reordered = [...actions] + const target = (index + 1) % actions.length + ;[reordered[index], reordered[target]] = [reordered[target] as AdversarialAction, reordered[index] as AdversarialAction] + return reordered + } + return [...actions, actions[index] as AdversarialAction].slice(0, maximum) +} + +function adversarialBoundaryValue(value: unknown, seed: string, kind: AdversarialMutationKind): unknown { + if (kind === "binary") { + const bytes = typeof value === "string" ? Buffer.from(value) : Buffer.from(stableJson(value)) + if (bytes.length === 0) return { encoding: "base64", data: "/w==" } + bytes[deterministicInteger(seed, bytes.length)] = (bytes[deterministicInteger(`${seed}:byte`, bytes.length)] ?? 0) ^ 0xff + return { encoding: "base64", data: bytes.toString("base64") } + } + if (typeof value === "string") { + const boundaries = ["", "\u0000", "'\"<>\\", "../".repeat(32), "A".repeat(4096), "\u202e\u2066hostile\u2069", "😀".repeat(256)] + return boundaries[deterministicInteger(seed, boundaries.length)] + } + if (typeof value === "number") return [0, -1, Number.MAX_SAFE_INTEGER, Number.MIN_SAFE_INTEGER][deterministicInteger(seed, 4)] + if (typeof value === "boolean") return !value + if (value === null || value === undefined) return { unexpected: true } + if (Array.isArray(value)) return value.length === 0 ? [null] : [...value, cloneJsonValue(value[0])] + if (typeof value === "object") return { ...(value as Record), __unexpected: { nested: true } } + return String(value) +} + +function shrinkAdversarialValue(value: unknown): unknown[] { + if (typeof value === "string") return ["", value.slice(0, Math.floor(value.length / 2)), value.slice(0, 1)] + if (typeof value === "number") return [0, Math.sign(value)] + if (Array.isArray(value)) return [[], value.slice(0, Math.max(1, Math.floor(value.length / 2)))] + if (value && typeof value === "object") { + const entries = Object.entries(value as Record) + return entries.map(([key]) => Object.fromEntries(entries.filter(([candidate]) => candidate !== key))) + } + return [] +} + +function collectValueLeaves(value: unknown, path: Array = []): Array<{ path: Array; value: unknown }> { + if (Array.isArray(value)) return value.flatMap((item, index) => collectValueLeaves(item, [...path, index])) + if (value && typeof value === "object") return Object.entries(value as Record).flatMap(([key, item]) => collectValueLeaves(item, [...path, key])) + return [{ path, value }] +} + +function setValueAtPath(root: unknown, path: Array, value: unknown): void { + if (path.length === 0) return + let current = root as Record + for (const segment of path.slice(0, -1)) current = current[segment] as Record + current[path[path.length - 1] as string | number] = value +} + +function matrixCell(dimensions: AdversarialMatrixDimension[] | undefined, iteration: number): Record { + const cell: Record = {} + let divisor = 1 + for (const dimension of dimensions ?? []) { + if (dimension.values.length === 0) continue + cell[dimension.name] = dimension.values[Math.floor(iteration / divisor) % dimension.values.length] as string + divisor *= dimension.values.length + } + return cell +} + +function truncateJsonValue(value: unknown, maximumBytes: number): unknown { + if (typeof value === "string") return value.slice(0, maximumBytes) + const serialized = stableJson(value) + return { truncated: true, sha256: createHash("sha256").update(serialized).digest("hex"), originalBytes: Buffer.byteLength(serialized) } +} + +function cloneJsonValue(value: T): T { + if (value === undefined) return value + return structuredClone(value) +} + +function jsonBytes(value: unknown): number { + return value === undefined ? 0 : Buffer.byteLength(stableJson(value)) +} + +function jsonPath(path: Array): string { + return `$${path.map((segment) => typeof segment === "number" ? `[${segment}]` : `.${segment}`).join("")}` +} + +function deterministicInteger(seed: string, maximum: number): number { + if (maximum <= 1) return 0 + return createHash("sha256").update(seed).digest().readUInt32BE(0) % maximum +} + +function boundedInteger(value: number | undefined, fallback: number, minimum: number, maximum: number): number { + if (!Number.isFinite(value)) return fallback + return Math.max(minimum, Math.min(maximum, Math.floor(value as number))) +} + +function isDifferentialFailure(cell: DifferentialCell): boolean { + return cell.status !== "passed" && cell.status !== "missing" +} diff --git a/packages/runtime-core/src/contracts.ts b/packages/runtime-core/src/contracts.ts index 067c27cca..57dce0ef5 100644 --- a/packages/runtime-core/src/contracts.ts +++ b/packages/runtime-core/src/contracts.ts @@ -6,6 +6,10 @@ export * from "./fuzz-fixture-plan-contracts.js" export * from "./fuzz-coverage-plan-contracts.js" export * from "./fuzz-case-minimizer.js" export * from "./fuzz-suite-contracts.js" +export * from "./adversarial-campaign.js" +export * from "./adversarial-browser.js" +export * from "./adversarial-artifacts.js" +export * from "./transport-faults.js" export * from "./mutation-isolation-contracts.js" export * from "./performance-observation.js" export * from "./cache-churn-observation.js" diff --git a/packages/runtime-core/src/index.ts b/packages/runtime-core/src/index.ts index e67776484..d3beb9db3 100644 --- a/packages/runtime-core/src/index.ts +++ b/packages/runtime-core/src/index.ts @@ -91,6 +91,10 @@ export * from "./fuzz-coverage-plan-contracts.js" export * from "./fuzz-suite-contracts.js" export * from "./fuzz-suite-runner.js" export * from "./fuzz-case-minimizer.js" +export * from "./adversarial-campaign.js" +export * from "./adversarial-browser.js" +export * from "./adversarial-artifacts.js" +export * from "./transport-faults.js" export * from "./rest-matrix-contracts.js" export * from "./mutation-isolation-contracts.js" export * from "./source-root-preparation.js" diff --git a/packages/runtime-core/src/public.ts b/packages/runtime-core/src/public.ts index 5e1a2d6e7..139acab4d 100644 --- a/packages/runtime-core/src/public.ts +++ b/packages/runtime-core/src/public.ts @@ -42,6 +42,10 @@ export * from "./fuzz-coverage-plan-contracts.js" export * from "./fuzz-case-minimizer.js" export * from "./fuzz-suite-contracts.js" export * from "./fuzz-suite-runner.js" +export * from "./adversarial-campaign.js" +export * from "./adversarial-browser.js" +export * from "./adversarial-artifacts.js" +export * from "./transport-faults.js" export * from "./rest-matrix-contracts.js" export * from "./host-command-executor.js" export * from "./host-tool-registry.js" diff --git a/packages/runtime-core/src/recipe-schema.ts b/packages/runtime-core/src/recipe-schema.ts index 083b7d3a4..4ab69f271 100644 --- a/packages/runtime-core/src/recipe-schema.ts +++ b/packages/runtime-core/src/recipe-schema.ts @@ -917,7 +917,7 @@ export function createWorkspaceRecipeJsonSchema(options: WorkspaceRecipeJsonSche required: ["id", "kind", "outputs"], properties: { id: { type: "string", pattern: "^[A-Za-z0-9][A-Za-z0-9_.-]*$" }, - kind: { const: "mysql" }, + kind: { enum: ["mysql", "redis", "smtp", "http"] }, configuration: { type: "object", additionalProperties: false, @@ -925,6 +925,9 @@ export function createWorkspaceRecipeJsonSchema(options: WorkspaceRecipeJsonSche engine: { enum: ["mysql", "mariadb"] }, rootAuthentication: { enum: ["generated-password", "empty-password"] }, foreignKeyTargetPolicy: { enum: ["unique-only", "indexed"] }, + image: { type: "string", minLength: 1 }, + responseStatus: { type: "integer", minimum: 100, maximum: 599 }, + responseBody: { type: "string", maxLength: 65536 }, }, }, outputs: { diff --git a/packages/runtime-core/src/runtime-contract-manifest.ts b/packages/runtime-core/src/runtime-contract-manifest.ts index 5b6638579..afe6cd8b6 100644 --- a/packages/runtime-core/src/runtime-contract-manifest.ts +++ b/packages/runtime-core/src/runtime-contract-manifest.ts @@ -33,6 +33,10 @@ import { BROWSER_CONTAINED_SITE_APPLY_PLAN_SCHEMA, BROWSER_CONTAINED_SITE_APPLY_ import { SANDBOX_ISOLATION_PROOF_SCHEMA } from "./sandbox-isolation-proof-contracts.js" import { CACHE_CHURN_OBSERVATION_SCHEMA } from "./cache-churn-observation.js" import { QUERY_OBSERVATION_SCHEMA } from "./query-observation-contracts.js" +import { ADVERSARIAL_CAMPAIGN_RESULT_SCHEMA, ADVERSARIAL_CAMPAIGN_SCHEMA, ADVERSARIAL_FINDING_SCHEMA, ADVERSARIAL_REPLAY_SCHEMA, DIFFERENTIAL_RESULT_SCHEMA } from "./adversarial-campaign.js" +import { ADVERSARIAL_BROWSER_ORACLE_RESULT_SCHEMA, ADVERSARIAL_BROWSER_PLAN_SCHEMA, CLOCK_CONTROL_CAPABILITIES_SCHEMA } from "./adversarial-browser.js" +import { ADVERSARIAL_EVIDENCE_BUNDLE_SCHEMA } from "./adversarial-artifacts.js" +import { TRANSPORT_FAULT_CAPABILITIES_SCHEMA, TRANSPORT_FAULT_EVIDENCE_SCHEMA, TRANSPORT_FAULT_MODEL_SCHEMA } from "./transport-faults.js" export const RUNTIME_CONTRACT_MANIFEST_SCHEMA = "wp-codebox/runtime-contract-manifest/v1" as const export const AGENT_TASK_RUN_REQUEST_SCHEMA = "wp-codebox/agent-task-run-request/v1" as const @@ -97,6 +101,20 @@ export const CODEBOX_PUBLIC_RUNTIME_READINESS = { } as const export const RUNTIME_CONTRACT_SCHEMAS = { + adversarial: { + campaign: ADVERSARIAL_CAMPAIGN_SCHEMA, + campaignResult: ADVERSARIAL_CAMPAIGN_RESULT_SCHEMA, + finding: ADVERSARIAL_FINDING_SCHEMA, + replay: ADVERSARIAL_REPLAY_SCHEMA, + evidenceBundle: ADVERSARIAL_EVIDENCE_BUNDLE_SCHEMA, + transportFaultModel: TRANSPORT_FAULT_MODEL_SCHEMA, + transportFaultCapabilities: TRANSPORT_FAULT_CAPABILITIES_SCHEMA, + transportFaultEvidence: TRANSPORT_FAULT_EVIDENCE_SCHEMA, + browserPlan: ADVERSARIAL_BROWSER_PLAN_SCHEMA, + browserOracleResult: ADVERSARIAL_BROWSER_ORACLE_RESULT_SCHEMA, + clockControlCapabilities: CLOCK_CONTROL_CAPABILITIES_SCHEMA, + differentialResult: DIFFERENTIAL_RESULT_SCHEMA, + }, agentTask: { runRequest: AGENT_TASK_RUN_REQUEST_SCHEMA, runResult: AGENT_TASK_RUN_RESULT_SCHEMA, diff --git a/packages/runtime-core/src/runtime-contracts.ts b/packages/runtime-core/src/runtime-contracts.ts index 3e018a643..9ea0d0c7e 100644 --- a/packages/runtime-core/src/runtime-contracts.ts +++ b/packages/runtime-core/src/runtime-contracts.ts @@ -136,11 +136,14 @@ export interface WorkspaceRecipeExternalServiceBoundary { /** A short-lived host resource provisioned before the sandbox runtime starts. */ export interface WorkspaceRecipeRuntimeService { id: string - kind: "mysql" | (string & {}) + kind: "mysql" | "redis" | "smtp" | "http" | (string & {}) configuration?: { engine?: "mysql" | "mariadb" rootAuthentication?: "generated-password" | "empty-password" foreignKeyTargetPolicy?: "unique-only" | "indexed" + image?: string + responseStatus?: number + responseBody?: string } /** Explicit map from a provider output (for example `port`) to a runtime env name. */ outputs: Record diff --git a/packages/runtime-core/src/transport-faults.ts b/packages/runtime-core/src/transport-faults.ts new file mode 100644 index 000000000..99d104c37 --- /dev/null +++ b/packages/runtime-core/src/transport-faults.ts @@ -0,0 +1,296 @@ +import { createHash } from "node:crypto" + +import { stableJson, stripUndefined } from "./object-utils.js" + +export const TRANSPORT_FAULT_MODEL_SCHEMA = "wp-codebox/transport-fault-model/v1" as const +export const TRANSPORT_FAULT_CAPABILITIES_SCHEMA = "wp-codebox/transport-fault-capabilities/v1" as const +export const TRANSPORT_FAULT_EVIDENCE_SCHEMA = "wp-codebox/transport-fault-evidence/v1" as const + +export type TransportFaultSemantic = + | "response-substitution" + | "malformed-response" + | "truncated-response" + | "chunked-response" + | "delay" + | "jitter" + | "bandwidth" + | "timeout" + | "connection-refusal" + | "connection-reset" + | "half-close" + | "disconnect-after-bytes" + | "host-remap" + | "request-corruption" + | "response-corruption" + +export type TransportFaultFidelity = "exact" | "emulated" | "unsupported" + +export interface TransportRequestMatcher { + host?: string + method?: string + path?: string + pathPattern?: string + headers?: Record +} + +export interface TransportFaultOutcome { + status?: number + headers?: Record + body?: string + bodyBase64?: string + malformed?: boolean + truncateAfterBytes?: number + chunkBytes?: number + delayMs?: number + jitterMs?: number + bandwidthBytesPerSecond?: number + timeoutMs?: number + connection?: "refuse" | "reset" | "half-close" + disconnectAfterBytes?: number + remapHost?: string + requestCorruption?: "truncate" | "flip-byte" | "invalid-encoding" + responseCorruption?: "flip-byte" | "invalid-encoding" + metadata?: Record +} + +export interface TransportFaultRule { + id: string + match: TransportRequestMatcher + sequence: TransportFaultOutcome[] + repeat?: "last" | "cycle" | "none" + metadata?: Record +} + +export interface TransportFaultModel { + schema: typeof TRANSPORT_FAULT_MODEL_SCHEMA + seed: string + rules: TransportFaultRule[] + redactHeaders?: string[] + metadata?: Record +} + +export interface TransportFaultCapability { + semantic: TransportFaultSemantic + fidelity: TransportFaultFidelity + reason?: string +} + +export interface TransportFaultCapabilities { + schema: typeof TRANSPORT_FAULT_CAPABILITIES_SCHEMA + adapter: string + capabilities: TransportFaultCapability[] +} + +export interface TransportFaultNegotiation { + supported: boolean + required: TransportFaultSemantic[] + unsupported: TransportFaultCapability[] + capabilities: TransportFaultCapabilities +} + +export interface TransportFaultRequest { + url: string + method: string + headers?: Record + body?: Uint8Array | string +} + +export interface TransportFaultDecision { + ruleId: string + sequenceIndex: number + invocation: number + outcome: TransportFaultOutcome + semantics: TransportFaultSemantic[] + delayMs: number +} + +export interface TransportFaultEvidence { + schema: typeof TRANSPORT_FAULT_EVIDENCE_SCHEMA + fingerprint: string + adapter: string + fidelity: TransportFaultCapability[] + request: { + url: string + method: string + headers: Record + bodyBytes?: number + } + fault?: { + ruleId: string + sequenceIndex: number + invocation: number + semantics: TransportFaultSemantic[] + delayMs: number + } + response?: { + status?: number + headers?: Record + bodyBytes?: number + connection?: string + } +} + +export class TransportFaultEngine { + readonly model: TransportFaultModel + readonly capabilities: TransportFaultCapabilities + readonly evidence: TransportFaultEvidence[] = [] + readonly #counts = new Map() + + constructor(model: TransportFaultModel, capabilities: TransportFaultCapabilities) { + this.model = transportFaultModel(model) + this.capabilities = capabilities + } + + decide(request: TransportFaultRequest): TransportFaultDecision | undefined { + const rule = this.model.rules.find((candidate) => transportRequestMatches(candidate.match, request)) + if (!rule) return undefined + const invocation = this.#counts.get(rule.id) ?? 0 + this.#counts.set(rule.id, invocation + 1) + const sequenceIndex = faultSequenceIndex(rule, invocation) + if (sequenceIndex === undefined) return undefined + const outcome = rule.sequence[sequenceIndex] as TransportFaultOutcome + const jitter = outcome.jitterMs ? deterministicRange(`${this.model.seed}:${rule.id}:${invocation}`, outcome.jitterMs) : 0 + return { + ruleId: rule.id, + sequenceIndex, + invocation, + outcome, + semantics: transportFaultOutcomeSemantics(outcome), + delayMs: Math.max(0, (outcome.delayMs ?? 0) + jitter), + } + } + + record(request: TransportFaultRequest, decision: TransportFaultDecision | undefined, response?: TransportFaultEvidence["response"]): TransportFaultEvidence { + const fidelity = decision + ? decision.semantics.map((semantic) => this.capabilities.capabilities.find((item) => item.semantic === semantic) ?? { semantic, fidelity: "unsupported" as const, reason: "Adapter did not declare this semantic." }) + : [] + const requestHeaders = redactTransportHeaders(request.headers ?? {}, this.model.redactHeaders) + const responseHeaders = response?.headers ? redactTransportHeaders(response.headers, this.model.redactHeaders) : undefined + const evidence = stripUndefined({ + schema: TRANSPORT_FAULT_EVIDENCE_SCHEMA, + fingerprint: transportFaultFingerprint({ adapter: this.capabilities.adapter, request: { url: redactTransportUrl(request.url), method: request.method.toUpperCase() }, fault: decision ? { ruleId: decision.ruleId, sequenceIndex: decision.sequenceIndex, semantics: decision.semantics } : undefined, response: response ? { status: response.status, connection: response.connection } : undefined }), + adapter: this.capabilities.adapter, + fidelity, + request: { + url: redactTransportUrl(request.url), + method: request.method.toUpperCase(), + headers: requestHeaders, + bodyBytes: transportBodyBytes(request.body), + }, + fault: decision ? { ruleId: decision.ruleId, sequenceIndex: decision.sequenceIndex, invocation: decision.invocation, semantics: decision.semantics, delayMs: decision.delayMs } : undefined, + response: response ? { ...response, headers: responseHeaders } : undefined, + }) + this.evidence.push(evidence) + return evidence + } +} + +export function transportFaultModel(input: Omit & { schema?: string }): TransportFaultModel { + if (!input.seed) throw new Error("Transport fault model requires a non-empty seed.") + const ids = new Set() + const rules = input.rules.map((rule) => { + if (!rule.id || ids.has(rule.id)) throw new Error(`Transport fault rule ids must be non-empty and unique: ${rule.id}`) + if (rule.sequence.length === 0) throw new Error(`Transport fault rule ${rule.id} requires at least one outcome.`) + ids.add(rule.id) + if (rule.match.pathPattern) new RegExp(rule.match.pathPattern) + return { ...rule, sequence: rule.sequence.map(normalizeTransportFaultOutcome) } + }) + return stripUndefined({ schema: TRANSPORT_FAULT_MODEL_SCHEMA, seed: input.seed, rules, redactHeaders: input.redactHeaders, metadata: input.metadata }) +} + +export function transportFaultCapabilities(adapter: string, capabilities: readonly TransportFaultCapability[]): TransportFaultCapabilities { + const bySemantic = new Map() + for (const capability of capabilities) bySemantic.set(capability.semantic, capability) + return { schema: TRANSPORT_FAULT_CAPABILITIES_SCHEMA, adapter, capabilities: [...bySemantic.values()].sort((left, right) => left.semantic.localeCompare(right.semantic)) } +} + +export function negotiateTransportFaults(model: TransportFaultModel, capabilities: TransportFaultCapabilities): TransportFaultNegotiation { + const required = [...new Set(model.rules.flatMap((rule) => rule.sequence.flatMap(transportFaultOutcomeSemantics)))].sort() + const unsupported = required.map((semantic) => capabilities.capabilities.find((item) => item.semantic === semantic) ?? { semantic, fidelity: "unsupported" as const, reason: "Adapter did not declare this semantic." }).filter((item) => item.fidelity === "unsupported") + return { supported: unsupported.length === 0, required, unsupported, capabilities } +} + +export function transportRequestMatches(matcher: TransportRequestMatcher, request: TransportFaultRequest): boolean { + let url: URL + try { url = new URL(request.url) } catch { return false } + if (matcher.host && url.host.toLowerCase() !== matcher.host.toLowerCase() && url.hostname.toLowerCase() !== matcher.host.toLowerCase()) return false + if (matcher.method && request.method.toUpperCase() !== matcher.method.toUpperCase()) return false + if (matcher.path && url.pathname !== matcher.path) return false + if (matcher.pathPattern && !new RegExp(matcher.pathPattern).test(url.pathname)) return false + const headers = normalizeHeaders(request.headers ?? {}) + return Object.entries(matcher.headers ?? {}).every(([name, value]) => headers[name.toLowerCase()] === value) +} + +export function transportFaultOutcomeSemantics(outcome: TransportFaultOutcome): TransportFaultSemantic[] { + const semantics: TransportFaultSemantic[] = [] + if (outcome.status !== undefined || outcome.headers || outcome.body !== undefined || outcome.bodyBase64 !== undefined) semantics.push("response-substitution") + if (outcome.malformed) semantics.push("malformed-response") + if (outcome.truncateAfterBytes !== undefined) semantics.push("truncated-response") + if (outcome.chunkBytes !== undefined) semantics.push("chunked-response") + if (outcome.delayMs !== undefined) semantics.push("delay") + if (outcome.jitterMs !== undefined) semantics.push("jitter") + if (outcome.bandwidthBytesPerSecond !== undefined) semantics.push("bandwidth") + if (outcome.timeoutMs !== undefined) semantics.push("timeout") + if (outcome.connection === "refuse") semantics.push("connection-refusal") + if (outcome.connection === "reset") semantics.push("connection-reset") + if (outcome.connection === "half-close") semantics.push("half-close") + if (outcome.disconnectAfterBytes !== undefined) semantics.push("disconnect-after-bytes") + if (outcome.remapHost) semantics.push("host-remap") + if (outcome.requestCorruption) semantics.push("request-corruption") + if (outcome.responseCorruption) semantics.push("response-corruption") + return [...new Set(semantics)] +} + +export function transportFaultFingerprint(value: unknown): string { + return createHash("sha256").update("wp-codebox/transport-fault-fingerprint/v1\n").update(stableJson(value)).digest("hex") +} + +export function redactTransportHeaders(headers: Record, extra: readonly string[] = []): Record { + const sensitive = new Set(["authorization", "cookie", "proxy-authorization", "set-cookie", "x-api-key", ...extra.map((name) => name.toLowerCase())]) + return Object.fromEntries(Object.entries(headers).map(([name, value]) => [name, sensitive.has(name.toLowerCase()) ? "[redacted]" : value])) +} + +export function redactTransportUrl(raw: string): string { + try { + const url = new URL(raw) + url.username = "" + url.password = "" + for (const key of [...url.searchParams.keys()]) { + if (/token|secret|nonce|password|passwd|pwd|key|signature|authorization/i.test(key)) url.searchParams.set(key, "[redacted]") + } + return url.toString() + } catch { + return raw.replace(/([?&](?:token|secret|nonce|password|key)=[^&#]*)/gi, "$1[redacted]") + } +} + +function normalizeTransportFaultOutcome(outcome: TransportFaultOutcome): TransportFaultOutcome { + for (const [name, value] of Object.entries(outcome)) { + if ((name.endsWith("Ms") || name.endsWith("Bytes") || name === "bandwidthBytesPerSecond") && typeof value === "number" && (!Number.isFinite(value) || value < 0)) { + throw new Error(`Transport fault outcome ${name} must be a finite non-negative number.`) + } + } + if (outcome.status !== undefined && (!Number.isInteger(outcome.status) || outcome.status < 100 || outcome.status > 599)) throw new Error("Transport fault status must be between 100 and 599.") + return { ...outcome, headers: outcome.headers ? normalizeHeaders(outcome.headers) : undefined } +} + +function faultSequenceIndex(rule: TransportFaultRule, invocation: number): number | undefined { + if (invocation < rule.sequence.length) return invocation + if ((rule.repeat ?? "last") === "cycle") return invocation % rule.sequence.length + if ((rule.repeat ?? "last") === "last") return rule.sequence.length - 1 + return undefined +} + +function deterministicRange(seed: string, maximum: number): number { + const digest = createHash("sha256").update(seed).digest() + return digest.readUInt32BE(0) % (Math.floor(maximum) + 1) +} + +function normalizeHeaders(headers: Record): Record { + return Object.fromEntries(Object.entries(headers).map(([name, value]) => [name.toLowerCase(), value])) +} + +function transportBodyBytes(body: TransportFaultRequest["body"]): number | undefined { + if (body === undefined) return undefined + return typeof body === "string" ? Buffer.byteLength(body) : body.byteLength +} diff --git a/packages/runtime-playground/src/browser-clock-control.ts b/packages/runtime-playground/src/browser-clock-control.ts new file mode 100644 index 000000000..cd926d346 --- /dev/null +++ b/packages/runtime-playground/src/browser-clock-control.ts @@ -0,0 +1,39 @@ +import { clockControlCapabilities, type ClockControlCapabilities } from "@automattic/wp-codebox-core" +import type { Page } from "playwright" + +export const PLAYWRIGHT_CLOCK_CONTROL_CAPABILITIES: ClockControlCapabilities = clockControlCapabilities("playwright", [ + { surface: "browser", freeze: true, advance: true, skew: true, restore: true, fidelity: "exact" }, + { surface: "runtime", freeze: false, advance: false, skew: false, restore: false, fidelity: "unsupported", reason: "Browser clock control does not alter the server process clock." }, + { surface: "scheduler", freeze: false, advance: false, skew: false, restore: false, fidelity: "unsupported", reason: "Server scheduler control requires a runtime extension." }, + { surface: "database", freeze: false, advance: false, skew: false, restore: false, fidelity: "unsupported", reason: "Browser clock control does not alter database time functions." }, +]) + +export interface BrowserClockController { + capabilities: ClockControlCapabilities + freeze(time: number | string | Date): Promise + advance(milliseconds: number): Promise + skew(time: number | string | Date): Promise + restore(): Promise +} + +export function createBrowserClockController(page: Page, initialTime: number | string | Date = Date.now()): BrowserClockController { + let installed = false + return { + capabilities: PLAYWRIGHT_CLOCK_CONTROL_CAPABILITIES, + async freeze(time) { + if (!installed) { await page.clock.install({ time: initialTime }); installed = true } + await page.clock.pauseAt(time) + }, + async advance(milliseconds) { + if (!installed) { await page.clock.install({ time: initialTime }); installed = true } + await page.clock.fastForward(milliseconds) + }, + async skew(time) { + if (!installed) { await page.clock.install({ time: initialTime }); installed = true } + await page.clock.setSystemTime(time) + }, + async restore() { + if (installed) await page.clock.resume() + }, + } +} diff --git a/packages/runtime-playground/src/browser-transport-faults.ts b/packages/runtime-playground/src/browser-transport-faults.ts new file mode 100644 index 000000000..37a3f0047 --- /dev/null +++ b/packages/runtime-playground/src/browser-transport-faults.ts @@ -0,0 +1,127 @@ +import { TransportFaultEngine, negotiateTransportFaults, transportFaultCapabilities, type TransportFaultCapability, type TransportFaultDecision, type TransportFaultEvidence, type TransportFaultModel } from "@automattic/wp-codebox-core" +import type { Route } from "playwright" + +const browserCapabilities: TransportFaultCapability[] = [ + { semantic: "response-substitution", fidelity: "exact" }, + { semantic: "malformed-response", fidelity: "emulated", reason: "Browser routing can return malformed payload bytes but not a malformed HTTP framing layer." }, + { semantic: "truncated-response", fidelity: "emulated", reason: "Browser routing returns a shortened complete body; it cannot terminate the transport mid-frame." }, + { semantic: "chunked-response", fidelity: "unsupported", reason: "The browser routing API owns response framing." }, + { semantic: "delay", fidelity: "exact" }, + { semantic: "jitter", fidelity: "exact" }, + { semantic: "bandwidth", fidelity: "emulated", reason: "Delivery time is delayed from body size; per-chunk transport pacing is unavailable." }, + { semantic: "timeout", fidelity: "emulated", reason: "The route is aborted with the browser timeout failure code after the declared interval." }, + { semantic: "connection-refusal", fidelity: "emulated", reason: "The browser reports a connection-refused request failure through route abortion." }, + { semantic: "connection-reset", fidelity: "emulated", reason: "The browser reports a connection-reset request failure through route abortion." }, + { semantic: "half-close", fidelity: "unsupported", reason: "The browser routing API does not expose socket half-close." }, + { semantic: "disconnect-after-bytes", fidelity: "unsupported", reason: "The browser routing API cannot disconnect a response after an exact byte offset." }, + { semantic: "host-remap", fidelity: "exact" }, + { semantic: "request-corruption", fidelity: "emulated", reason: "Request payload bytes can be replaced, but HTTP framing corruption is unavailable." }, + { semantic: "response-corruption", fidelity: "emulated", reason: "Response payload bytes can be replaced, but HTTP framing corruption is unavailable." }, +] + +export const BROWSER_TRANSPORT_FAULT_CAPABILITIES = transportFaultCapabilities("playwright-route", browserCapabilities) + +export interface BrowserTransportFaultAdapter { + engine: TransportFaultEngine + negotiation: ReturnType + handle(route: Route): Promise + evidence(): TransportFaultEvidence[] +} + +export function createBrowserTransportFaultAdapter(model: TransportFaultModel): BrowserTransportFaultAdapter { + const engine = new TransportFaultEngine(model, BROWSER_TRANSPORT_FAULT_CAPABILITIES) + const negotiation = negotiateTransportFaults(model, BROWSER_TRANSPORT_FAULT_CAPABILITIES) + return { + engine, + negotiation, + async handle(route) { return await applyBrowserTransportFault(route, engine) }, + evidence() { return [...engine.evidence] }, + } +} + +export async function applyBrowserTransportFault(route: Route, engine: TransportFaultEngine): Promise { + const request = route.request() + const transportRequest = { url: request.url(), method: request.method(), headers: request.headers(), body: request.postDataBuffer() ?? undefined } + const decision = engine.decide(transportRequest) + if (!decision) return false + const unsupported = decision.semantics.map((semantic) => engine.capabilities.capabilities.find((item) => item.semantic === semantic)).filter((item) => item?.fidelity === "unsupported") + if (unsupported.length > 0) { + engine.record(transportRequest, decision, { connection: "unsupported" }) + throw new Error(`Browser transport fault semantics are unsupported: ${unsupported.map((item) => item?.semantic).join(", ")}`) + } + + if (decision.delayMs > 0) await delay(decision.delayMs) + if (decision.outcome.timeoutMs !== undefined) { + await delay(decision.outcome.timeoutMs) + await route.abort("timedout") + engine.record(transportRequest, decision, { connection: "timedout" }) + return true + } + if (decision.outcome.connection) { + const code = decision.outcome.connection === "refuse" ? "connectionrefused" : decision.outcome.connection === "reset" ? "connectionreset" : "failed" + await route.abort(code) + engine.record(transportRequest, decision, { connection: decision.outcome.connection }) + return true + } + + const continuation = browserFaultContinuation(decision, request.url(), request.postDataBuffer() ?? undefined) + const needsResponse = browserFaultNeedsResponse(decision) + if (!needsResponse) { + await route.continue(continuation) + engine.record(transportRequest, decision) + return true + } + + const upstream = decision.outcome.status === undefined || decision.outcome.body === undefined && decision.outcome.bodyBase64 === undefined || decision.outcome.responseCorruption || decision.outcome.truncateAfterBytes !== undefined + ? await route.fetch(continuation) + : undefined + const originalBody = upstream ? await upstream.body() : Buffer.alloc(0) + let body = decision.outcome.bodyBase64 !== undefined ? Buffer.from(decision.outcome.bodyBase64, "base64") : decision.outcome.body !== undefined ? Buffer.from(decision.outcome.body) : originalBody + body = mutateResponseBody(body, decision) + if (decision.outcome.bandwidthBytesPerSecond && body.length > 0) await delay(Math.ceil(body.length / decision.outcome.bandwidthBytesPerSecond * 1000)) + const status = decision.outcome.status ?? upstream?.status() ?? 200 + const headers = { ...(upstream?.headers() ?? {}), ...(decision.outcome.headers ?? {}) } + await route.fulfill({ status, headers, body }) + engine.record(transportRequest, decision, { status, headers, bodyBytes: body.length }) + return true +} + +function browserFaultContinuation(decision: TransportFaultDecision, requestUrl: string, originalBody: Buffer | undefined): { url?: string; postData?: Buffer } { + const continuation: { url?: string; postData?: Buffer } = {} + if (decision.outcome.remapHost) { + const url = new URL(requestUrl) + const target = decision.outcome.remapHost.includes("://") ? new URL(decision.outcome.remapHost) : new URL(`${url.protocol}//${decision.outcome.remapHost}`) + url.protocol = target.protocol + url.hostname = target.hostname + url.port = target.port + continuation.url = url.toString() + } + if (decision.outcome.requestCorruption && originalBody) continuation.postData = corruptBytes(originalBody, decision.outcome.requestCorruption) + return continuation +} + +function browserFaultNeedsResponse(decision: TransportFaultDecision): boolean { + const outcome = decision.outcome + return outcome.status !== undefined || outcome.headers !== undefined || outcome.body !== undefined || outcome.bodyBase64 !== undefined || outcome.malformed === true || outcome.truncateAfterBytes !== undefined || outcome.responseCorruption !== undefined || outcome.bandwidthBytesPerSecond !== undefined +} + +function mutateResponseBody(body: Buffer, decision: TransportFaultDecision): Buffer { + let result: Buffer = Buffer.from(body) + if (decision.outcome.malformed) result = Buffer.from([0xff, 0xfe, 0x00, ...result.subarray(0, Math.min(result.length, 16))]) + if (decision.outcome.responseCorruption) result = corruptBytes(result, decision.outcome.responseCorruption) + if (decision.outcome.truncateAfterBytes !== undefined) result = result.subarray(0, decision.outcome.truncateAfterBytes) + return result +} + +function corruptBytes(input: Buffer, mode: "truncate" | "flip-byte" | "invalid-encoding"): Buffer { + if (mode === "truncate") return input.subarray(0, Math.floor(input.length / 2)) + if (mode === "invalid-encoding") return Buffer.concat([Buffer.from([0xff, 0xfe]), input]) + const output = Buffer.from(input) + if (output.length === 0) return Buffer.from([0xff]) + output[Math.floor(output.length / 2)] = (output[Math.floor(output.length / 2)] ?? 0) ^ 0xff + return output +} + +function delay(milliseconds: number): Promise { + return new Promise((resolve) => setTimeout(resolve, milliseconds)) +} diff --git a/packages/runtime-playground/src/index.ts b/packages/runtime-playground/src/index.ts index bd703e8b3..49ec6ddca 100644 --- a/packages/runtime-playground/src/index.ts +++ b/packages/runtime-playground/src/index.ts @@ -15,6 +15,8 @@ export { maintainPlaygroundCustomArchiveCache, playgroundWordPressArchiveCacheDi export { collectBrowserArtifactMetrics, collectWordPressEpisodeArtifacts, collectWordPressRuntimeArtifacts, createWordPressEpisode, createWordPressRuntime, runWordPressEpisodeActions, type WordPressEpisodeSpec, type WordPressRuntimeActionHooks, type WordPressRuntimeSpec } from "./public.js" export { preflightPhpWasmRuntimeAssets, PhpWasmRuntimeAssetIntegrityError, type PhpWasmRuntimeAssetPreflight, type PhpWasmRuntimeAssetPreflightOptions } from "./php-wasm-preflight.js" export { browserPreviewAuthCookieUrls, browserPreviewNetworkPolicySummary, browserPreviewReadinessError, browserPreviewRouting, browserPreviewSecureContextError, browserPreviewTopology, browserPreviewOrigins, resolveBrowserPreviewUrl, type BrowserPreviewNetworkPolicy, type BrowserPreviewTopology } from "./browser-preview-routing.js" +export { BROWSER_TRANSPORT_FAULT_CAPABILITIES, applyBrowserTransportFault, createBrowserTransportFaultAdapter, type BrowserTransportFaultAdapter } from "./browser-transport-faults.js" +export { PLAYWRIGHT_CLOCK_CONTROL_CAPABILITIES, createBrowserClockController, type BrowserClockController } from "./browser-clock-control.js" export { normalizePreviewReviewerAccess, previewReviewerAccess } from "./preview-reviewer-access.js" export { applyVfsMountSnapshots, materializePlaygroundMountsFromVfs, materializePlaygroundStagedInputs, type HostMountSnapshot, type MountMaterializationResult, type StagedInputMaterializationResult, type VfsMountSnapshot } from "./mount-materialization.js" export { buildReplayExportBlueprint, buildReplayableWordPressSiteBlueprint, buildReplayableWordPressSiteLimitations, writeReplayExportPackage, writeReplayableWordPressSiteBundle, type ReplayExportPackage, type ReplayExportPackageOptions, type ReplayableWordPressSiteBundle, type ReplayableWordPressSiteBundleManifest, type ReplayableWordPressSiteBundleOptions } from "./replayable-wordpress-site-bundle.js" diff --git a/tests/adversarial-browser.test.ts b/tests/adversarial-browser.test.ts new file mode 100644 index 000000000..69ad9153e --- /dev/null +++ b/tests/adversarial-browser.test.ts @@ -0,0 +1,36 @@ +import assert from "node:assert/strict" +import test from "node:test" + +import { evaluateAdversarialBrowserOracles, minimizeAdversarialBrowserJourney, planAdversarialBrowserJourney } from "../packages/runtime-core/src/adversarial-browser.js" + +const descriptors = [ + { id: "input:name", kind: "input" as const, selector: "#name", type: "text" }, + { id: "button:save", kind: "button" as const, selector: "#save", type: "submit" }, +] + +test("browser adversary discovers hostile inputs and repeated interactions deterministically", () => { + const first = planAdversarialBrowserJourney({ seed: "ui-seed", startUrl: "/fixture", descriptors, maxSteps: 8 }) + const second = planAdversarialBrowserJourney({ seed: "ui-seed", startUrl: "/fixture", descriptors, maxSteps: 8 }) + assert.deepEqual(first, second) + assert.equal(first.steps.filter((step) => step.kind === "click").length, 2) + assert.ok(first.steps.some((step) => step.kind === "fill")) +}) + +test("generic browser oracles report crashes, dead controls, layout, accessibility, duplicate effects, and stuck state", () => { + const result = evaluateAdversarialBrowserOracles({ + pageErrors: ["uncaught fixture error"], + controls: [{ id: "save", expectedAction: true, actionObserved: false }], + loadingIndicators: [{ id: "spinner", visibleForMs: 20_000 }], + boxes: [{ id: "dialog", x: 900, y: 0, width: 300, height: 100, viewportWidth: 1024, viewportHeight: 768 }], + accessibilityViolations: [{ rule: "label", target: "#name" }], + effects: [{ id: "save", count: 2 }], + }) + assert.equal(result.failed, true) + assert.deepEqual(new Set(result.failures.map((failure) => failure.oracle)), new Set(["crash", "dead-control", "stuck-interaction", "layout", "accessibility", "duplicate-effect"])) +}) + +test("browser journeys minimize automatically to the shortest reproduced failure", async () => { + const journey = planAdversarialBrowserJourney({ seed: "ui-seed", startUrl: "/fixture", descriptors, maxSteps: 8 }).steps + const minimized = await minimizeAdversarialBrowserJourney(journey, async (candidate) => candidate.filter((step) => step.kind === "click").length >= 2) + assert.deepEqual(minimized, [{ kind: "click", selector: "#save" }, { kind: "click", selector: "#save" }]) +}) diff --git a/tests/adversarial-campaign.test.ts b/tests/adversarial-campaign.test.ts new file mode 100644 index 000000000..15045c7e5 --- /dev/null +++ b/tests/adversarial-campaign.test.ts @@ -0,0 +1,99 @@ +import assert from "node:assert/strict" +import { mkdtemp, readFile, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" +import test from "node:test" + +import { adversarialCampaign, adversarialFindingFingerprint, classifyDifferentialResult, runAdversarialCampaign, type AdversarialCasePlan, type AdversarialExecutionObservation } from "../packages/runtime-core/src/adversarial-campaign.js" +import { writeAdversarialEvidenceBundle } from "../packages/runtime-core/src/adversarial-artifacts.js" + +const campaign = adversarialCampaign({ + id: "neutral-component", + seed: "repeatable-seed", + corpus: [{ + id: "stateful-seed", + input: { title: "safe", count: 1 }, + actions: [ + { type: "authenticate", input: { role: "reader" } }, + { type: "create", input: { value: "safe" } }, + { type: "commit" }, + { type: "observe" }, + ], + }], + mutationKinds: ["sequence"], + budgets: { maxCases: 6, workers: 3, maxCaseTimeMs: 1_000, maxWallTimeMs: 10_000 }, + oracles: [{ schema: "wp-codebox/adversarial-oracle/v1", id: "neutral-integrity", severity: "high" }], + provenance: { component: "neutral-intentionally-vulnerable-fixture", version: "1" }, +}) + +function vulnerableExecutor(active: { count: number; maximum: number; starts: string[] }) { + return async (plan: AdversarialCasePlan): Promise => { + active.count += 1 + active.maximum = Math.max(active.maximum, active.count) + active.starts.push(plan.caseId) + await new Promise((resolve) => setTimeout(resolve, 5)) + active.count -= 1 + const actionTypes = plan.actions.map((action) => action.type) + const duplicateCommit = actionTypes.filter((type) => type === "commit").length > 1 + const stateCorruption = actionTypes.indexOf("observe") < actionTypes.indexOf("commit") + const authorization = !actionTypes.includes("authenticate") + const diagnostics = [ + ...(duplicateCommit ? [{ code: "duplicate-effect", message: "commit executed twice" }] : []), + ...(stateCorruption ? [{ code: "state-corruption", message: "state observed before commit" }] : []), + ...(authorization ? [{ code: "authorization-bypass", message: "mutation executed without an actor" }] : []), + ] + return { status: diagnostics.length > 0 ? "failed" : "passed", diagnostics, signals: actionTypes.map((type) => `route:${type}`), artifacts: [{ path: `cases/${plan.caseId}.json`, kind: "case", bytes: 32 }] } + } +} + +test("stateful campaigns use true workers and retain deterministic replay schedules", async () => { + const firstActive = { count: 0, maximum: 0, starts: [] as string[] } + const secondActive = { count: 0, maximum: 0, starts: [] as string[] } + const first = await runAdversarialCampaign(campaign, { execute: vulnerableExecutor(firstActive) }) + const second = await runAdversarialCampaign(campaign, { execute: vulnerableExecutor(secondActive) }) + + assert.equal(firstActive.maximum, 3, "three cases execute concurrently in each deterministic round") + assert.deepEqual(first.schedule, second.schedule) + assert.deepEqual(first.corpus, second.corpus) + assert.deepEqual(first.findings.map((finding) => finding.fingerprint), second.findings.map((finding) => finding.fingerprint)) + assert.ok(first.findings.length > 0) + assert.ok(first.findings.every((finding) => finding.replay.command.includes("adversarial replay"))) + assert.ok(first.findings.every((finding) => finding.minimized.actions.length <= finding.original.actions.length)) + assert.ok(first.findings.every((finding) => finding.replay.schedule.length > 0)) + assert.equal(first.resourceUsage.artifactBytes <= campaign.budgets.maxArtifactBytes, true) +}) + +test("finding fingerprints deduplicate equivalent failures independently of payload", () => { + const left = adversarialFindingFingerprint({ oracleIds: ["authorization"], status: "failed", diagnosticCodes: ["denied"] }) + const right = adversarialFindingFingerprint({ status: "failed", diagnosticCodes: ["denied"], oracleIds: ["authorization"] }) + assert.equal(left, right) +}) + +test("differential matrices classify regressions and platform differences", () => { + assert.equal(classifyDifferentialResult([ + { id: "php-83-base", role: "base", status: "passed" }, + { id: "php-83-candidate", role: "candidate", status: "failed", fingerprint: "new" }, + ]).classification, "candidate-regression") + assert.equal(classifyDifferentialResult([ + { id: "sqlite", status: "passed", fingerprint: "a" }, + { id: "mysql", status: "passed", fingerprint: "b" }, + ]).classification, "platform-difference") +}) + +test("sealed finding bundles redact secrets and machine-specific paths", async () => { + const active = { count: 0, maximum: 0, starts: [] as string[] } + const result = await runAdversarialCampaign(campaign, { execute: vulnerableExecutor(active) }) + result.findings[0]!.replay.provenance = { password: "fixture-secret", cwd: "/var/lib/private/worktree" } + const directory = await mkdtemp(join(tmpdir(), "wp-codebox-adversarial-evidence-")) + try { + const bundle = await writeAdversarialEvidenceBundle(directory, result, { sensitiveValues: ["fixture-secret"], createdAt: "2026-07-24T00:00:00.000Z" }) + const manifest = JSON.parse(await readFile(join(directory, bundle.manifestPath), "utf8")) + const replay = await readFile(join(directory, bundle.replayPaths[0] as string), "utf8") + assert.equal(manifest.contentDigest.value, bundle.contentDigest) + assert.ok(manifest.files.some((file: { path: string }) => file.path === bundle.secretScanPath)) + assert.doesNotMatch(replay, /fixture-secret|\/var\/lib\/private/) + assert.match(replay, /\[redacted\]/) + } finally { + await rm(directory, { recursive: true, force: true }) + } +}) diff --git a/tests/browser-clock-control.test.ts b/tests/browser-clock-control.test.ts new file mode 100644 index 000000000..c90e50d37 --- /dev/null +++ b/tests/browser-clock-control.test.ts @@ -0,0 +1,23 @@ +import assert from "node:assert/strict" +import test from "node:test" + +import { PLAYWRIGHT_CLOCK_CONTROL_CAPABILITIES, createBrowserClockController } from "../packages/runtime-playground/src/browser-clock-control.js" + +test("browser clock control is executable and reports server surfaces unsupported", async () => { + const calls: Array<[string, unknown?]> = [] + const page = { clock: { + async install(options: unknown) { calls.push(["install", options]) }, + async pauseAt(time: unknown) { calls.push(["pauseAt", time]) }, + async fastForward(time: unknown) { calls.push(["fastForward", time]) }, + async setSystemTime(time: unknown) { calls.push(["setSystemTime", time]) }, + async resume() { calls.push(["resume"]) }, + } } as never + const controller = createBrowserClockController(page, "2026-01-01T00:00:00Z") + await controller.freeze("2026-01-02T00:00:00Z") + await controller.advance(5_000) + await controller.skew("2026-02-01T00:00:00Z") + await controller.restore() + assert.deepEqual(calls.map(([method]) => method), ["install", "pauseAt", "fastForward", "setSystemTime", "resume"]) + assert.equal(PLAYWRIGHT_CLOCK_CONTROL_CAPABILITIES.capabilities.find((item) => item.surface === "browser")?.fidelity, "exact") + assert.equal(PLAYWRIGHT_CLOCK_CONTROL_CAPABILITIES.capabilities.find((item) => item.surface === "runtime")?.fidelity, "unsupported") +}) diff --git a/tests/public-api-contract.test.ts b/tests/public-api-contract.test.ts index dde1589a2..36e1bb498 100644 --- a/tests/public-api-contract.test.ts +++ b/tests/public-api-contract.test.ts @@ -177,6 +177,10 @@ assert.deepEqual(barrelExportModules(publicBarrel), [ "./fuzz-case-minimizer.js", "./fuzz-suite-contracts.js", "./fuzz-suite-runner.js", + "./adversarial-campaign.js", + "./adversarial-browser.js", + "./adversarial-artifacts.js", + "./transport-faults.js", "./rest-matrix-contracts.js", "./host-command-executor.js", "./host-tool-registry.js", @@ -243,6 +247,10 @@ assert.deepEqual(barrelExportModules(contractsBarrel), [ "./fuzz-coverage-plan-contracts.js", "./fuzz-case-minimizer.js", "./fuzz-suite-contracts.js", + "./adversarial-campaign.js", + "./adversarial-browser.js", + "./adversarial-artifacts.js", + "./transport-faults.js", "./mutation-isolation-contracts.js", "./performance-observation.js", "./cache-churn-observation.js", diff --git a/tests/runtime-services.test.ts b/tests/runtime-services.test.ts index 9887a8b3c..9b64ac956 100644 --- a/tests/runtime-services.test.ts +++ b/tests/runtime-services.test.ts @@ -15,9 +15,16 @@ const plan = runtimeServicePlan([service]) assert.deepEqual(plan, [{ id: "test-db", kind: "mysql", provider: "docker", version: "mysql:8.4", bind: "loopback", port: "ephemeral", persistentVolume: false, outputs: service.outputs }]) assert.equal(parseLoopbackPort("127.0.0.1:44001\n"), 44001) assert.throws(() => parseLoopbackPort("0.0.0.0:3306"), /loopback/) +const auxiliaryServices = [ + { id: "cache", kind: "redis", outputs: { host: "REDIS_HOST", port: "REDIS_PORT", url: "REDIS_URL" } }, + { id: "mail", kind: "smtp", outputs: { host: "SMTP_HOST", port: "SMTP_PORT", httpPort: "SMTP_HTTP_PORT" } }, + { id: "upstream", kind: "http", configuration: { responseStatus: 503, responseBody: "unavailable" }, outputs: { url: "FIXTURE_URL" } }, +] satisfies WorkspaceRecipe["inputs"] extends { services?: infer T } ? NonNullable : never +assert.deepEqual(runtimeServicePlan(auxiliaryServices).map(({ kind, version }) => [kind, version]), [["redis", "redis:7.4-alpine"], ["smtp", "axllent/mailpit:v1.27"], ["http", "hashicorp/http-echo:1.0"]]) const valid = validateWorkspaceRecipeJsonSchema({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [service] }, workflow: { steps: [{ command: "wordpress.run-php" }] } }) assert.equal(valid.valid, true) +assert.equal(validateWorkspaceRecipeJsonSchema({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: auxiliaryServices }, workflow: { steps: [{ command: "wordpress.run-php" }] } }).valid, true) const unsafe = validateWorkspaceRecipeJsonSchema({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [{ ...service, outputs: { port: "bad-name" } }] }, workflow: { steps: [{ command: "wordpress.run-php" }] } }) assert.equal(unsafe.valid, false) const emptyRootService = { ...service, configuration: { rootAuthentication: "empty-password" as const } } @@ -158,6 +165,30 @@ const indexedForeignKeyRun = indexedForeignKeyCalls.find((call) => call.args[0] assert.deepEqual(indexedForeignKeyRun?.args.slice(-2), ["mysql:8.4", "--restrict-fk-on-non-standard-key=OFF"], "indexed foreign-key targets opt into the MySQL compatibility mode") await indexedForeignKey.release() +const auxiliaryCalls: string[][] = [] +const auxiliaryDependencies: RuntimeServiceDependencies = { + ...dependencies, + async execute(command, args, options) { + auxiliaryCalls.push(args) + return await dependencies.execute(command, args, options) + }, +} +const auxiliary = await provisionRuntimeServices(auxiliaryServices, { dependencies: auxiliaryDependencies }) +assert.equal(auxiliary.env.REDIS_URL, "redis://127.0.0.1:41001") +assert.equal(auxiliary.env.FIXTURE_URL, "http://127.0.0.1:41001") +assert.equal((await auxiliary.control("cache", "pause")).status, "applied") +assert.equal((await auxiliary.control("cache", "resume")).status, "applied") +assert.equal((await auxiliary.control("cache", "stop")).status, "applied") +assert.equal((await auxiliary.control("cache", "start")).status, "applied") +assert.equal((await auxiliary.control("cache", "flush")).status, "applied") +assert.equal((await auxiliary.control("mail", "restart")).status, "applied") +assert.equal((await auxiliary.control("upstream", "latency", { milliseconds: 50 })).status, "unsupported") +assert.ok(auxiliaryCalls.some((args) => args.includes("redis:7.4-alpine"))) +assert.ok(auxiliaryCalls.some((args) => args.includes("axllent/mailpit:v1.27"))) +assert.ok(auxiliaryCalls.some((args) => args.includes("hashicorp/http-echo:1.0"))) +assert.equal(auxiliary.evidence.find((item) => item.id === "cache")?.controls?.length, 5) +await auxiliary.release() + const mariaDbCalls: Array<{ args: string[]; env?: NodeJS.ProcessEnv }> = [] const mariaDbDependencies: RuntimeServiceDependencies = { ...dependencies, diff --git a/tests/transport-faults.test.ts b/tests/transport-faults.test.ts new file mode 100644 index 000000000..eba19caed --- /dev/null +++ b/tests/transport-faults.test.ts @@ -0,0 +1,68 @@ +import assert from "node:assert/strict" +import test from "node:test" + +import { TransportFaultEngine, negotiateTransportFaults, redactTransportHeaders, transportFaultCapabilities, transportFaultModel } from "../packages/runtime-core/src/transport-faults.js" +import { applyBrowserTransportFault, createBrowserTransportFaultAdapter } from "../packages/runtime-playground/src/browser-transport-faults.js" + +const model = transportFaultModel({ + seed: "fault-seed", + redactHeaders: ["x-private"], + rules: [{ + id: "verification-sequence", + match: { host: "service.example", method: "POST", path: "/verify", headers: { "x-mode": "test" } }, + sequence: [{ status: 500 }, { delayMs: 25, jitterMs: 5, timeoutMs: 10 }, { status: 200, body: "malformed", truncateAfterBytes: 4 }], + repeat: "cycle", + }], +}) + +test("transport fault sequences and evidence are deterministic and redacted", () => { + const capabilities = transportFaultCapabilities("fixture", [ + { semantic: "response-substitution", fidelity: "exact" }, + { semantic: "delay", fidelity: "exact" }, + { semantic: "jitter", fidelity: "exact" }, + { semantic: "timeout", fidelity: "exact" }, + { semantic: "truncated-response", fidelity: "emulated" }, + ]) + const first = new TransportFaultEngine(model, capabilities) + const second = new TransportFaultEngine(model, capabilities) + const request = { url: "https://service.example/verify?token=secret", method: "post", headers: { authorization: "Bearer secret", "x-private": "private", "x-mode": "test" }, body: "payload" } + + const firstDecisions = [first.decide(request), first.decide(request), first.decide(request), first.decide(request)] + const secondDecisions = [second.decide(request), second.decide(request), second.decide(request), second.decide(request)] + assert.deepEqual(firstDecisions, secondDecisions) + assert.deepEqual(firstDecisions.map((decision) => decision?.sequenceIndex), [0, 1, 2, 0]) + assert.ok((firstDecisions[1]?.delayMs ?? 0) >= 25) + + const evidence = first.record(request, firstDecisions[0], { status: 500, headers: { "set-cookie": "session=secret" } }) + assert.equal(evidence.request.headers.authorization, "[redacted]") + assert.equal(evidence.request.headers["x-private"], "[redacted]") + assert.equal(evidence.response?.headers?.["set-cookie"], "[redacted]") + assert.doesNotMatch(JSON.stringify(evidence), /Bearer secret|session=secret|token=secret/) +}) + +test("fault negotiation reports unsupported transport semantics honestly", () => { + const capabilities = transportFaultCapabilities("limited", [{ semantic: "response-substitution", fidelity: "exact" }]) + const negotiation = negotiateTransportFaults(model, capabilities) + assert.equal(negotiation.supported, false) + assert.deepEqual(negotiation.unsupported.map((item) => item.semantic), ["delay", "jitter", "timeout", "truncated-response"]) + assert.deepEqual(redactTransportHeaders({ Cookie: "secret", Accept: "json" }), { Cookie: "[redacted]", Accept: "json" }) +}) + +test("browser adapter applies response faults and rejects socket semantics it cannot provide", async () => { + const adapter = createBrowserTransportFaultAdapter(transportFaultModel({ seed: "browser", rules: [{ id: "replace", match: { host: "fixture.test", path: "/api" }, sequence: [{ status: 503, body: "offline", headers: { "x-fixture": "yes" } }] }] })) + const calls: Array<{ method: string; input?: unknown }> = [] + const route = { + request: () => ({ url: () => "https://fixture.test/api", method: () => "GET", headers: () => ({}), postDataBuffer: () => null }), + async fulfill(input: unknown) { calls.push({ method: "fulfill", input }) }, + async continue(input: unknown) { calls.push({ method: "continue", input }) }, + async abort(input: unknown) { calls.push({ method: "abort", input }) }, + async fetch() { throw new Error("substituted response must not fetch upstream") }, + } as never + assert.equal(await applyBrowserTransportFault(route, adapter.engine), true) + assert.equal((calls[0]?.input as { status?: number }).status, 503) + assert.equal(adapter.evidence().length, 1) + + const unsupported = createBrowserTransportFaultAdapter(transportFaultModel({ seed: "socket", rules: [{ id: "half-close", match: { host: "fixture.test" }, sequence: [{ connection: "half-close" }] }] })) + assert.equal(unsupported.negotiation.supported, false) + await assert.rejects(applyBrowserTransportFault(route, unsupported.engine), /half-close/) +})