diff --git a/.github/workflows/agent-task-contracts.yml b/.github/workflows/agent-task-contracts.yml index 616c976b5..466528580 100644 --- a/.github/workflows/agent-task-contracts.yml +++ b/.github/workflows/agent-task-contracts.yml @@ -14,6 +14,9 @@ on: - "fixtures/agent-task-runtime-paths-run-29305012941.json" - "package-lock.json" - "package.json" + - "packages/runtime-playground/src/**" + - "tests/browser-*.test.ts" + - "tests/editor-*.test.ts" - "tests/agent-task-*.test.ts" - "tests/runtime-sources-materialization.test.ts" - "tests/runtime-sources-playground-integration.test.ts" @@ -58,6 +61,9 @@ on: - "fixtures/agent-task-runtime-paths-run-29305012941.json" - "package-lock.json" - "package.json" + - "packages/runtime-playground/src/**" + - "tests/browser-*.test.ts" + - "tests/editor-*.test.ts" - "tests/agent-task-*.test.ts" - "tests/runtime-sources-materialization.test.ts" - "tests/runtime-sources-playground-integration.test.ts" @@ -129,6 +135,8 @@ jobs: - run: npm run test:trusted-apply-artifact-channel - run: npm run test:runtime-command-artifact-bounds - run: npm run test:redaction + - run: npm run test:browser-preview-routing + - run: npm run test:browser-routed-command-security - run: npm run test:production-boundary-enforcement - run: npm run test:runtime-tool-policy diff --git a/package.json b/package.json index 358fbae3a..f4cd8b4ce 100644 --- a/package.json +++ b/package.json @@ -109,6 +109,7 @@ "smoke": "tsx scripts/run-smoke.ts", "test:redaction": "tsx tests/redaction.test.ts", "test:browser-preview-routing": "tsx --test tests/browser-preview-routing.test.ts", + "test:browser-routed-command-security": "tsx --test tests/browser-routed-command-security.test.ts", "test:cloudflare-runtime": "node --test tests/cloudflare-d1-provisioner.test.mjs && tsx tests/cloudflare-site-context.test.ts && tsx tests/cloudflare-coordinator-site-partitioning.test.ts && tsx tests/cloudflare-d1-operation-repository.test.ts && tsx tests/cloudflare-provisioning-api.test.ts && tsx tests/cloudflare-runtime.test.ts && node ./node_modules/typescript/bin/tsc -p packages/runtime-cloudflare --noEmit", "test:cloudflare-administrator-claim": "tsx tests/cloudflare-provisioning-api.test.ts", "test:cloudflare-wordpress-auth": "tsx tests/cloudflare-wordpress-auth.test.ts", diff --git a/packages/runtime-playground/src/browser-actions-runner.ts b/packages/runtime-playground/src/browser-actions-runner.ts index dfb888b48..611adc3c6 100644 --- a/packages/runtime-playground/src/browser-actions-runner.ts +++ b/packages/runtime-playground/src/browser-actions-runner.ts @@ -12,7 +12,8 @@ import { browserAssertionsSummary, browserStepRecord, executeBrowserInteractionS import { browserCommandLivenessPolicy, isBrowserCommandLivenessError, withBrowserCommandLiveness } from "./browser-liveness.js" import { serializeBrowserError } from "./browser-metrics.js" import { executeBrowserObservationAssertion } from "./browser-observation-assertions.js" -import { browserPreviewNetworkPolicyIsActive, browserPreviewNetworkPolicySummary, browserPreviewNeedsContextRouting, browserPreviewReadinessError, browserPreviewTopology, resolveBrowserPreviewUrl, routeBrowserPreviewContextNetwork } from "./browser-preview-routing.js" +import { browserPreviewNetworkPolicyIsActive, browserPreviewNetworkPolicySummary, browserPreviewNeedsContextRouting, browserPreviewReadinessError, browserPreviewTopology, closeBrowserAndDrainPreviewRoutes, createBrowserPreviewRouteTracker, resolveBrowserPreviewUrl, routeBrowserPreviewContextNetwork } from "./browser-preview-routing.js" +import { browserCommandResult } from "./browser-result-sanitization.js" import { BROWSER_PROBE_STATE_INIT_SCRIPT, browserProbeReplayability, browserProbeViewport } from "./browser-probe.js" import { runBrowserProbeCommand, type BrowserProbeRunPlan } from "./browser-probe-runner.js" import { browserActionTargetUrls, browserAuthRequest, browserProbeWaterfallArtifact, browserProbeWebSocketArtifact, browserProbeWebSocketSummary, browserRedirectDiagnosticsArtifact, browserRequestCoverageArtifact, browserStorageStateAuthSummary, browserStorageStateImportFromArgs, browserWordPressDiagnosticsArtifact, createBrowserProbeProgressTracker, fileSha256, installBrowserWordPressDiagnostics, installWordPressAdminAuthCookies, livenessRemainingWallTimeMs, normalizeBrowserProbeScriptCheckpoint, type BrowserCommandProgressEvent, type BrowserStorageStateImport } from "./browser-probe-support.js" @@ -133,6 +134,7 @@ export async function runBrowserActionsCommand({ const browser = session?.browser ?? await launchChromiumBrowser() const topology = browserPreviewTopology(args, runtimeSpec, server.serverUrl, server.previewProxyDiagnostics?.targetOrigin) const { preview, networkPolicy } = topology + const routeTracker = session?.routeTracker ?? createBrowserPreviewRouteTracker() let requestedUrl = initialUrl ? topology.resolveUrl(initialUrl) : preview.effectiveOrigin let finalUrl = requestedUrl let htmlSha256: string | undefined @@ -173,7 +175,7 @@ export async function runBrowserActionsCommand({ }) : undefined) const context = environmentRuntime?.context ?? null if (context && !session) { - await routeBrowserPreviewContextNetwork(context, networkPolicy, topology.origins.localProxyOrigin) + await routeBrowserPreviewContextNetwork(context, networkPolicy, topology.origins.localProxyOrigin, routeTracker) } const page = activePage = environmentRuntime?.page ?? await browser.newPage() if (onProgress) { @@ -437,11 +439,12 @@ export async function runBrowserActionsCommand({ pendingError = error instanceof Error ? error : new Error(String(error)) errors.push(serializeBrowserError("probe-error", error)) } finally { - await settleBrowserNetworkTasks(networkTasks, livenessPolicy.networkSettleTimeoutMs) + await settleBrowserNetworkTasks(networkTasks, livenessPolicy.networkSettleTimeoutMs).catch((error) => errors.push(serializeBrowserError("probe-error", error))) if (activePage && resolvedEnvironment) environmentEvidence = await observePlaywrightBrowserEnvironment(activePage, requestedEnvironment, resolvedEnvironment).catch(() => environmentEvidence) - if (!session) { - await environmentRuntime?.close().catch(() => undefined) - await browser.close() + const cleanupBrowser = session ? { close: async () => {} } : { close: async () => { await environmentRuntime?.close(); await browser.close() } } + for (const routeError of await closeBrowserAndDrainPreviewRoutes(cleanupBrowser, routeTracker)) { + errors.push(serializeBrowserError("probe-error", routeError)) + pendingError ??= routeError } if (capture.has("steps")) { await artifactSession.writeJsonLines("steps", "steps.jsonl", stepRecords) @@ -580,9 +583,7 @@ export async function runBrowserActionsCommand({ throw new Error("wordpress.browser-actions did not produce a browser artifact") } - return { - artifact, - output: `${JSON.stringify({ + return browserCommandResult(artifact, { command: "wordpress.browser-actions", requestedUrl, preview, @@ -591,8 +592,7 @@ export async function runBrowserActionsCommand({ files: artifact.files, summary: artifact.summary, steps: stepRecords, - }, null, 2)}\n`, - } + }) } export function browserToolVerifierUnsupportedResult(step: BrowserInteractionStep, index: number, startedAt: string): BrowserToolVerifierResult { @@ -967,6 +967,7 @@ export async function runBrowserScenarioCommand({ let pendingError: Error | undefined let scenarioSession: PlaywrightBrowserEnvironmentSession | undefined let scenarioBrowser: Awaited> | undefined + let scenarioRouteTracker: ReturnType | undefined try { if (runPlan.probe && runPlan.actions) { @@ -976,12 +977,16 @@ export async function runBrowserScenarioCommand({ if (unsupported.length > 0) { throw new Error(`wordpress.browser-scenario browser environment is unsupported: ${unsupported.join(", ")}`) } + const topology = browserPreviewTopology(args, runtimeSpec, server.serverUrl, server.previewProxyDiagnostics?.targetOrigin) const runtime = await createPlaywrightBrowserEnvironmentContext(browser, resolved, { - ...(runPlan.actions.storageStateImport ? { contextOptions: { storageState: runPlan.actions.storageStateImport.storageState } } : {}), + contextOptions: { + ...topology.contextOptions(), + ...(runPlan.actions.storageStateImport ? { storageState: runPlan.actions.storageStateImport.storageState } : {}), + }, }) - const topology = browserPreviewTopology(args, runtimeSpec, server.serverUrl) - if (browserPreviewNeedsContextRouting(topology.networkPolicy)) await routeBrowserPreviewContextNetwork(runtime.context, topology.networkPolicy, topology.preview.effectiveOrigin) - scenarioSession = { browser, requested: requestedEnvironment, resolved, runtime } + const routeTracker = scenarioRouteTracker = createBrowserPreviewRouteTracker() + if (browserPreviewNeedsContextRouting(topology.networkPolicy)) await routeBrowserPreviewContextNetwork(runtime.context, topology.networkPolicy, topology.origins.localProxyOrigin, routeTracker) + scenarioSession = { browser, requested: requestedEnvironment, resolved, routeTracker, runtime } } if (runPlan.probe) { @@ -1006,8 +1011,15 @@ export async function runBrowserScenarioCommand({ } } } finally { - await scenarioSession?.runtime.close().catch(() => undefined) - await scenarioBrowser?.close().catch(() => undefined) + if (scenarioSession && scenarioBrowser && scenarioRouteTracker) { + const activeSession = scenarioSession + const activeBrowser = scenarioBrowser + const cleanupBrowser = { close: async () => { await activeSession.runtime.close(); await activeBrowser.close() } } + const routeErrors = await closeBrowserAndDrainPreviewRoutes(cleanupBrowser, scenarioRouteTracker) + pendingError ??= routeErrors[0] + } else { + await scenarioBrowser?.close().catch(() => undefined) + } } const primaryArtifact = actionsResult?.artifact ?? probeResult?.artifact @@ -1067,17 +1079,14 @@ export async function runBrowserScenarioCommand({ throw new BrowserCommandArtifactError(`wordpress.browser-scenario failed: ${pendingError.message}`, artifact) } - return { - artifact, - output: `${JSON.stringify({ + return browserCommandResult(artifact, { command: "wordpress.browser-scenario", requestedUrl: artifact.requestedUrl, finalUrl, files: artifact.files, summary: artifact.summary, scenario: scenarioSummary, - }, null, 2)}\n`, - } + }) } async function browserScenarioFromArgs(args: string[]): Promise { diff --git a/packages/runtime-playground/src/browser-artifact-session.ts b/packages/runtime-playground/src/browser-artifact-session.ts index c17b91c8e..4be741320 100644 --- a/packages/runtime-playground/src/browser-artifact-session.ts +++ b/packages/runtime-playground/src/browser-artifact-session.ts @@ -3,6 +3,7 @@ import { basename, join } from "node:path" import type { ArtifactProvenanceMetadata } from "@automattic/wp-codebox-core" import { ArtifactBundleWriter } from "./artifact-bundle-writer.js" import { browserArtifactFileManifest, type BrowserArtifactFiles } from "./browser-artifacts.js" +import { sanitizeBrowserResultValue } from "./browser-result-sanitization.js" export class BrowserArtifactSession { readonly writer: ArtifactBundleWriter @@ -32,11 +33,11 @@ export class BrowserArtifactSession { } async writeJson(key: keyof BrowserArtifactFiles, fileName: string, value: unknown): Promise { - await this.writer.writeJson(this.path(fileName), value, this.manifestWithoutContentType(key)) + await this.writer.writeJson(this.path(fileName), sanitizeBrowserResultValue(value), this.manifestWithoutContentType(key)) } async writeJsonLines(key: keyof BrowserArtifactFiles, fileName: string, records: unknown[]): Promise { - await this.writer.writeJsonLines(this.path(fileName), records, this.manifestWithoutContentType(key)) + await this.writer.writeJsonLines(this.path(fileName), sanitizeBrowserResultValue(records), this.manifestWithoutContentType(key)) } async writeGenerated(key: keyof BrowserArtifactFiles, fileName: string, write: (absolutePath: string) => Promise): Promise { diff --git a/packages/runtime-playground/src/browser-command-artifact-error.ts b/packages/runtime-playground/src/browser-command-artifact-error.ts index 340ee90da..1c4e5be4d 100644 --- a/packages/runtime-playground/src/browser-command-artifact-error.ts +++ b/packages/runtime-playground/src/browser-command-artifact-error.ts @@ -1,9 +1,14 @@ import type { BrowserArtifact } from "./browser-artifacts.js" +import { sanitizeBrowserArtifact, sanitizeBrowserResultValue } from "./browser-result-sanitization.js" export class BrowserCommandArtifactError extends Error { - constructor(message: string, readonly artifact: BrowserArtifact, readonly artifactRoot?: string) { - super(message) + readonly artifact: BrowserArtifact + + constructor(message: string, artifact: BrowserArtifact, readonly artifactRoot?: string) { + super(sanitizeBrowserResultValue(message, "message")) this.name = "BrowserCommandArtifactError" + Object.assign(artifact, sanitizeBrowserArtifact(artifact)) + this.artifact = artifact } } diff --git a/packages/runtime-playground/src/browser-environment-matrix.ts b/packages/runtime-playground/src/browser-environment-matrix.ts index 176cb08cc..134645b4e 100644 --- a/packages/runtime-playground/src/browser-environment-matrix.ts +++ b/packages/runtime-playground/src/browser-environment-matrix.ts @@ -13,6 +13,7 @@ import { } from "@automattic/wp-codebox-core" import type { Browser, BrowserContext, BrowserContextOptions, Page } from "playwright" import type { BrowserArtifactSummary } from "./browser-artifacts.js" +import type { BrowserPreviewRouteTracker } from "./browser-preview-routing.js" export const PLAYWRIGHT_BROWSER_ENVIRONMENT_CAPABILITIES = [ "browser.environment.viewport", @@ -75,6 +76,7 @@ export interface PlaywrightBrowserEnvironmentSession { browser: Browser requested: BrowserEnvironment resolved: ResolvedBrowserEnvironment + routeTracker?: BrowserPreviewRouteTracker runtime: PlaywrightBrowserEnvironmentRuntime } diff --git a/packages/runtime-playground/src/browser-metrics.ts b/packages/runtime-playground/src/browser-metrics.ts index fe3b7a0d1..e49fef2bf 100644 --- a/packages/runtime-playground/src/browser-metrics.ts +++ b/packages/runtime-playground/src/browser-metrics.ts @@ -527,7 +527,7 @@ export async function serializeBrowserFinishedRequest(request: Request, timestam if (!response) { return { type: "response", - url: request.url(), + url: redactBrowserNetworkUrl(request.url()), method: request.method(), resourceType: request.resourceType(), timestamp, @@ -545,7 +545,7 @@ export async function serializeBrowserResponse(response: Response, timestamp = n const responseTextPreview = await browserDocument5xxResponsePreview(response) return { type: "response", - url: response.url(), + url: redactBrowserNetworkUrl(response.url()), method: request.method(), resourceType: request.resourceType(), status: response.status(), @@ -596,17 +596,22 @@ function redactBrowserResponseText(body: string): string { } export function serializeBrowserRequestFailure(request: Request, timestamp = now()): BrowserProbeNetworkRecord { + const failure = request.failure() return { type: "requestfailed", - url: request.url(), + url: redactBrowserNetworkUrl(request.url()), method: request.method(), resourceType: request.resourceType(), timing: browserRequestTiming(request), - failure: request.failure(), + failure: failure ? { errorText: redactString(failure.errorText, { redactAllUrlQueryValues: true, redactUrlHash: true, redactQueryAssignments: true }) } : null, timestamp, } } +function redactBrowserNetworkUrl(url: string): string { + return redactString(url, { redactAllUrlQueryValues: true, redactUrlHash: true, redactQueryAssignments: true }) +} + function browserRequestTiming(request: Request): Record { return Object.fromEntries( Object.entries(request.timing()).filter((entry): entry is [string, number] => typeof entry[1] === "number" && Number.isFinite(entry[1])), diff --git a/packages/runtime-playground/src/browser-multi-actor-scenario-runner.ts b/packages/runtime-playground/src/browser-multi-actor-scenario-runner.ts index c37415fbd..aa9a19e68 100644 --- a/packages/runtime-playground/src/browser-multi-actor-scenario-runner.ts +++ b/packages/runtime-playground/src/browser-multi-actor-scenario-runner.ts @@ -6,7 +6,8 @@ import { BrowserCommandArtifactError } from "./browser-command-artifact-error.js import { attachBrowserCaptureListeners, launchChromiumBrowser, settleBrowserNetworkTasks } from "./browser-capture-session.js" import { executeBrowserInteractionStep } from "./browser-interactions.js" import { browserProbeReplayability } from "./browser-probe.js" -import { browserPreviewReadinessError, browserPreviewTopology, routeBrowserPreviewContextNetwork } from "./browser-preview-routing.js" +import { browserPreviewReadinessError, browserPreviewTopology, closeBrowserAndDrainPreviewRoutes, createBrowserPreviewRouteTracker, routeBrowserPreviewContextNetwork } from "./browser-preview-routing.js" +import { browserCommandResult } from "./browser-result-sanitization.js" import { installWordPressAdminAuthCookies } from "./browser-probe-support.js" import { bootstrapPhpCode } from "./php-bootstrap.js" import { assertPlaygroundResponseOk, type PlaygroundRunResponse } from "./playground-command-errors.js" @@ -33,6 +34,7 @@ export async function runBrowserMultiActorScenarioCommand(input: { const browserArgs = routeHost && !scenario.browserArgs.some((arg) => arg.startsWith("route-host=")) ? [...scenario.browserArgs, `route-host=${routeHost}`] : scenario.browserArgs const topology = browserPreviewTopology(browserArgs, runtimeSpec, server.serverUrl, server.previewProxyDiagnostics?.targetOrigin) const browser = await launchChromiumBrowser() + const routeTracker = createBrowserPreviewRouteTracker() const evidence: Record = {} let result: BrowserMultiActorScenarioResult | undefined let failure: Error | undefined @@ -56,7 +58,7 @@ export async function runBrowserMultiActorScenarioCommand(input: { const userId = await actorUserId(actor.name, session.user.userId, session.user, runtimeSpec, runPlaygroundCommand, server) const environmentRuntime = await createPlaywrightBrowserEnvironmentContext(browser, resolvedEnvironment, { contextOptions: topology.contextOptions() }) const context = environmentRuntime.context - await routeBrowserPreviewContextNetwork(context, topology.networkPolicy, topology.origins.localProxyOrigin) + await routeBrowserPreviewContextNetwork(context, topology.networkPolicy, topology.origins.localProxyOrigin, routeTracker) const page = environmentRuntime.page await context.tracing.start({ screenshots: true, snapshots: true }) await installWordPressAdminAuthCookies({ command: "wordpress.browser-scenario", cookieUrls: topology.authCookieUrls([topology.resolveUrl(scenario.url)]), page, runPlaygroundCommand, runtimeSpec, server, userId }) @@ -73,7 +75,8 @@ export async function runBrowserMultiActorScenarioCommand(input: { failure = error instanceof Error ? error : new Error(String(error)) if (error instanceof BrowserMultiActorScenarioError) result = error.result } finally { - await browser.close() + const routeErrors = await closeBrowserAndDrainPreviewRoutes(browser, routeTracker) + failure ??= routeErrors[0] } const replay = result?.replay ?? { schema: "wp-codebox/browser-multi-actor-replay/v1", seed: scenario.seed, scenario, schedule: [] } @@ -91,7 +94,7 @@ export async function runBrowserMultiActorScenarioCommand(input: { const traces = Object.values(evidence).map((actor) => actor.files.trace).filter((path): path is string => Boolean(path)) const artifact = { artifactType: "scenario" as const, requestedUrl: target, url: target, preview: topology.preview, ...topology.origins, files: { summary: "files/browser/multi-actor-scenario-summary.json", steps: "files/browser/multi-actor-events.json", network: "files/browser/multi-actor-network.json", requestCoverage: "files/browser/multi-actor-request-coverage.json", waterfall: "files/browser/multi-actor-waterfall.json", ...(traces.length > 0 ? { traces } : {}) }, summary: { actions: scenario.actions.length, steps: scenario.actions.length, consoleMessages: Object.values(evidence).reduce((total, actor) => total + actor.console.length, 0), errors: Object.values(evidence).reduce((total, actor) => total + actor.errors.length, 0), finalUrl: target, htmlSnapshot: false, networkEvents: network.length, replayability: browserProbeReplayability(captures), screenshot: captures.has("screenshot"), viewport: null, environment: Object.values(evidence)[0]?.environment, multiActor: { seed: scenario.seed, finalState: result?.finalState ?? "failed", actors: Object.keys(evidence), replay: "files/browser/multi-actor-replay.json" } } } satisfies BrowserArtifact if (failure) throw new BrowserCommandArtifactError(`wordpress.browser-scenario failed: ${failure.message}`, artifact) - return { artifact, output: `${JSON.stringify({ command: "wordpress.browser-scenario", files: artifact.files, summary: artifact.summary, scenario: summary }, null, 2)}\n` } + return browserCommandResult(artifact, { command: "wordpress.browser-scenario", files: artifact.files, summary: artifact.summary, scenario: summary }) } export async function navigateBrowserMultiActorPages( diff --git a/packages/runtime-playground/src/browser-preview-routing.ts b/packages/runtime-playground/src/browser-preview-routing.ts index acc301a3f..b89d8609d 100644 --- a/packages/runtime-playground/src/browser-preview-routing.ts +++ b/packages/runtime-playground/src/browser-preview-routing.ts @@ -64,10 +64,11 @@ export interface BrowserPreviewOrigins { export interface BrowserPreviewRouteTracker { pending: Set> errors: unknown[] + registrations: number } export function createBrowserPreviewRouteTracker(): BrowserPreviewRouteTracker { - return { pending: new Set(), errors: [] } + return { pending: new Set(), errors: [], registrations: 0 } } export function browserPreviewRouting(args: string[], runtimeSpec: RuntimeCreateSpec | undefined, localPreviewOrigin: string): BrowserProbePreviewRouting { @@ -325,19 +326,23 @@ export async function routeBrowserPreviewContextNetwork(context: import("playwri export async function drainBrowserPreviewRouteTracker(tracker: BrowserPreviewRouteTracker, timeoutMs = BROWSER_PREVIEW_ROUTE_DRAIN_TIMEOUT_MS): Promise { const deadline = Date.now() + timeoutMs - while (tracker.pending.size > 0) { + let observedRegistrations = -1 + while (tracker.pending.size > 0 || tracker.registrations !== observedRegistrations) { + observedRegistrations = tracker.registrations const remainingMs = deadline - Date.now() if (remainingMs <= 0) { throw new Error(`wordpress.browser-probe route-host timed out waiting for ${tracker.pending.size} routed request(s) to finish`) } + const pending = [...tracker.pending] const result = await Promise.race([ - Promise.allSettled([...tracker.pending]).then(() => "drained" as const), + Promise.allSettled(pending).then(() => "drained" as const), wait(remainingMs).then(() => "timeout" as const), ]) if (result === "timeout") { throw new Error(`wordpress.browser-probe route-host timed out waiting for ${tracker.pending.size} routed request(s) to finish`) } + await wait(0) } if (tracker.errors.length > 0) { @@ -345,6 +350,34 @@ export async function drainBrowserPreviewRouteTracker(tracker: BrowserPreviewRou } } +export async function closeBrowserAndDrainPreviewRoutes(browser: Pick, tracker: BrowserPreviewRouteTracker, closeTimeoutMs = 1_000): Promise { + const errors: Error[] = [] + let closeTimer: ReturnType | undefined + try { + const closeResult = await Promise.race([ + browser.close().then(() => ({ status: "closed" as const }), (error: unknown) => ({ status: "failed" as const, error })), + new Promise<{ status: "timeout" }>((resolve) => { + closeTimer = setTimeout(() => resolve({ status: "timeout" }), closeTimeoutMs) + }), + ]) + if (closeResult.status === "failed") { + errors.push(browserPreviewLifecycleError("browser-close", closeResult.error)) + } else if (closeResult.status === "timeout") { + errors.push(browserPreviewLifecycleError("browser-close-timeout", new Error(`Browser close exceeded ${closeTimeoutMs}ms`))) + } + } finally { + if (closeTimer) { + clearTimeout(closeTimer) + } + try { + await drainBrowserPreviewRouteTracker(tracker) + } catch (error) { + errors.push(browserPreviewLifecycleError("route-drain", error)) + } + } + return errors +} + function browserPreviewMode(args: string[], publicOrigin: string | undefined): BrowserProbePreviewMode { const raw = argValue(args, "preview-mode")?.trim() || (publicOrigin ? "public" : "local") if (raw === "local" || raw === "public" || raw === "secure") { @@ -361,53 +394,77 @@ async function routeBrowserPreviewNetwork(routePattern: (url: string, handler: ( const origin = new URL(previewOrigin) await routePattern("**/*", async (route) => { - const request = route.request() - let requestUrl: URL - try { - requestUrl = new URL(request.url()) - } catch { - await route.continue() - return - } - - const host = normalizeBrowserPreviewHost(requestUrl.hostname) - const stat = browserPreviewNetworkPolicyHostStat(policy, host) - stat.requests += 1 - stat.external = !policy.firstPartyHosts.has(host) - - if (policy.blockHosts.has(host)) { - stat.blocked += 1 - await route.abort("blockedbyclient") - return + let operation = "inspect-request" + const task = handleBrowserPreviewRoute(route, policy, origin, (nextOperation) => { + operation = nextOperation + }) + if (tracker) { + tracker.registrations += 1 + tracker.pending.add(task) } - - if (policy.routeHosts.has(host)) { - stat.routed += 1 - if (policy.preserveRoutedOrigin) { - await route.continue() - return + try { + await task + } catch (error) { + if (!isBrowserPreviewRouteClosedError(error)) { + tracker?.errors.push(browserPreviewRouteCallbackError(route, operation, error)) } - const task = fulfillBrowserPreviewRoutedHost(route, requestUrl, policy, origin) - tracker?.pending.add(task) try { - await task - } catch (error) { - tracker?.errors.push(sanitizeBrowserPreviewRouteError(error)) - await route.abort("failed").catch(() => undefined) - } finally { - tracker?.pending.delete(task) + await route.abort("failed") + } catch (abortError) { + if (!isBrowserPreviewRouteClosedError(abortError)) { + tracker?.errors.push(browserPreviewRouteCallbackError(route, "abort-after-error", abortError)) + } } - return + } finally { + tracker?.pending.delete(task) } + }) +} - if (policy.preserveRoutedOrigin || (policy.mode === "block" && stat.external && !policy.allowHosts.has(host)) || (request.resourceType() === "document" && stat.external)) { - stat.blocked += 1 - await route.abort("blockedbyclient") +async function handleBrowserPreviewRoute(route: Route, policy: BrowserPreviewNetworkPolicy, origin: URL, setOperation: (operation: string) => void): Promise { + const request = route.request() + let requestUrl: URL + try { + requestUrl = new URL(request.url()) + } catch { + setOperation("continue-invalid-url") + await route.continue() + return + } + + const host = normalizeBrowserPreviewHost(requestUrl.hostname) + const stat = browserPreviewNetworkPolicyHostStat(policy, host) + stat.requests += 1 + stat.external = !policy.firstPartyHosts.has(host) + + if (policy.blockHosts.has(host)) { + stat.blocked += 1 + setOperation("abort-policy-block") + await route.abort("blockedbyclient") + return + } + + if (policy.routeHosts.has(host)) { + stat.routed += 1 + if (policy.preserveRoutedOrigin) { + setOperation("continue-preserved-routed-origin") + await route.continue() return } + setOperation("fulfill-routed-host") + await fulfillBrowserPreviewRoutedHost(route, requestUrl, policy, origin) + return + } - await route.continue() - }) + if (policy.preserveRoutedOrigin || (policy.mode === "block" && stat.external && !policy.allowHosts.has(host)) || (request.resourceType() === "document" && stat.external)) { + stat.blocked += 1 + setOperation("abort-policy-block") + await route.abort("blockedbyclient") + return + } + + setOperation("continue-unrouted") + await route.continue() } async function fulfillBrowserPreviewRoutedHost(route: Route, requestUrl: URL, policy: BrowserPreviewNetworkPolicy, localOrigin: URL): Promise { @@ -494,7 +551,9 @@ async function fetchBrowserPreviewRoutedHost(route: Route, requestUrl: URL, poli let response: Awaited> | undefined const resourceType = route.request().resourceType() - const maxAttempts = resourceType === "document" ? BROWSER_PREVIEW_ROUTE_DOCUMENT_FETCH_ATTEMPTS : BROWSER_PREVIEW_ROUTE_SUBRESOURCE_FETCH_ATTEMPTS + const method = route.request().method().toUpperCase() + const methodCanRetry = browserPreviewRouteMethodCanRetry(method) + const maxAttempts = methodCanRetry ? (resourceType === "document" ? BROWSER_PREVIEW_ROUTE_DOCUMENT_FETCH_ATTEMPTS : BROWSER_PREVIEW_ROUTE_SUBRESOURCE_FETCH_ATTEMPTS) : 1 for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { try { response = await route.fetch({ @@ -521,7 +580,7 @@ async function fetchBrowserPreviewRoutedHost(route: Route, requestUrl: URL, poli } if (resourceType !== "document" || !retryable) { - await route.abort("failed").catch(() => undefined) + await abortBrowserPreviewRoute(route, "abort-recoverable-fetch") return undefined } throw browserPreviewRouteFetchExhaustedError(route, currentUrl, attempt, error) @@ -530,6 +589,9 @@ async function fetchBrowserPreviewRoutedHost(route: Route, requestUrl: URL, poli if (!response) { return undefined } + if (!methodCanRetry) { + return response + } const location = response.headers().location if (!location || response.status() < 300 || response.status() >= 400) { @@ -547,7 +609,7 @@ async function fetchBrowserPreviewRoutedHost(route: Route, requestUrl: URL, poli stat.external = true stat.blocked += 1 policy.routedRedirectEscapes.push({ rawOrigin: redirectedUrl.origin, effectiveOrigin: origin.origin, reason: "redirect-host-not-routed-to-preview" }) - await route.abort("blockedbyclient") + await abortBrowserPreviewRoute(route, "abort-redirect-escape", "blockedbyclient") return undefined } @@ -555,7 +617,7 @@ async function fetchBrowserPreviewRoutedHost(route: Route, requestUrl: URL, poli } if (route.request().resourceType() !== "document") { - await route.abort("failed").catch(() => undefined) + await abortBrowserPreviewRoute(route, "abort-redirect-limit") return undefined } @@ -578,6 +640,56 @@ export function isBrowserPreviewRouteFetchTransientTransportError(error: unknown return error instanceof Error && /\b(?:ECONNRESET|ECONNREFUSED|EPIPE|ETIMEDOUT|UND_ERR_SOCKET|socket (?:hang up|closed|ended)|connection (?:reset|refused|closed)|other side closed)\b/i.test(error.message) } +export function isBrowserPreviewRouteClosedError(error: unknown): boolean { + return error instanceof Error && /(?:Request context disposed|Target (?:page, context or browser|page|context|browser) has been closed|Browser has been closed|context closed|page closed)/i.test(error.message) +} + +function browserPreviewRouteMethodCanRetry(method: string): boolean { + return method === "GET" || method === "HEAD" || method === "OPTIONS" +} + +function browserPreviewRouteCallbackError(route: Route, operation: string, error: unknown): Error { + const request = browserPreviewRouteRequestSummary(route) + const cause = sanitizeBrowserPreviewRouteError(error).message.replace(/[\r\n]+/g, " ") + const diagnostic = new Error(`wordpress.browser-probe route callback failed: operation=${operation} method=${request.method} resourceType=${request.resourceType} url=${request.url} cause=${cause}`) + diagnostic.name = "BrowserPreviewRouteCallbackError" + return diagnostic +} + +function browserPreviewRouteRequestSummary(route: Route): { method: string; resourceType: string; url: string } { + try { + const request = route.request() + return { + method: request.method(), + resourceType: request.resourceType(), + url: redactString(request.url(), { redactAllUrlQueryValues: true, redactUrlHash: true, redactQueryAssignments: true }), + } + } catch { + return { method: "unknown", resourceType: "unknown", url: "[unavailable]" } + } +} + +function browserPreviewLifecycleError(operation: string, error: unknown): Error { + const cause = sanitizeBrowserPreviewRouteError(error).message.replace(/[\r\n]+/g, " ") + const diagnostic = new Error(`wordpress.browser-probe route lifecycle failed: operation=${operation} cause=${cause}`) + diagnostic.name = "BrowserPreviewRouteLifecycleError" + return diagnostic +} + +async function abortBrowserPreviewRoute(route: Route, operation: string, errorCode: Parameters[0] = "failed"): Promise { + try { + await route.abort(errorCode) + } catch (error) { + if (isBrowserPreviewRouteClosedError(error)) { + return + } + const cause = sanitizeBrowserPreviewRouteError(error).message.replace(/[\r\n]+/g, " ") + const diagnostic = new Error(`wordpress.browser-probe route operation failed: operation=${operation} cause=${cause}`) + diagnostic.name = "BrowserPreviewRouteOperationError" + throw diagnostic + } +} + function browserPreviewRouteFetchExhaustedError(route: Route, requestUrl: URL, attempts: number, error: unknown): Error { const method = route.request().method() const resourceType = route.request().resourceType() diff --git a/packages/runtime-playground/src/browser-probe-runner.ts b/packages/runtime-playground/src/browser-probe-runner.ts index 1b79afa37..880dfab6d 100644 --- a/packages/runtime-playground/src/browser-probe-runner.ts +++ b/packages/runtime-playground/src/browser-probe-runner.ts @@ -6,7 +6,7 @@ import { attachBrowserCaptureListeners, chromiumBrowserMetadata, launchChromiumB import { browserCommandLivenessPolicy, isBrowserCommandLivenessError, withBrowserCommandLiveness } from "./browser-liveness.js" import { browserProbeLifecycleArtifact, browserProbeLifecycleInitScript, collectBrowserProbeLifecycle } from "./browser-lifecycle.js" import { browserProbeBenchMetrics, serializeBrowserError } from "./browser-metrics.js" -import { browserPreviewNetworkPolicyIsActive, browserPreviewNetworkPolicySummary, browserPreviewNeedsContextRouting, browserPreviewReadinessError, browserPreviewSecureContextError, browserPreviewTopology, createBrowserPreviewRouteTracker, drainBrowserPreviewRouteTracker, routeBrowserPreviewContextNetwork, routeBrowserPreviewPageNetwork } from "./browser-preview-routing.js" +import { browserPreviewNetworkPolicyIsActive, browserPreviewNetworkPolicySummary, browserPreviewNeedsContextRouting, browserPreviewReadinessError, browserPreviewSecureContextError, browserPreviewTopology, closeBrowserAndDrainPreviewRoutes, createBrowserPreviewRouteTracker, drainBrowserPreviewRouteTracker, routeBrowserPreviewContextNetwork, routeBrowserPreviewPageNetwork } from "./browser-preview-routing.js" import { BROWSER_PROBE_PERFORMANCE_INIT_SCRIPT, BROWSER_PROBE_STATE_INIT_SCRIPT, browserProbeAssertionsFromArgs, browserProbeCheckpoint, browserProbeMemoryArtifact, browserProbePendingCheckpoints, browserProbePerformanceArtifact, browserProbeViewport, executeBrowserProbeAssertions, navigateBrowserProbe } from "./browser-probe.js" import { argValue, commaListArg, durationArg, strictBooleanArg, viewportArg } from "./commands.js" import type { PlaygroundRunResponse } from "./playground-command-errors.js" @@ -248,7 +248,7 @@ export async function runSingleBrowserProbeCommand({ const prePageScriptMetadata = prePageScript ? browserProbeScriptMetadata(prePageScript) : undefined const topology = browserPreviewTopology(args, runtimeSpec, server.serverUrl, server.previewProxyDiagnostics?.targetOrigin) const { preview, networkPolicy } = topology - const routeTracker = createBrowserPreviewRouteTracker() + const routeTracker = session?.routeTracker ?? createBrowserPreviewRouteTracker() const targetUrl = topology.resolveUrl(runPlan.url) const artifactSession = new BrowserArtifactSession(artifactRoot, browserFilesDirectory, { source: command, operation: "browser-probe" }) @@ -296,7 +296,6 @@ export async function runSingleBrowserProbeCommand({ pendingError = pendingError ?? new Error("Browser command aborted during runtime cleanup") void page?.close().catch(() => undefined) void context?.close().catch(() => undefined) - void browser.close().catch(() => undefined) } abortSignal?.addEventListener("abort", abortHandler, { once: true }) @@ -455,35 +454,38 @@ export async function runSingleBrowserProbeCommand({ errors.push(serializeBrowserError("probe-error", error)) } finally { if (abortSignal?.aborted) { - await closeBrowserBestEffort(browser) - abortSignal.removeEventListener("abort", abortHandler) - throw pendingError ?? new Error("Browser command aborted during runtime cleanup") - } - try { - await drainBrowserPreviewRouteTracker(routeTracker) - } catch (error) { - const routeError = redactError(error, { redactAllUrlQueryValues: true, redactUrlHash: true, redactQueryAssignments: true }) - if (!pendingError && runPlan.routeHostDrain === "required") { - pendingError = routeError - progress.fail("probe-error", routeError) - } - errors.push(serializeBrowserError("probe-error", error)) + pendingError ??= new Error("Browser command aborted during runtime cleanup") + progress.fail("probe-error", pendingError) } if (page) { - finalUrl = page.url() + try { + finalUrl = page.url() + } catch (error) { + errors.push(serializeBrowserError("probe-error", error)) + } windowLocationOrigin = windowLocationOrigin ?? await page.evaluate(() => window.location.origin).catch(() => undefined) if (captureSelection.metrics) { - checkpoints.push(await browserProbeCheckpoint(page, "final")) - if (capture.has("memory")) { - memoryArtifact = browserProbeMemoryArtifact(checkpoints) - } - if (capture.has("performance")) { - performanceArtifact = browserProbePerformanceArtifact(checkpoints, { consoleMessages, errors, network, startedAt }) + try { + checkpoints.push(await browserProbeCheckpoint(page, "final")) + if (capture.has("memory")) { + memoryArtifact = browserProbeMemoryArtifact(checkpoints) + } + if (capture.has("performance")) { + performanceArtifact = browserProbePerformanceArtifact(checkpoints, { consoleMessages, errors, network, startedAt }) + } + } catch (error) { + pendingError ??= redactError(error, { redactAllUrlQueryValues: true, redactUrlHash: true, redactQueryAssignments: true }) + errors.push(serializeBrowserError("probe-error", error)) } } - const lifecycle = lifecycleSelectors.length > 0 ? await collectBrowserProbeLifecycle(page) : undefined - if (lifecycle) { - lifecycleArtifact = browserProbeLifecycleArtifact(lifecycle) + try { + const lifecycle = lifecycleSelectors.length > 0 ? await collectBrowserProbeLifecycle(page) : undefined + if (lifecycle) { + lifecycleArtifact = browserProbeLifecycleArtifact(lifecycle) + } + } catch (error) { + pendingError ??= redactError(error, { redactAllUrlQueryValues: true, redactUrlHash: true, redactQueryAssignments: true }) + errors.push(serializeBrowserError("probe-error", error)) } if (capture.has("html")) { @@ -506,10 +508,19 @@ export async function runSingleBrowserProbeCommand({ } } } - await settleBrowserNetworkTasks(networkTasks, livenessPolicy.networkSettleTimeoutMs) - if (!session) { - await environmentRuntime?.close().catch(() => undefined) - await browser.close() + try { + await settleBrowserNetworkTasks(networkTasks, livenessPolicy.networkSettleTimeoutMs) + } catch (error) { + errors.push(serializeBrowserError("probe-error", error)) + } + const cleanupBrowser = session ? { close: async () => {} } : { close: async () => { await environmentRuntime?.close(); await browser.close() } } + for (const routeError of await closeBrowserAndDrainPreviewRoutes(cleanupBrowser, routeTracker)) { + const browserCloseFailed = routeError.message.includes("operation=browser-close") + if (!pendingError && (browserCloseFailed || runPlan.routeHostDrain === "required")) { + pendingError = routeError + progress.fail("probe-error", routeError) + } + errors.push(serializeBrowserError("probe-error", routeError)) } if (captureSelection.console) { await artifactSession.writeJsonLines("console", "console.jsonl", consoleMessages) @@ -637,16 +648,6 @@ export async function runSingleBrowserProbeCommand({ } } -async function closeBrowserBestEffort(browser: import("playwright").Browser): Promise { - await Promise.race([ - browser.close().catch(() => undefined), - new Promise((resolve) => { - const timeout = setTimeout(resolve, 1_000) - timeout.unref() - }), - ]) -} - export type BoundedBrowserDiagnosticResult = { ok: true; value: T } | { ok: false; error: Error } /** diff --git a/packages/runtime-playground/src/browser-probe-session-result-builder.ts b/packages/runtime-playground/src/browser-probe-session-result-builder.ts index 72bcd9961..ff7f4ddc9 100644 --- a/packages/runtime-playground/src/browser-probe-session-result-builder.ts +++ b/packages/runtime-playground/src/browser-probe-session-result-builder.ts @@ -3,6 +3,7 @@ import type { BrowserPreviewTopology } from "./browser-preview-routing.js" import { addBrowserProbeNetworkCount, browserProbeArtifactRefs, browserProbeWebSocketSummary, createBrowserProbeProgressTracker, now, requestHost, safeBrowserProbeUrl, sortBrowserProbeNetworkCounts } from "./browser-probe-support.js" import { browserProbeBenchMetrics } from "./browser-metrics.js" import { browserProbeAssertionsFromArgs, browserProbeAssertionsNeedMetrics, browserProbeAssertionsNeedNetwork, browserProbeReplayability } from "./browser-probe.js" +import { sanitizeBrowserResultValue } from "./browser-result-sanitization.js" import type { BrowserProbeArtifact, BrowserProbeArtifactRef, BrowserProbeAuthSummary, BrowserProbeCapabilityDiagnostics, BrowserProbeCheckpointRecord, BrowserProbeContextDetails, BrowserProbeErrorRecord, BrowserProbeLifecycleArtifact, BrowserProbeMeasuredMetric, BrowserProbeMemoryArtifact, BrowserProbeNetworkCountSummary, BrowserProbeNetworkPolicySummary, BrowserProbeNetworkRecord, BrowserProbeNetworkReviewSummary, BrowserProbePerformanceArtifact, BrowserProbePreviewRouting, BrowserProbeReviewSummary, BrowserProbeScriptMetadata, BrowserProbeViewport, BrowserProbeWebSocketRecord, BrowserProbeWebSocketReviewSummary, BrowserRedirectDiagnosticsSummary, BrowserStepAssertion, BrowserWordPressDiagnosticsSummary } from "./browser-artifacts.js" export interface BrowserProbeCaptureSelection { @@ -86,7 +87,8 @@ export interface BrowserProbeSessionResult { } export class BrowserProbeSessionResultBuilder { - compose(input: BrowserProbeSessionResultInput): BrowserProbeSessionResult { + compose(rawInput: BrowserProbeSessionResultInput): BrowserProbeSessionResult { + const input = sanitizeBrowserResultValue(rawInput) const assertionSummary = browserProbeAssertionSummary(input.assertions) const finishedAt = now() const files = browserProbeArtifactFileMap(input) diff --git a/packages/runtime-playground/src/browser-probe-support.ts b/packages/runtime-playground/src/browser-probe-support.ts index 9455241a0..5e592efbf 100644 --- a/packages/runtime-playground/src/browser-probe-support.ts +++ b/packages/runtime-playground/src/browser-probe-support.ts @@ -287,7 +287,7 @@ export function safeBrowserProbeUrl(value: string | undefined): string | null { if (/^data:/i.test(value)) { return "data:[redacted]" } - return value + return browserRedirectSafeUrl(value) } export interface BrowserRedirectDiagnosticsArtifact { @@ -400,7 +400,7 @@ export function browserRedirectSafeUrl(value: string): string { } const parsed = parseBrowserRedirectUrl(value) if (!parsed) { - return value + return redactString(value, { redactAllUrlQueryValues: true, redactUrlHash: true, redactQueryAssignments: true }) } const search = parsed.queryKeys.length > 0 ? `?${parsed.queryKeys.map((key) => `${encodeURIComponent(key)}=[redacted]`).join("&")}` diff --git a/packages/runtime-playground/src/browser-result-sanitization.ts b/packages/runtime-playground/src/browser-result-sanitization.ts new file mode 100644 index 000000000..70a86a372 --- /dev/null +++ b/packages/runtime-playground/src/browser-result-sanitization.ts @@ -0,0 +1,53 @@ +import { redactString } from "@automattic/wp-codebox-core" +import type { BrowserArtifact } from "./browser-artifacts.js" + +const URL_KEY_PATTERN = /(?:url|uri|href|origin|location|filename|src|action)$/i +const DIAGNOSTIC_KEY_PATTERN = /(?:error|message|stack|reason)$/i +const URL_VALUE_PATTERN = /^(?:data|https?|wss?):/i + +export function sanitizeBrowserResultValue(value: T, key = ""): T { + if (Array.isArray(value)) { + return value.map((entry) => sanitizeBrowserResultValue(entry)) as T + } + if (value && typeof value === "object") { + const prototype = Object.getPrototypeOf(value) + if (prototype !== Object.prototype && prototype !== null) { + return value + } + return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [entryKey, sanitizeBrowserResultValue(entryValue, entryKey)])) as T + } + if (typeof value !== "string") { + return value + } + if (URL_KEY_PATTERN.test(key) || URL_VALUE_PATTERN.test(value)) { + return sanitizeBrowserResultUrl(value) as T + } + if (DIAGNOSTIC_KEY_PATTERN.test(key)) { + return redactString(value, { redactAllUrlQueryValues: true, redactUrlHash: true, redactQueryAssignments: true }) as T + } + return sanitizePersistedBrowserText(value) as T +} + +function sanitizePersistedBrowserText(value: string): string { + return value + .replace(/(?:https?|wss?):\/\/[^\s"'<>]+|\/[A-Za-z0-9._~!$&'()*+,;=:@%/-]*\?[^\s"'<>]+/gi, (url) => sanitizeBrowserResultUrl(url)) + .replace(/\b((?:access[_-]?token|api[_-]?key|authorization|cookie|nonce|password|secret|session[_-]?token)\s*[=:]\s*)[^\s,;]+/gi, "$1[redacted]") +} + +export function sanitizeBrowserArtifact(artifact: BrowserArtifact): BrowserArtifact { + return sanitizeBrowserResultValue(artifact) +} + +export function browserCommandResult(artifact: BrowserArtifact, output: unknown): { artifact: BrowserArtifact; output: string } { + return { + artifact: sanitizeBrowserArtifact(artifact), + output: `${JSON.stringify(sanitizeBrowserResultValue(output), null, 2)}\n`, + } +} + +function sanitizeBrowserResultUrl(value: string): string { + if (/^data:/i.test(value)) { + return "data:[redacted]" + } + return redactString(value, { redactAllUrlQueryValues: true, redactUrlHash: true, redactQueryAssignments: true }) +} diff --git a/packages/runtime-playground/src/editor-command-runners.ts b/packages/runtime-playground/src/editor-command-runners.ts index 58ed21b62..334e0c58c 100644 --- a/packages/runtime-playground/src/editor-command-runners.ts +++ b/packages/runtime-playground/src/editor-command-runners.ts @@ -6,7 +6,8 @@ import { BrowserCommandArtifactError } from "./browser-command-artifact-error.js import type { BrowserArtifact, BrowserArtifactFiles, BrowserArtifactSummary, BrowserEditorCanvasProbeDiagnostic, BrowserEditorCanvasProbeSummary, BrowserEditorCanvasSelectorGroupSummary, BrowserEditorCanvasSelectorSummary, BrowserEditorReadinessSummary, BrowserEditorSaveSummary, BrowserEditorValidateBlocksSummary, BrowserEditorValiditySummary, BrowserProbeAuthSummary, BrowserProbeErrorRecord, BrowserProbeViewport, BrowserStepRecord } from "./browser-artifacts.js" import { attachBrowserCaptureListeners, launchChromiumBrowser } from "./browser-capture-session.js" import { browserStepRecord } from "./browser-interactions.js" -import { browserPreviewNetworkPolicyIsActive, browserPreviewNetworkPolicySummary, browserPreviewNeedsContextRouting, browserPreviewReadinessError, browserPreviewSecureContextError, browserPreviewTopology, routeBrowserPreviewContextNetwork } from "./browser-preview-routing.js" +import { browserPreviewNetworkPolicyIsActive, browserPreviewNetworkPolicySummary, browserPreviewNeedsContextRouting, browserPreviewReadinessError, browserPreviewSecureContextError, browserPreviewTopology, closeBrowserAndDrainPreviewRoutes, createBrowserPreviewRouteTracker, routeBrowserPreviewContextNetwork } from "./browser-preview-routing.js" +import { browserCommandResult } from "./browser-result-sanitization.js" import { browserProbeReplayability, browserProbeViewport } from "./browser-probe.js" import { argValue, commaListArg, durationArg, jsonArrayArg } from "./commands.js" import { DEFAULT_EDITOR_WAIT_SELECTOR, editorActionStepsFromArgs, editorOpenTargetFromArgs, editorValidateContentFromArgs, editorValidateProviderFromArgs, resolveEditorOpenTarget, type EditorActionStep, type EditorBlockSpec, type EditorBlockTarget } from "./editor-actions.js" @@ -66,6 +67,7 @@ export async function runEditorCanvasProbeCommand({ const startedAt = now() const startedAtMs = Date.now() const browser = await launchChromiumBrowser() + const routeTracker = createBrowserPreviewRouteTracker() const errors: BrowserProbeErrorRecord[] = [] let artifact: BrowserArtifact | undefined let finalUrl = targetUrl @@ -82,7 +84,7 @@ export async function runEditorCanvasProbeCommand({ const context = browserPreviewNeedsContextRouting(networkPolicy) ? await browser.newContext(topology.contextOptions()) : null if (context) { - await routeBrowserPreviewContextNetwork(context, networkPolicy, topology.origins.localProxyOrigin) + await routeBrowserPreviewContextNetwork(context, networkPolicy, topology.origins.localProxyOrigin, routeTracker) } const page = context ? await context.newPage() : await browser.newPage() viewport = await browserProbeViewport(page) @@ -231,23 +233,47 @@ export async function runEditorCanvasProbeCommand({ }) } } finally { - await browser.close() + for (const routeError of await closeBrowserAndDrainPreviewRoutes(browser, routeTracker)) { + errors.push(serializeBrowserError("probe-error", routeError)) + pendingError ??= routeError + } + if (artifact) { + artifact.summary.errors = errors.length + await artifactSession.writeJson("summary", "editor-canvas-summary.json", { + schema: "wp-codebox/editor-canvas-probe/v1", + requestedUrl: targetUrl, + preview, + ...previewOrigins, + finalUrl, + ...(windowLocationOrigin ? { windowLocationOrigin } : {}), + startedAt, + finishedAt: now(), + timeoutMs, + files: artifact.files, + hashes: { + ...(screenshotSha256 ? { screenshot: { algorithm: "sha256", value: screenshotSha256 } } : {}), + }, + viewport, + errors, + summary: artifact.summary.editorCanvas, + }) + } } + if (!artifact) { + throw pendingError ?? new Error("wordpress.editor-canvas-probe did not produce an artifact") + } if (pendingError) { throw new BrowserCommandArtifactError(pendingError.message, artifact) } - return { - artifact, - output: `${JSON.stringify({ + return browserCommandResult(artifact, { command: "wordpress.editor-canvas-probe", requestedUrl: targetUrl, finalUrl: artifact.summary.finalUrl, files: artifact.files, summary: artifact.summary.editorCanvas, - }, null, 2)}\n`, - } + }) } interface EditorCanvasSelectorGroupInput { @@ -527,6 +553,7 @@ export async function runEditorOpenCommand({ const waitTimeoutMs = durationArg(args, "wait-timeout", BROWSER_STEP_DEFAULT_TIMEOUT_MS) const topology = browserPreviewTopology(args, runtimeSpec, server.serverUrl, server.previewProxyDiagnostics?.targetOrigin) const { preview, networkPolicy } = topology + const routeTracker = createBrowserPreviewRouteTracker() const targetUrl = topology.resolveUrl(target.url) const artifactPathPrefix = editorOpenArtifactPathPrefixFromArgs(args) const artifactSession = new BrowserArtifactSession(artifactRoot, artifactPathPrefix, { source: "wordpress.editor-open", operation: "editor-open" }) @@ -556,7 +583,7 @@ export async function runEditorOpenCommand({ } const context = browserPreviewNeedsContextRouting(networkPolicy) ? await browser.newContext(topology.contextOptions()) : null if (context) { - await routeBrowserPreviewContextNetwork(context, networkPolicy, topology.origins.localProxyOrigin) + await routeBrowserPreviewContextNetwork(context, networkPolicy, topology.origins.localProxyOrigin, routeTracker) } const page = context ? await context.newPage() : await browser.newPage() authSummary = await installWordPressAdminAuthCookies({ command: "wordpress.editor-open", cookieUrls: topology.authCookieUrls([targetUrl]), page, runPlaygroundCommand, runtimeSpec, server, userId: 1 }) @@ -634,7 +661,10 @@ export async function runEditorOpenCommand({ pendingError = error instanceof Error ? error : new Error(String(error)) errors.push(serializeBrowserError("probe-error", error)) } finally { - await browser.close() + for (const routeError of await closeBrowserAndDrainPreviewRoutes(browser, routeTracker)) { + errors.push(serializeBrowserError("probe-error", routeError)) + pendingError ??= routeError + } if (capture.has("steps")) { await artifactSession.writeJsonLines("steps", "editor-steps.jsonl", stepRecords) } @@ -702,9 +732,7 @@ export async function runEditorOpenCommand({ throw editorOpenArtifactError(stepRecords.length, pendingError, artifact) } - return { - artifact, - output: `${JSON.stringify({ + return browserCommandResult(artifact, { command: "wordpress.editor-open", target, requestedUrl: targetUrl, @@ -714,8 +742,7 @@ export async function runEditorOpenCommand({ files: artifact.files, summary: artifact.summary, steps: stepRecords, - }, null, 2)}\n`, - } + }) } export function editorOpenArtifactPathPrefixFromArgs(args: string[]): string { @@ -817,6 +844,7 @@ export async function runEditorActionsCommand({ const totalTimeoutMs = durationArg(args, "timeout", BROWSER_SCRIPT_DEFAULT_TIMEOUT_MS) const topology = browserPreviewTopology(args, runtimeSpec, server.serverUrl, server.previewProxyDiagnostics?.targetOrigin) const { preview, networkPolicy } = topology + const routeTracker = createBrowserPreviewRouteTracker() const targetUrl = topology.resolveUrl(target.url) const artifactSession = new BrowserArtifactSession(artifactRoot, "files/browser", { source: "wordpress.editor-actions", operation: "editor-actions" }) @@ -846,7 +874,7 @@ export async function runEditorActionsCommand({ } const context = browserPreviewNeedsContextRouting(networkPolicy) ? await browser.newContext(topology.contextOptions()) : null if (context) { - await routeBrowserPreviewContextNetwork(context, networkPolicy, topology.origins.localProxyOrigin) + await routeBrowserPreviewContextNetwork(context, networkPolicy, topology.origins.localProxyOrigin, routeTracker) } const page = context ? await context.newPage() : await browser.newPage() authSummary = await installWordPressAdminAuthCookies({ command: "wordpress.editor-actions", cookieUrls: topology.authCookieUrls([targetUrl]), page, runPlaygroundCommand, runtimeSpec, server, userId: 1 }) @@ -953,7 +981,10 @@ export async function runEditorActionsCommand({ pendingError = error instanceof Error ? error : new Error(String(error)) errors.push(serializeBrowserError("probe-error", error)) } finally { - await browser.close() + for (const routeError of await closeBrowserAndDrainPreviewRoutes(browser, routeTracker)) { + errors.push(serializeBrowserError("probe-error", routeError)) + pendingError ??= routeError + } if (capture.has("steps")) { await artifactSession.writeJsonLines("steps", "editor-action-steps.jsonl", stepRecords) } @@ -1035,9 +1066,7 @@ export async function runEditorActionsCommand({ throw new BrowserCommandArtifactError(`wordpress.editor-actions failed after ${stepRecords.length} step(s): ${pendingError.message}`, artifact) } - return { - artifact, - output: `${JSON.stringify({ + return browserCommandResult(artifact, { command: "wordpress.editor-actions", target, actions: actionSteps.length, @@ -1048,8 +1077,7 @@ export async function runEditorActionsCommand({ files: artifact.files, summary: artifact.summary, steps: stepRecords, - }, null, 2)}\n`, - } + }) } interface EditorActionStepResult { @@ -1931,6 +1959,7 @@ export async function runEditorValidateBlocksCommand({ const waitTimeoutMs = durationArg(args, "wait-timeout", EDITOR_VALIDATE_BLOCKS_READY_TIMEOUT_MS) const topology = browserPreviewTopology(args, runtimeSpec, server.serverUrl, server.previewProxyDiagnostics?.targetOrigin) const { preview, networkPolicy } = topology + const routeTracker = createBrowserPreviewRouteTracker() const targetUrl = topology.resolveUrl(target.url) const artifactSession = new BrowserArtifactSession(artifactRoot, "files/browser", { source: "wordpress.editor-validate-blocks", operation: "editor-validate-blocks" }) @@ -1951,7 +1980,7 @@ export async function runEditorValidateBlocksCommand({ } const context = browserPreviewNeedsContextRouting(networkPolicy) ? await browser.newContext(topology.contextOptions()) : null if (context) { - await routeBrowserPreviewContextNetwork(context, networkPolicy, topology.origins.localProxyOrigin) + await routeBrowserPreviewContextNetwork(context, networkPolicy, topology.origins.localProxyOrigin, routeTracker) } const page = context ? await context.newPage() : await browser.newPage() authSummary = await installWordPressAdminAuthCookies({ command: "wordpress.editor-validate-blocks", cookieUrls: topology.authCookieUrls([targetUrl]), page, runPlaygroundCommand, runtimeSpec, server, userId: 1 }) @@ -1976,7 +2005,10 @@ export async function runEditorValidateBlocksCommand({ pendingError = error instanceof Error ? error : new Error(String(error)) errors.push(serializeBrowserError("probe-error", error)) } finally { - await browser.close() + for (const routeError of await closeBrowserAndDrainPreviewRoutes(browser, routeTracker)) { + errors.push(serializeBrowserError("probe-error", routeError)) + pendingError ??= routeError + } const summary: BrowserEditorValidateBlocksSummary | undefined = validation ? { @@ -2065,10 +2097,7 @@ export async function runEditorValidateBlocksCommand({ throw new BrowserCommandArtifactError("wordpress.editor-validate-blocks failed: block validation did not complete", artifact) } - return { - artifact, - output: `${JSON.stringify(validation.result, null, 2)}\n`, - } + return browserCommandResult(artifact, validation.result) } async function waitForEditorBlocksRuntime(page: import("playwright").Page, timeoutMs: number): Promise { diff --git a/scripts/smoke-manifest.ts b/scripts/smoke-manifest.ts index bcc0182f6..999cf8562 100644 --- a/scripts/smoke-manifest.ts +++ b/scripts/smoke-manifest.ts @@ -47,6 +47,8 @@ export const smokeGroups = { npmScript("test:browser-runner-template"), npmScript("test:browser-runtime-file-ops"), npmScript("test:browser-provider-bridge-inheritance"), + npmScript("test:browser-preview-routing"), + npmScript("test:browser-routed-command-security"), tsxSmoke("runtime-backend-registry-smoke"), tsxSmoke("backend-package-adapter-registry-smoke"), tsxSmoke("command-registry-smoke"), diff --git a/tests/browser-actions-environment.browser.test.ts b/tests/browser-actions-environment.browser.test.ts index 31ebe2c0b..64937660c 100644 --- a/tests/browser-actions-environment.browser.test.ts +++ b/tests/browser-actions-environment.browser.test.ts @@ -10,7 +10,7 @@ import { runBrowserActionsCommand, runBrowserScenarioCommand } from "../packages import { runBrowserProbeCommand } from "../packages/runtime-playground/src/browser-probe-runner.js" import type { BrowserArtifact } from "../packages/runtime-playground/src/browser-artifacts.js" import { isBrowserCommandArtifactError } from "../packages/runtime-playground/src/browser-command-artifact-error.js" -import type { PlaygroundCliServer } from "../packages/runtime-playground/src/preview-server.js" +import { closeHttpServer, listenLocalHttpServer, withPreviewProxy, type PlaygroundCliServer } from "../packages/runtime-playground/src/preview-server.js" const runtimeSpec: RuntimeCreateSpec = { backend: "wordpress-playground", @@ -95,6 +95,56 @@ test("authored scenarios preserve init-script and page state from probe collecti } }) +test("shared probe and actions sessions preserve canonical proxy transport and topology evidence", async () => { + const canonicalHost = "shared-scenario.invalid" + const upstreamRequests: Array<{ host?: string; url?: string }> = [] + const upstream = createServer((request, response) => { + upstreamRequests.push({ host: request.headers.host, url: request.url }) + response.setHeader("content-type", "text/html") + response.end("Shared canonical scenario") + }) + const upstreamUrl = await listenLocalHttpServer(upstream) + const proxy = await withPreviewProxy({ + playground: { async run() { return { text: "", exitCode: 0 } } }, + serverUrl: upstreamUrl, + async [Symbol.asyncDispose]() {}, + } satisfies PlaygroundCliServer, 0) + const artifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-browser-scenario-canonical-")) + const canonicalUrl = `http://${canonicalHost}/preview/` + try { + const result = await runBrowserScenarioCommand({ + artifactRoot, + runtimeSpec: { ...runtimeSpec, preview: { siteUrl: canonicalUrl } }, + server: proxy, + spec: { + command: "wordpress.browser-scenario", + args: [ + `scenario-json=${JSON.stringify({ + url: canonicalUrl, + captures: ["performance", "steps", "network"], + prePageScript: "sessionStorage.setItem('shared-scenario-state', 'probe-preserved')", + steps: [{ kind: "evaluate", expression: "({ origin: location.origin, state: sessionStorage.getItem('shared-scenario-state') })", assert: { origin: `http://${canonicalHost}`, state: "probe-preserved" } }], + })}`, + `route-host=${canonicalHost}`, + ], + }, + }) + + assert.equal(new URL(result.artifact.canonicalBrowserOrigin!).origin, `http://${canonicalHost}`) + assert.equal(new URL(result.artifact.localProxyOrigin!).origin, new URL(proxy.serverUrl).origin) + assert.equal(new URL(result.artifact.upstreamRuntimeOrigin!).origin, new URL(upstreamUrl).origin) + assert.equal(new URL(result.artifact.summary.finalUrl).hostname, canonicalHost) + assert(upstreamRequests.some((request) => request.host === canonicalHost && request.url === "/preview/"), JSON.stringify(upstreamRequests)) + const output = JSON.parse(result.output) + assert.equal(output.scenario.summary.probe.finalUrl, canonicalUrl) + assert.equal(output.scenario.summary.actions.finalUrl, canonicalUrl) + } finally { + await rm(artifactRoot, { recursive: true, force: true }) + await proxy[Symbol.asyncDispose]() + await closeHttpServer(upstream) + } +}) + test("scenario storage state and auth remain available after probe collection", async () => { const fixture = await browserFixture() try { diff --git a/tests/browser-preview-routing.test.ts b/tests/browser-preview-routing.test.ts index 28c1eff38..dca0fc1e9 100644 --- a/tests/browser-preview-routing.test.ts +++ b/tests/browser-preview-routing.test.ts @@ -1,10 +1,16 @@ import assert from "node:assert/strict" +import { readFile } from "node:fs/promises" +import { join } from "node:path" import test from "node:test" -import type { BrowserContext, Route } from "playwright" +import type { BrowserContext, Request, Response, Route } from "playwright" -import { browserPreviewNetworkPolicy, browserPreviewRouting, createBrowserPreviewRouteTracker, drainBrowserPreviewRouteTracker, isBrowserPreviewRouteFetchContentDecodingError, isBrowserPreviewRouteFetchRecoverableError, isBrowserPreviewRouteFetchRequestContextDisposedError, isBrowserPreviewRouteFetchTransientTransportError, routeBrowserPreviewContextNetwork } from "../packages/runtime-playground/src/browser-preview-routing.js" -import { jsonLines, serializeBrowserError } from "../packages/runtime-playground/src/browser-metrics.js" +import { BrowserArtifactSession } from "../packages/runtime-playground/src/browser-artifact-session.js" +import { serializeBrowserError, serializeBrowserRequestFailure, serializeBrowserResponse } from "../packages/runtime-playground/src/browser-metrics.js" +import { browserPreviewNetworkPolicy, browserPreviewRouting, closeBrowserAndDrainPreviewRoutes, createBrowserPreviewRouteTracker, drainBrowserPreviewRouteTracker, isBrowserPreviewRouteClosedError, isBrowserPreviewRouteFetchContentDecodingError, isBrowserPreviewRouteFetchRecoverableError, isBrowserPreviewRouteFetchRequestContextDisposedError, isBrowserPreviewRouteFetchTransientTransportError, routeBrowserPreviewContextNetwork } from "../packages/runtime-playground/src/browser-preview-routing.js" +import { BrowserProbeSessionResultBuilder, type BrowserProbeSessionResultInput } from "../packages/runtime-playground/src/browser-probe-session-result-builder.js" +import { createBrowserProbeProgressTracker } from "../packages/runtime-playground/src/browser-probe-support.js" +import { withTempDir } from "../scripts/test-kit.js" const SENTINELS = ["SENTINEL_COOKIE_2094", "SENTINEL_AUTH_2094", "SENTINEL_NONCE_2094", "SENTINEL_TOKEN_2094"] const ROUTED_URL = "http://routed.test/wp-includes/app.js?token=SENTINEL_TOKEN_2094" @@ -16,6 +22,7 @@ test("routed fetch classifiers include transport resets and preserve disposal/de assert.equal(isBrowserPreviewRouteFetchRequestContextDisposedError(routeFetchError("Request context disposed.")), true) assert.equal(isBrowserPreviewRouteFetchContentDecodingError(routeFetchError("failed to decompress 'br' encoding")), true) assert.equal(isBrowserPreviewRouteFetchRecoverableError(routeFetchError("read ECONNRESET")), true) + assert.equal(isBrowserPreviewRouteClosedError(new Error("route.continue: Target page, context or browser has been closed")), true) }) test("subresource resets retry once, abort safely, and never reject the route callback", async () => { @@ -37,7 +44,7 @@ test("document resets retry deterministically and fulfill after recovery", async assert.equal(fixture.abortCalls(), 0) }) -test("exhausted document resets fail through a sanitized tracker error without leaking to persisted surfaces", async () => { +test("exhausted document resets fail through a sanitized tracker error", async () => { const fixture = await routedFixture("document", [routeFetchError("read ECONNRESET"), routeFetchError("read ECONNRESET"), routeFetchError("read ECONNRESET")]) await assert.doesNotReject(fixture.run()) assert.equal(fixture.fetchCalls(), 3) @@ -45,21 +52,28 @@ test("exhausted document resets fail through a sanitized tracker error without l assert.equal(fixture.tracker.pending.size, 0) assert.equal(fixture.tracker.errors.length, 1) - const tracked = fixture.tracker.errors[0] await assert.rejects(drainBrowserPreviewRouteTracker(fixture.tracker), /classification=upstream-transport.*resourceType=document.*token=\[redacted\]/) - const serialized = serializeBrowserError("probe-error", tracked) - const persistedSurfaces = { - stdout: JSON.stringify(serialized), - stderr: tracked instanceof Error ? `${tracked.message}\n${tracked.stack}` : String(tracked), - diagnostics: JSON.stringify({ errors: [serialized] }), - manifest: JSON.stringify({ files: [{ diagnostics: serialized }] }), - artifact: jsonLines([serialized]), - tracker: JSON.stringify(fixture.tracker.errors.map((error) => serializeBrowserError("probe-error", error))), - snapshot: JSON.stringify(serialized), - } - for (const [surface, contents] of Object.entries(persistedSurfaces)) { - for (const sentinel of SENTINELS) assert.doesNotMatch(contents, new RegExp(sentinel), `${surface} must not contain ${sentinel}`) - } +}) + +test("POST document and subresource resets are never replayed", async () => { + const document = await routedFixture("document", [routeFetchError("read ECONNRESET"), routedResponse()], undefined, { method: "POST" }) + await assert.doesNotReject(document.run()) + assert.equal(document.fetchCalls(), 1) + assert.equal(document.fulfilledResponse(), undefined) + await assert.rejects(drainBrowserPreviewRouteTracker(document.tracker), /method=POST.*resourceType=document/) + + const subresource = await routedFixture("fetch", [routeFetchError("read ECONNRESET"), routedResponse()], undefined, { method: "POST" }) + await assert.doesNotReject(subresource.run()) + await assert.doesNotReject(drainBrowserPreviewRouteTracker(subresource.tracker)) + assert.equal(subresource.fetchCalls(), 1) + assert.equal(subresource.abortCalls(), 1) + assert.equal(subresource.fulfilledResponse(), undefined) + + const redirect = routedResponse(307, { location: "http://routed.test/after-mutation" }) + const redirectedPost = await routedFixture("document", [redirect], undefined, { method: "POST" }) + await assert.doesNotReject(redirectedPost.run()) + assert.equal(redirectedPost.fetchCalls(), 1) + assert.equal(redirectedPost.fulfilledResponse(), redirect) }) test("concurrent routed requests drain after independent retry and cleanup", async () => { @@ -73,6 +87,101 @@ test("concurrent routed requests drain after independent retry and cleanup", asy assert.equal(second.abortCalls(), 1) }) +test("drain observes a routed request registered while another request is settling", async () => { + let resolveFirst!: (response: ReturnType) => void + let resolveSecond!: (response: ReturnType) => void + const firstResponse = new Promise>((resolve) => { resolveFirst = resolve }) + const secondResponse = new Promise>((resolve) => { resolveSecond = resolve }) + const tracker = createBrowserPreviewRouteTracker() + const first = await routedFixture("script", [firstResponse], tracker) + const second = await routedFixture("image", [secondResponse], tracker) + const firstRun = first.run() + const drain = drainBrowserPreviewRouteTracker(tracker) + const secondRun = second.run() + resolveFirst(routedResponse()) + await firstRun + await new Promise((resolve) => setTimeout(resolve, 5)) + let drained = false + void drain.then(() => { drained = true }) + assert.equal(drained, false) + resolveSecond(routedResponse()) + await Promise.all([secondRun, drain]) + assert.equal(tracker.pending.size, 0) +}) + +test("shared browser cleanup drains late registrations even when browser close fails", async () => { + const tracker = createBrowserPreviewRouteTracker() + let release!: () => void + const task = new Promise((resolve) => { release = resolve }) + const browser = { + async close() { + tracker.registrations += 1 + tracker.pending.add(task) + setTimeout(() => { + tracker.pending.delete(task) + release() + }, 5) + throw routeFetchError("browser close failed") + }, + } + const lifecycleErrors = await closeBrowserAndDrainPreviewRoutes(browser, tracker) + assert.equal(tracker.pending.size, 0) + assert.equal(lifecycleErrors.length, 1) + assert.match(lifecycleErrors[0]!.message, /operation=browser-close/) + assertNoSentinels(lifecycleErrors[0]!.message, "cleanup diagnostic") +}) + +test("shared browser cleanup bounds a close operation that never settles", async () => { + const tracker = createBrowserPreviewRouteTracker() + const startedAt = Date.now() + const lifecycleErrors = await closeBrowserAndDrainPreviewRoutes({ close: () => new Promise(() => {}) }, tracker, 10) + assert(Date.now() - startedAt < 250) + assert.equal(lifecycleErrors.length, 1) + assert.match(lifecycleErrors[0]!.message, /operation=browser-close-timeout/) +}) + +test("the complete route callback contains continue and policy abort failures", async () => { + const cases = [ + { name: "invalid URL continue", options: { url: "not-a-url", continueErrors: [routeFetchError("continue failed")] }, operation: "continue-invalid-url" }, + { name: "ordinary continue", options: { url: "https://ordinary.test/path?token=SENTINEL_TOKEN_2094", continueErrors: [routeFetchError("continue failed")] }, operation: "continue-unrouted" }, + { name: "policy abort", options: { url: "https://blocked.test/path?token=SENTINEL_TOKEN_2094", policyArgs: ["network-policy=block"], abortErrors: [routeFetchError("abort failed")] }, operation: "abort-policy-block" }, + ] + for (const item of cases) { + const fixture = await routedFixture("script", [], undefined, item.options) + await assert.doesNotReject(fixture.run(), item.name) + await assert.rejects(drainBrowserPreviewRouteTracker(fixture.tracker), new RegExp(`operation=${item.operation}.*token=\\[redacted\\]`)) + assertNoSentinels(JSON.stringify(fixture.tracker.errors), item.name) + } +}) + +test("closed-context failures across the route callback are swallowed during cleanup", async () => { + const closed = new Error("route.continue: Target page, context or browser has been closed") + const fixture = await routedFixture("script", [], undefined, { url: "https://ordinary.test/", continueErrors: [closed], abortErrors: [closed] }) + await assert.doesNotReject(fixture.run()) + await assert.doesNotReject(drainBrowserPreviewRouteTracker(fixture.tracker)) + assert.equal(fixture.tracker.errors.length, 0) +}) + +test("unexpected recoverable and redirect abort failures are operation-classified", async () => { + const cases = [ + { outcomes: [routeFetchError("failed to decompress 'gzip' encoding")], operation: "abort-recoverable-fetch" }, + { outcomes: [routedResponse(302, { location: "https://outside.test/escape" })], operation: "abort-redirect-escape" }, + { outcomes: Array.from({ length: 10 }, () => routedResponse(302, { location: "http://routed.test/loop" })), operation: "abort-redirect-limit" }, + ] + for (const item of cases) { + const fixture = await routedFixture("script", item.outcomes, undefined, { abortErrors: [routeFetchError("abort transport failed")] }) + await assert.doesNotReject(fixture.run()) + await assert.rejects(drainBrowserPreviewRouteTracker(fixture.tracker), new RegExp(`operation=${item.operation}`)) + assertNoSentinels(JSON.stringify(fixture.tracker.errors), item.operation) + } + + const closedAbort = await routedFixture("script", [routeFetchError("failed to decompress 'gzip' encoding")], undefined, { + abortErrors: [new Error("route.abort: Target page, context or browser has been closed")], + }) + await assert.doesNotReject(closedAbort.run()) + await assert.doesNotReject(drainBrowserPreviewRouteTracker(closedAbort.tracker)) +}) + test("disposed contexts and decompression failures abort without retrying or tracking errors", async () => { for (const message of ["Request context disposed.", "failed to decompress 'gzip' encoding"]) { const fixture = await routedFixture("document", [routeFetchError(message)]) @@ -83,10 +192,55 @@ test("disposed contexts and decompression failures abort without retrying or tra } }) -async function routedFixture(resourceType: string, outcomes: unknown[], tracker = createBrowserPreviewRouteTracker()) { +test("real network serialization and artifact composition remove synthetic request secrets", async () => { + const rawUrl = ROUTED_URL + const rawError = routeFetchError("read ECONNRESET") + const networkRecord = serializeBrowserRequestFailure({ + url: () => rawUrl, + method: () => "GET", + resourceType: () => "script", + timing: () => ({}), + failure: () => ({ errorText: rawError.message }), + } as unknown as Request, "2026-01-01T00:00:00.000Z") + assertNoSentinels(JSON.stringify(networkRecord), "serialized network record") + const responseRecord = await serializeBrowserResponse({ + url: () => rawUrl, + request: () => ({ method: () => "GET", resourceType: () => "script", timing: () => ({}) }), + status: () => 200, + statusText: () => "OK", + ok: () => true, + headers: () => ({ "content-type": "application/javascript" }), + } as unknown as Response, "2026-01-01T00:00:01.000Z") + assertNoSentinels(JSON.stringify(responseRecord), "serialized response record") + + const serializedError = serializeBrowserError("probe-error", rawError) + const result = new BrowserProbeSessionResultBuilder().compose(resultBuilderInput([networkRecord, responseRecord], serializedError)) + await withTempDir("wp-codebox-route-redaction-", async (artifactRoot) => { + const session = new BrowserArtifactSession(artifactRoot, "files/browser", { source: "wordpress.browser-probe", operation: "browser-probe" }) + await session.writeJsonLines("network", "network.jsonl", [networkRecord, responseRecord]) + await session.writeJsonLines("errors", "errors.jsonl", [serializedError]) + await session.writeJson("review", "review.json", result.review) + await session.writeJson("summary", "summary.json", result.summary) + const persisted = { + network: await readFile(join(artifactRoot, "files/browser/network.jsonl"), "utf8"), + review: await readFile(join(artifactRoot, "files/browser/review.json"), "utf8"), + summary: await readFile(join(artifactRoot, "files/browser/summary.json"), "utf8"), + errors: await readFile(join(artifactRoot, "files/browser/errors.jsonl"), "utf8"), + artifact: JSON.stringify(result.artifact), + stdout: result.output, + stderr: `${serializedError.message}\n${serializedError.stack}`, + } + for (const [surface, contents] of Object.entries(persisted)) assertNoSentinels(contents, surface) + assert.match(persisted.network, /token=\[redacted\]/) + assert.match(persisted.stdout, /token=\[redacted\]/) + }) +}) + +async function routedFixture(resourceType: string, outcomes: unknown[], tracker = createBrowserPreviewRouteTracker(), options: { method?: string; url?: string; policyArgs?: string[]; continueErrors?: Error[]; abortErrors?: Error[] } = {}) { let handler: ((route: Route) => Promise) | undefined let fetchCalls = 0 let abortCalls = 0 + let continueCalls = 0 let fulfilled: unknown const context = { route: async (_pattern: string, nextHandler: (route: Route) => Promise) => { @@ -94,13 +248,13 @@ async function routedFixture(resourceType: string, outcomes: unknown[], tracker }, } as BrowserContext const preview = browserPreviewRouting([], undefined, "http://127.0.0.1:9400") - const policy = browserPreviewNetworkPolicy([], ["routed.test"], preview) + const policy = browserPreviewNetworkPolicy(options.policyArgs ?? [], ["routed.test"], preview) await routeBrowserPreviewContextNetwork(context, policy, preview.effectiveOrigin, tracker) const route = { request: () => ({ - url: () => ROUTED_URL, - method: () => "GET", + url: () => options.url ?? ROUTED_URL, + method: () => options.method ?? "GET", resourceType: () => resourceType, headers: () => ({ cookie: `wordpress_logged_in=${SENTINELS[0]}`, @@ -116,12 +270,18 @@ async function routedFixture(resourceType: string, outcomes: unknown[], tracker return outcome }, abort: async () => { + const error = options.abortErrors?.[abortCalls] abortCalls += 1 + if (error) throw error }, fulfill: async ({ response }: { response: unknown }) => { fulfilled = response }, - continue: async () => {}, + continue: async () => { + const error = options.continueErrors?.[continueCalls] + continueCalls += 1 + if (error) throw error + }, } as unknown as Route return { @@ -136,6 +296,41 @@ async function routedFixture(resourceType: string, outcomes: unknown[], tracker } } +function resultBuilderInput(network: BrowserProbeSessionResultInput["network"], error: ReturnType): BrowserProbeSessionResultInput { + return { + assertions: [], + browser: { name: "chromium", channel: "bundled", version: null }, + browserFilesDirectory: "files/browser", + capture: new Set(["network", "errors"]), + captureSelection: { console: false, errors: true, network: true, metrics: false, consoleForAssertions: false, errorsForAssertions: false, networkForAssertions: false }, + checkpoints: [], + command: "wordpress.browser-probe", + consoleMessages: [], + durationMs: 0, + errors: [error], + failFast: false, + finalUrl: ROUTED_URL, + hashes: {}, + lifecycleSelectors: [], + liveness: { wallTimeoutMs: 30_000, stallTimeoutMs: 0, networkSettleTimeoutMs: 500 }, + network, + preview: { requestedMode: "local", effectiveMode: "local", localOrigin: "http://127.0.0.1:9400", effectiveOrigin: "http://127.0.0.1:9400", diagnostics: [] }, + progress: createBrowserProbeProgressTracker("2026-01-01T00:00:00.000Z", 0), + requestedUrl: ROUTED_URL, + startedAt: "2026-01-01T00:00:00.000Z", + startedAtMs: Date.now(), + throttleId: null, + topologyOrigins: { localPreviewOrigin: "http://127.0.0.1:9400", effectivePreviewOrigin: "http://127.0.0.1:9400" }, + viewport: null, + waitFor: "domcontentloaded", + webSockets: [], + } +} + +function assertNoSentinels(contents: string, surface: string): void { + for (const sentinel of SENTINELS) assert.doesNotMatch(contents, new RegExp(sentinel), `${surface} must not contain ${sentinel}`) +} + function routeFetchError(reason: string): Error { const message = `route.fetch: ${reason}\nCall log:\n - → GET ${ROUTED_URL}\n cookie: wordpress_logged_in=${SENTINELS[0]}\n authorization: Bearer ${SENTINELS[1]}\n x-wp-nonce: ${SENTINELS[2]}\n x-session-token: ${SENTINELS[3]}` const error = new Error(message) @@ -143,9 +338,9 @@ function routeFetchError(reason: string): Error { return error } -function routedResponse() { +function routedResponse(status = 200, headers: Record = {}) { return { - status: () => 200, - headers: () => ({}), + status: () => status, + headers: () => headers, } } diff --git a/tests/browser-routed-command-security.test.ts b/tests/browser-routed-command-security.test.ts new file mode 100644 index 000000000..c74117cf3 --- /dev/null +++ b/tests/browser-routed-command-security.test.ts @@ -0,0 +1,225 @@ +import assert from "node:assert/strict" +import { readFile } from "node:fs/promises" +import { createServer } from "node:http" +import { join } from "node:path" +import test from "node:test" + +import type { RuntimeCreateSpec } from "../packages/runtime-core/src/runtime-contracts.js" +import { runBrowserActionsCommand, runBrowserScenarioCommand } from "../packages/runtime-playground/src/browser-actions-runner.js" +import { isBrowserCommandArtifactError } from "../packages/runtime-playground/src/browser-command-artifact-error.js" +import { runBrowserMultiActorScenarioCommand } from "../packages/runtime-playground/src/browser-multi-actor-scenario-runner.js" +import { runBrowserProbeCommand } from "../packages/runtime-playground/src/browser-probe-runner.js" +import { runEditorCanvasProbeCommand, runEditorOpenCommand } from "../packages/runtime-playground/src/editor-command-runners.js" +import { closeHttpServer, listenLocalHttpServer, type PlaygroundCliServer } from "../packages/runtime-playground/src/preview-server.js" +import { withTempDir } from "../scripts/test-kit.js" + +const TOKEN = "SENTINEL_ROUTED_COMMAND_TOKEN_2094" +const PUBLIC_URL = `http://routed.test/editor?token=${TOKEN}` +const editorHtml = `
Editor fixture
` + +test("real browser commands sanitize console, artifacts, stdout, and failure stderr", async () => { + const httpServer = createServer((request, response) => { + response.setHeader("content-type", "text/html") + response.end(request.url?.startsWith("/broken") ? "
Broken editor fixture
" : editorHtml) + }) + const serverUrl = await listenLocalHttpServer(httpServer) + const server = { serverUrl, playground: {} } as PlaygroundCliServer + const runtimeSpec = { environment: { blueprint: {} } } as RuntimeCreateSpec + const actorRuntimeSpec = { + environment: { blueprint: {} }, + metadata: { + recipe: { + inputs: { + fixtureUsers: [{ name: "author", username: "fixture-author", role: "author", userId: 1 }], + userSessions: [{ name: "author-session", user: "author" }], + }, + }, + }, + } as RuntimeCreateSpec + const runPlaygroundCommand = async () => ({ text: "[]", exitCode: 0 }) + + try { + await withTempDir("wp-codebox-real-browser-actions-security-", async (artifactRoot) => { + const result = await runBrowserActionsCommand({ + artifactRoot, + plan: { initialUrl: PUBLIC_URL, steps: [], capture: new Set(["steps", "network", "console"]), stepTimeoutMs: 5_000, totalTimeoutMs: 15_000, networkSettleTimeoutMs: 500, maxDomSnapshotElements: 50 }, + runtimeSpec, + server, + spec: { command: "wordpress.browser-actions", args: ["route-host=routed.test"] }, + }) + await assertCommandSurfacesSafe(result, artifactRoot, ["files/browser/action-summary.json", "files/browser/network.jsonl", "files/browser/console.jsonl"]) + }) + + await withTempDir("wp-codebox-real-editor-security-", async (artifactRoot) => { + const result = await runEditorOpenCommand({ + artifactRoot, + runPlaygroundCommand, + runtimeSpec, + server, + spec: { command: "wordpress.editor-open", args: [`url=${PUBLIC_URL}`, "route-host=routed.test", "capture=steps,errors,console", "wait-timeout=5s"] }, + }) + await assertCommandSurfacesSafe(result, artifactRoot, ["files/browser/editor-summary.json", "files/browser/editor-steps.jsonl", "files/browser/editor-console.jsonl"]) + }) + + await withTempDir("wp-codebox-real-editor-canvas-security-", async (artifactRoot) => { + const result = await runEditorCanvasProbeCommand({ + artifactRoot, + runtimeSpec, + server, + spec: { command: "wordpress.editor-canvas-probe", args: [`url=${PUBLIC_URL}`, "route-host=routed.test", "timeout=5s"] }, + }) + await assertCommandSurfacesSafe(result, artifactRoot, ["files/browser/editor-canvas-summary.json"]) + }) + + await withTempDir("wp-codebox-real-multi-actor-security-", async (artifactRoot) => { + const result = await runBrowserMultiActorScenarioCommand({ + artifactRoot, + runtimeSpec: actorRuntimeSpec, + runPlaygroundCommand, + scenario: { + schema: "wp-codebox/browser-multi-actor-scenario/v1", + seed: "route-security", + url: `${serverUrl}/editor?token=${TOKEN}`, + actors: [{ name: "author", userSession: "author-session" }], + actions: [], + browserArgs: [], + captures: ["steps", "network", "console"], + environment: {}, + }, + server, + }) + await assertCommandSurfacesSafe(result, artifactRoot, ["files/browser/multi-actor-scenario-summary.json", "files/browser/multi-actor-network.json", "files/browser/author-console.jsonl"]) + }) + + await withTempDir("wp-codebox-real-scenario-security-", async (artifactRoot) => { + const result = await runBrowserScenarioCommand({ + artifactRoot, + runtimeSpec, + server, + spec: { command: "wordpress.browser-scenario", args: [`scenario-json=${JSON.stringify({ url: PUBLIC_URL, captures: ["console", "network"] })}`, "route-host=routed.test"] }, + }) + await assertCommandSurfacesSafe(result, artifactRoot, ["files/browser/scenario-summary.json", "files/browser/console.jsonl"]) + }) + + await assertFailingCommandStderrSafe("actions", async (artifactRoot) => runBrowserActionsCommand({ + artifactRoot, + plan: { initialUrl: PUBLIC_URL, steps: [{ kind: "click", selector: "#missing" }], capture: new Set(["errors"]), stepTimeoutMs: 250, totalTimeoutMs: 5_000, networkSettleTimeoutMs: 100, maxDomSnapshotElements: 20 }, + runtimeSpec, + server, + spec: { command: "wordpress.browser-actions", args: ["route-host=routed.test"] }, + })) + await assertFailingCommandStderrSafe("scenario", async (artifactRoot) => runBrowserScenarioCommand({ + artifactRoot, + runtimeSpec, + server, + spec: { command: "wordpress.browser-scenario", args: [`scenario-json=${JSON.stringify({ url: PUBLIC_URL, captures: ["errors"], steps: [{ kind: "click", selector: "#missing" }], stepTimeout: "250ms", timeout: "5s" })}`, "route-host=routed.test"] }, + })) + await assertFailingCommandStderrSafe("editor", async (artifactRoot) => runEditorOpenCommand({ + artifactRoot, + runPlaygroundCommand, + runtimeSpec, + server, + spec: { command: "wordpress.editor-open", args: [`url=http://routed.test/broken?token=${TOKEN}`, "route-host=routed.test", "capture=errors", "wait-timeout=250ms"] }, + })) + await assertFailingCommandStderrSafe("multi-actor", async (artifactRoot) => runBrowserMultiActorScenarioCommand({ + artifactRoot, + runtimeSpec: actorRuntimeSpec, + runPlaygroundCommand, + scenario: { + schema: "wp-codebox/browser-multi-actor-scenario/v1", + seed: "route-security-failure", + url: `${serverUrl}/editor?token=${TOKEN}`, + actors: [{ name: "author", userSession: "author-session" }], + actions: [{ id: "missing-click", actor: "author", step: { kind: "click", selector: "#missing" } }], + browserArgs: [], + captures: ["errors"], + environment: {}, + stepTimeoutMs: 250, + }, + server, + })) + + const controller = new AbortController() + controller.abort() + await assertFailingCommandStderrSafe("cancelled-probe", async (artifactRoot) => runBrowserProbeCommand({ + abortSignal: controller.signal, + artifactRoot, + plan: { url: PUBLIC_URL, capture: new Set(["errors", "network"]), waitFor: "domcontentloaded", durationMs: 0, requestedContext: {}, routeHostDrain: "required", failFast: false, stallTimeoutMs: 0, wallTimeoutMs: 5_000, lifecycleSelectors: [], assertions: [] }, + runtimeSpec, + server, + spec: { command: "wordpress.browser-probe", args: ["route-host=routed.test"] }, + }), true) + await assertFailingCommandStderrSafe("final-checkpoint-lifecycle-probe", async (artifactRoot) => runBrowserProbeCommand({ + artifactRoot, + plan: { + url: PUBLIC_URL, + capture: new Set(["errors", "memory"]), + waitFor: "domcontentloaded", + durationMs: 0, + requestedContext: {}, + script: `Object.defineProperty(window, 'performance', { value: null }); Document.prototype.querySelectorAll = () => { throw new Error('lifecycle ${PUBLIC_URL}') }`, + routeHostDrain: "required", + failFast: false, + stallTimeoutMs: 0, + wallTimeoutMs: 5_000, + lifecycleSelectors: ["body"], + assertions: [], + }, + runtimeSpec, + server, + spec: { command: "wordpress.browser-probe", args: ["route-host=routed.test"] }, + }), true) + } finally { + await closeHttpServer(httpServer) + } +}) + +async function assertCommandSurfacesSafe(result: { artifact: unknown; output: string }, artifactRoot: string, files: string[]): Promise { + const surfaces: Record = { + artifact: JSON.stringify(result.artifact), + stdout: result.output, + } + for (const file of files) surfaces[file] = await readFile(join(artifactRoot, file), "utf8") + for (const [surface, contents] of Object.entries(surfaces)) { + assert.doesNotMatch(contents, new RegExp(TOKEN), `${surface} leaked the synthetic token`) + } + for (const [surface, contents] of Object.entries(surfaces).filter(([surface]) => surface.includes("console"))) { + assert.match(contents, /normal console text/, `${surface} over-redacted ordinary console text`) + } + assert.match(JSON.stringify(surfaces), /\[redacted\]/) +} + +async function assertFailingCommandStderrSafe(name: string, run: (artifactRoot: string) => Promise, requireArtifact = false): Promise { + await withTempDir(`wp-codebox-${name}-failure-security-`, async (artifactRoot) => { + let failure: unknown + try { + await run(artifactRoot) + } catch (error) { + failure = error + } + assert(failure instanceof Error, `${name} should fail`) + assert.doesNotMatch(`${failure.message}\n${failure.stack}`, new RegExp(TOKEN), `${name} stderr leaked the synthetic token`) + if (isBrowserCommandArtifactError(failure)) { + assert.doesNotMatch(JSON.stringify((failure as { artifact: unknown }).artifact), new RegExp(TOKEN)) + } + if (requireArtifact) { + assert.equal(isBrowserCommandArtifactError(failure), true, `${name} should retain a reviewer-safe artifact`) + await readFile(join(artifactRoot, "files/browser/summary.json"), "utf8") + } + }) +}