diff --git a/.github/workflows/agent-task-contracts.yml b/.github/workflows/agent-task-contracts.yml index fb397eaed..616c976b5 100644 --- a/.github/workflows/agent-task-contracts.yml +++ b/.github/workflows/agent-task-contracts.yml @@ -24,7 +24,10 @@ on: - "tests/production-boundary-enforcement.test.ts" - "tests/runtime-tool-policy.test.ts" - "tests/browser-canonical-preview-origin.test.ts" + - "tests/browser-adaptive-exploration.test.ts" + - "packages/runtime-core/src/browser-adaptive-exploration.ts" - "packages/runtime-playground/src/browser-actions-runner.ts" + - "packages/runtime-playground/src/browser-adaptive-explorer.ts" - "packages/runtime-playground/src/browser-artifacts.ts" - "packages/runtime-playground/src/browser-multi-actor-scenario-runner.ts" - "packages/runtime-playground/src/browser-preview-routing.ts" @@ -65,7 +68,10 @@ on: - "tests/production-boundary-enforcement.test.ts" - "tests/runtime-tool-policy.test.ts" - "tests/browser-canonical-preview-origin.test.ts" + - "tests/browser-adaptive-exploration.test.ts" + - "packages/runtime-core/src/browser-adaptive-exploration.ts" - "packages/runtime-playground/src/browser-actions-runner.ts" + - "packages/runtime-playground/src/browser-adaptive-explorer.ts" - "packages/runtime-playground/src/browser-artifacts.ts" - "packages/runtime-playground/src/browser-multi-actor-scenario-runner.ts" - "packages/runtime-playground/src/browser-preview-routing.ts" @@ -108,6 +114,7 @@ jobs: - name: Install Playwright Chromium run: npx playwright install --with-deps chromium - run: npm run build + - run: npm run test:browser-accessibility-oracles - run: npm run test:agent-task-contracts - run: npm run test:browser-canonical-preview-origin - run: npm run test:bounded-runtime-plan diff --git a/packages/runtime-core/src/browser-adaptive-exploration.ts b/packages/runtime-core/src/browser-adaptive-exploration.ts index dd96d833d..04b9600fb 100644 --- a/packages/runtime-core/src/browser-adaptive-exploration.ts +++ b/packages/runtime-core/src/browser-adaptive-exploration.ts @@ -86,6 +86,13 @@ export interface BrowserAdaptiveTransition { loadingAfter: number oracleFingerprints: string[] networkFailures?: BrowserAdaptiveNetworkFailure[] + networkFailureSummary?: { + total: number + retained: number + policyBlocks: number + oracleFindings: number + truncated: boolean + } accessibilityFindingFingerprints?: string[] } status: "ok" | "revisited" | "rejected" | "error" | "cancelled" diff --git a/packages/runtime-playground/src/browser-adaptive-explorer.ts b/packages/runtime-playground/src/browser-adaptive-explorer.ts index 96ac82c49..b3dc5b545 100644 --- a/packages/runtime-playground/src/browser-adaptive-explorer.ts +++ b/packages/runtime-playground/src/browser-adaptive-explorer.ts @@ -45,6 +45,11 @@ interface FrontierEntry { path: BrowserAdaptiveAction[] } +interface AdaptiveNetworkFailureRetention { + remainingCount: number + remainingBytes: number +} + export async function exploreAdaptiveBrowserStateMachine({ page, baseUrl, @@ -79,6 +84,10 @@ export async function exploreAdaptiveBrowserStateMachine({ let revisits = 0 let keyboardActions = 0 let exhausted: BrowserAdaptiveExplorationResult["summary"]["budgetExhausted"] + const networkFailureRetention: AdaptiveNetworkFailureRetention = { + remainingCount: Math.min(contract.budgets.maxErrors, contract.descriptorLimits.maxDiagnostics), + remainingBytes: Math.floor(contract.budgets.maxArtifactBytes / 8), + } const initial = await captureAdaptiveState(page, contract, 0, navigationScope) appendDiagnostics(diagnostics, initial.diagnostics, contract.descriptorLimits.maxDiagnostics) @@ -144,7 +153,7 @@ export async function exploreAdaptiveBrowserStateMachine({ const newConsoleRecords = consoleErrorRecords(observations.consoleMessages.slice(beforeConsole)) const newConsoleErrors = newConsoleRecords.map(recordMessage) const newPageErrors = errorMessages(observations.errors.slice(beforeErrors)) - const oracleEvidence = adaptiveOracleEvidence(newConsoleRecords, observations.network.slice(beforeNetwork), newPageErrors, contract, networkPolicy) + const oracleEvidence = adaptiveOracleEvidence(newConsoleRecords, observations.network.slice(beforeNetwork), newPageErrors, contract, networkPolicy, networkFailureRetention) const fingerprints = [...oracleEvidence.fingerprints] const existing = states.get(stabilized.state.digest) const newState = !existing @@ -189,6 +198,7 @@ export async function exploreAdaptiveBrowserStateMachine({ loadingAfter: stabilized.loading, oracleFingerprints: fingerprints, ...(oracleEvidence.networkFailures.length > 0 ? { networkFailures: oracleEvidence.networkFailures } : {}), + ...(oracleEvidence.networkFailureSummary ? { networkFailureSummary: oracleEvidence.networkFailureSummary } : {}), ...(accessibilityFingerprints.length > 0 ? { accessibilityFindingFingerprints: accessibilityFingerprints } : {}), }, status: actionError ? "error" : scopeRejection ? "rejected" : newState ? "ok" : "revisited", @@ -569,34 +579,70 @@ function consoleErrorRecords(records: object[]): Record[] { return records.map((record) => record as Record).filter((record) => record.type === "error" || record.level === "error").slice(0, 100) } -function adaptiveOracleEvidence(consoleRecords: Record[], networkRecords: object[], pageErrors: string[], contract: BrowserAdaptiveExplorationContract, networkPolicy?: BrowserPreviewNetworkPolicy): { fingerprints: string[]; networkFailures: BrowserAdaptiveNetworkFailure[]; errorCount: number } { +function adaptiveOracleEvidence(consoleRecords: Record[], networkRecords: object[], pageErrors: string[], contract: BrowserAdaptiveExplorationContract, networkPolicy?: BrowserPreviewNetworkPolicy, retention?: AdaptiveNetworkFailureRetention): { fingerprints: string[]; networkFailures: BrowserAdaptiveNetworkFailure[]; networkFailureSummary?: BrowserAdaptiveTransition["observations"]["networkFailureSummary"]; errorCount: number } { const failures = networkRecords.map((record) => record as Record).filter((record) => record.type === "requestfailed") - const networkFailures = failures.map((record): BrowserAdaptiveNetworkFailure => { + const classifiedFailures = failures.map((record): BrowserAdaptiveNetworkFailure & { expectedBlock: boolean } => { const url = typeof record.url === "string" ? record.url : "" const decision = networkPolicy ? browserPreviewNetworkDecision(url, networkPolicy) : { url, urlClassification: "invalid" as const, policyDecision: "unknown" as const, policyReason: "network-policy-unavailable" } const failure = networkFailureMessage(record) const expectedBlock = decision.policyDecision === "blocked" && /ERR_BLOCKED_BY_CLIENT|blockedbyclient/i.test(failure) - return { ...decision, ...(failure ? { failure } : {}), oracleFinding: !expectedBlock || contract.oraclePolicy.policyBlocks === "finding" } + return { ...decision, ...(failure ? { failure } : {}), oracleFinding: !expectedBlock || contract.oraclePolicy.policyBlocks === "finding", expectedBlock } }) - const expectedBlockUrls = networkFailures.filter((failure) => !failure.oracleFinding).map((failure) => failure.url) - const findingConsoleRecords = consoleRecords.filter((record) => { - if (!/ERR_BLOCKED_BY_CLIENT/i.test(recordMessage(record))) return true - const location = objectRecord(record.location) - const match = typeof location.url === "string" && location.url - ? expectedBlockUrls.indexOf(location.url) - : expectedBlockUrls.length > 0 ? 0 : -1 - if (match < 0) return true - expectedBlockUrls.splice(match, 1) - return false + const unmatchedFailures = new Set(classifiedFailures.map((_failure, index) => index)) + const consoleMessages = consoleRecords.flatMap((record) => { + const message = recordMessage(record) + const locationUrl = objectRecord(record.location).url + const match = classifiedFailures.findIndex((failure, index) => unmatchedFailures.has(index) + && (typeof locationUrl === "string" && locationUrl ? failure.url === locationUrl && failureTokenMatches(message, failure.failure) : failureTokenMatches(message, failure.failure))) + if (match < 0) return [message] + unmatchedFailures.delete(match) + const failure = classifiedFailures[match]! + return failure.expectedBlock && contract.oraclePolicy.policyBlocks === "evidence" ? [] : [message] }) - const consoleUrls = new Set(findingConsoleRecords.map((record) => objectRecord(record.location).url).filter((url): url is string => typeof url === "string")) - const networkMessages = networkFailures.filter((failure) => failure.oracleFinding && !consoleUrls.has(failure.url)).map((failure) => stableJson({ type: "requestfailed", url: failure.url, failure: failure.failure, urlClassification: failure.urlClassification, policyDecision: failure.policyDecision, policyReason: failure.policyReason })) - const messages = [...findingConsoleRecords.map(recordMessage), ...pageErrors, ...networkMessages] + const networkMessages = classifiedFailures.filter((failure, index) => unmatchedFailures.has(index) && failure.oracleFinding).map(networkFailureOracleMessage) + const messages = [...consoleMessages, ...pageErrors, ...networkMessages] + const networkFailureSummary = classifiedFailures.length > 0 ? boundedNetworkFailureEvidence(classifiedFailures, contract, retention) : undefined return { fingerprints: [...new Set(messages.map((message) => browserAdaptiveDigest("oracle", message)))].sort(), - networkFailures, - errorCount: findingConsoleRecords.length + pageErrors.length + networkMessages.length, + networkFailures: networkFailureSummary?.failures ?? [], + networkFailureSummary: networkFailureSummary?.summary, + errorCount: consoleMessages.length + pageErrors.length + networkMessages.length, + } +} + +function boundedNetworkFailureEvidence(failures: Array, contract: BrowserAdaptiveExplorationContract, cumulativeRetention?: AdaptiveNetworkFailureRetention): { failures: BrowserAdaptiveNetworkFailure[]; summary: NonNullable } { + const ordered = failures.map(({ expectedBlock: _expectedBlock, ...failure }) => failure).sort((left, right) => stableJson(left).localeCompare(stableJson(right))) + const retention = cumulativeRetention ?? { + remainingCount: Math.min(contract.budgets.maxErrors, contract.descriptorLimits.maxDiagnostics), + remainingBytes: Math.floor(contract.budgets.maxArtifactBytes / 8), + } + const retained: BrowserAdaptiveNetworkFailure[] = [] + for (const failure of ordered.slice(0, retention.remainingCount)) { + const bytes = Buffer.byteLength(stableJson(failure)) + 1 + if (bytes > retention.remainingBytes) break + retained.push(failure) + retention.remainingCount -= 1 + retention.remainingBytes -= bytes } + return { + failures: retained, + summary: { + total: failures.length, + retained: retained.length, + policyBlocks: failures.filter((failure) => failure.expectedBlock).length, + oracleFindings: failures.filter((failure) => failure.oracleFinding).length, + truncated: retained.length < failures.length, + }, + } +} + +function failureTokenMatches(message: string, failure: string | undefined): boolean { + const token = failure?.match(/ERR_[A-Z0-9_]+/i)?.[0] + return Boolean(token && message.toUpperCase().includes(token.toUpperCase())) +} + +function networkFailureOracleMessage(failure: BrowserAdaptiveNetworkFailure): string { + return stableJson({ type: "requestfailed", url: failure.url, failure: failure.failure, urlClassification: failure.urlClassification, policyDecision: failure.policyDecision, policyReason: failure.policyReason }) } function networkFailureMessage(record: Record): string { diff --git a/tests/browser-adaptive-exploration.test.ts b/tests/browser-adaptive-exploration.test.ts index 50002e1e6..b3a0fdcf7 100644 --- a/tests/browser-adaptive-exploration.test.ts +++ b/tests/browser-adaptive-exploration.test.ts @@ -5,6 +5,7 @@ import { chromium, type Page } from "playwright" import { BROWSER_ADAPTIVE_EXPLORATION_SCHEMA, + browserAdaptiveDigest, browserAdaptiveExplorationContract, planBrowserAdaptiveStateActions, } from "../packages/runtime-core/src/browser-adaptive-exploration.js" @@ -506,6 +507,45 @@ test("callers can opt policy blocks back into deterministic findings and replay" assert.equal(finding.replay.expectedFingerprint, finding.fingerprint) }) +test("URL-less policy-block console records correlate before promotion and preserve canonical fingerprints", async () => { + const evidenceOnly = await runNetworkOracleFixture("block", "blocked", {}, true) + assert.equal(evidenceOnly.result.findings.length, 0) + assert.deepEqual(evidenceOnly.result.transitions[0]?.observations.oracleFingerprints, []) + assert.equal(evidenceOnly.result.transitions[0]?.observations.networkFailureSummary?.policyBlocks, 1) + + const input = { oraclePolicy: { policyBlocks: "finding" } } + const attributed = await runNetworkOracleFixture("block", "blocked", input) + const urlLess = await runNetworkOracleFixture("block", "blocked", input, true) + const message = urlLess.result.transitions[0]?.observations.consoleErrors[0] + assert(message) + assert.equal(urlLess.result.transitions[0]?.observations.oracleFingerprints.length, 1) + assert.equal(urlLess.result.findings[0]?.fingerprint, browserAdaptiveDigest("oracle", message)) + assert.equal(urlLess.result.findings[0]?.fingerprint, attributed.result.findings[0]?.fingerprint, "removing console location must not change the historical console fingerprint") + + const mixed = await runNetworkOracleFixture("block", "mixed", {}, true) + assert.equal(mixed.result.findings.length, 1) + assert.equal(mixed.result.transitions[0]?.observations.oracleFingerprints.length, 1) + assert.equal(mixed.result.transitions[0]?.observations.networkFailureSummary?.policyBlocks, 1) + assert.equal(mixed.result.transitions[0]?.observations.networkFailureSummary?.oracleFindings, 1) +}) + +test("same-URL product errors require a matching failure token before policy correlation", async () => { + const productMessage = "same-URL product defect" + const productFingerprint = browserAdaptiveDigest("oracle", productMessage) + const evidence = await runNetworkOracleFixture("block", "blocked", {}, false, true) + assert.deepEqual(evidence.result.transitions[0]?.observations.oracleFingerprints, [productFingerprint]) + assert.equal(evidence.result.findings[0]?.fingerprint, productFingerprint) + + const finding = await runNetworkOracleFixture("block", "blocked", { oraclePolicy: { policyBlocks: "finding" } }, false, true) + const fingerprints = finding.result.transitions[0]?.observations.oracleFingerprints ?? [] + const blockMessage = finding.result.transitions[0]?.observations.consoleErrors.find((message) => message.includes("ERR_BLOCKED_BY_CLIENT")) + assert(blockMessage) + assert.equal(fingerprints.length, 2) + assert(fingerprints.includes(productFingerprint)) + assert(fingerprints.includes(browserAdaptiveDigest("oracle", blockMessage))) + assert.equal(finding.result.transitions[0]?.observations.networkFailureSummary?.policyBlocks, 1) +}) + test("allow and record policies do not explain unexpected same-origin request failures", async () => { for (const mode of ["allow", "record"] as const) { const run = await runNetworkOracleFixture(mode, "unexpected") @@ -536,6 +576,35 @@ test("page exceptions remain findings alongside policy-aware network classificat assert.deepEqual(run.result.findings[0]?.replay.actions, run.result.findings[0]?.minimizedPath) }) +test("policy-block floods retain bounded deterministic evidence without stopping later findings", async () => { + const [first, second] = await runPolicyBlockFloodFixture() + for (const result of [first, second]) { + const floods = result.transitions.filter((transition) => transition.observations.networkFailureSummary?.total === 80) + assert.equal(floods.length, 3) + assert.equal(floods.reduce((total, transition) => total + (transition.observations.networkFailures?.length ?? 0), 0), 2, "retained failures are bounded across the artifact") + assert.deepEqual(floods.map((transition) => transition.observations.networkFailureSummary), [ + { total: 80, retained: 2, policyBlocks: 80, oracleFindings: 0, truncated: true }, + { total: 80, retained: 0, policyBlocks: 80, oracleFindings: 0, truncated: true }, + { total: 80, retained: 0, policyBlocks: 80, oracleFindings: 0, truncated: true }, + ]) + assert(floods.every((transition) => transition.observations.oracleFingerprints.length === 0)) + assert.equal(result.status, "findings") + assert.equal(result.summary.errors, 1, "policy blocks must not consume the product error budget") + assert.notEqual(result.summary.budgetExhausted, "maxArtifactBytes") + assert.equal(result.findings.length, 1) + assert.equal(result.findings[0]?.originalPath.length, 4) + assert.deepEqual(result.findings[0]?.minimizedPath, result.findings[0]?.originalPath) + assert.deepEqual(result.findings[0]?.replay.actions, result.findings[0]?.minimizedPath) + } + const stable = (result: typeof first) => ({ + status: result.status, + summary: result.summary, + transitions: result.transitions.map((transition) => ({ action: transition.action.id, observations: transition.observations, status: transition.status })), + findings: result.findings, + }) + assert.deepEqual(stable(second), stable(first)) +}) + test("loops, budgets, cancellation, frames, and partial evidence remain bounded", async () => { const loopFixture = `
External` const bounded = await runFixture(loopFixture, { @@ -651,7 +720,7 @@ async function runRoutedFixture(topology: ReturnType = {}) { +async function runNetworkOracleFixture(mode: "allow" | "block" | "record", behavior: "blocked" | "unexpected" | "mixed" | "exception", input: Record = {}, urlLessConsole = false, sameUrlProductError = false) { const server = createServer((request, response) => { const path = new URL(request.url ?? "/", "http://preview.invalid").pathname if (path === "/failed.png") { @@ -677,7 +746,12 @@ async function runNetworkOracleFixture(mode: "allow" | "block" | "record", behav const consoleMessages: Record[] = [] const errors: Record[] = [] const network: Record[] = [] - attachBrowserCaptureListeners({ captureConsole: true, captureErrors: true, captureNetwork: true, captureWebSocket: false, consoleMessages, errors, network, page }) + attachBrowserCaptureListeners({ captureConsole: !urlLessConsole && !sameUrlProductError, captureErrors: true, captureNetwork: true, captureWebSocket: false, consoleMessages, errors, network, page }) + if (urlLessConsole || sameUrlProductError) page.on("console", (message) => { + const location = message.location() + if (sameUrlProductError && message.text().includes("ERR_BLOCKED_BY_CLIENT")) consoleMessages.push({ type: "error", text: "same-URL product defect", location }) + consoleMessages.push({ type: message.type(), text: message.text(), ...(urlLessConsole ? {} : { location }) }) + }) await page.addInitScript("globalThis.__name = value => value") await page.goto(startUrl, { waitUntil: "load" }) const contract = browserAdaptiveExplorationContract({ @@ -699,6 +773,64 @@ async function runNetworkOracleFixture(mode: "allow" | "block" | "record", behav } } +async function runPolicyBlockFloodFixture() { + const server = createServer((_request, response) => { + response.setHeader("content-type", "text/html") + response.end(``) + }) + const startUrl = await listenLocalHttpServer(server) + const topology = browserPreviewTopology(["network-policy=block"], undefined, startUrl) + const run = async () => { + const browser = await chromium.launch({ headless: true }) + const context = await browser.newContext() + await routeBrowserPreviewContextNetwork(context, topology.networkPolicy, startUrl) + const page = await context.newPage() + const consoleMessages: Record[] = [] + const errors: Record[] = [] + const network: Record[] = [] + attachBrowserCaptureListeners({ captureConsole: true, captureErrors: true, captureNetwork: true, captureWebSocket: false, consoleMessages, errors, network, page }) + await page.addInitScript("globalThis.__name = value => value") + await page.goto(startUrl, { waitUntil: "load" }) + const contract = browserAdaptiveExplorationContract({ + seed: "policy-block-flood", + startUrl, + failOnFinding: false, + actionFamilies: ["click"], + budgets: { maxActions: 40, maxStates: 8, maxTransitions: 20, maxDurationMs: 30_000, maxArtifactBytes: 40_000, maxErrors: 2 }, + descriptorLimits: { maxPerState: 10, maxDiagnostics: 3, maxTextLength: 500 }, + stabilization: { pollIntervalMs: 25, quietWindowMs: 100, maxWaitMs: 1_500, maxMutationRecords: 100 }, + }) + try { + return await exploreAdaptiveBrowserStateMachine({ page, baseUrl: startUrl, contract, observations: { consoleMessages, errors, network }, navigationScope: topology.navigationScope, networkPolicy: topology.networkPolicy }) + } finally { + await browser.close() + } + } + try { + return [await run(), await run()] as const + } finally { + await closeHttpServer(server) + } +} + function stableAdaptiveEvidence(result: Awaited>) { return { states: result.states.map((state) => ({ digest: state.digest, url: state.url, descriptors: state.descriptors.map((descriptor) => descriptor.id) })),