fix: bound adaptive policy block evidence

chubes4 · chubes4 · commit 9d921ba74858 · 2026-07-26T17:41:48.000Z
diff --git a/packages/runtime-core/src/browser-adaptive-exploration.ts b/packages/runtime-core/src/browser-adaptive-exploration.ts
@@ -86,6 +86,13 @@ export interface BrowserAdaptiveTransition {
     loadingAfter: number
     oracleFingerprints: string[]
     networkFailures?: BrowserAdaptiveNetworkFailure[]
+    networkFailureSummary?: {
+      total: number
+      retained: number
+      policyBlocks: number
+      oracleFindings: number
+      truncated: boolean
+    }
     accessibilityFindingFingerprints?: string[]
   }
   status: "ok" | "revisited" | "rejected" | "error" | "cancelled"
diff --git a/packages/runtime-playground/src/browser-adaptive-explorer.ts b/packages/runtime-playground/src/browser-adaptive-explorer.ts
@@ -189,6 +189,7 @@ export async function exploreAdaptiveBrowserStateMachine({
           loadingAfter: stabilized.loading,
           oracleFingerprints: fingerprints,
           ...(oracleEvidence.networkFailures.length > 0 ? { networkFailures: oracleEvidence.networkFailures } : {}),
+          ...(oracleEvidence.networkFailureSummary ? { networkFailureSummary: oracleEvidence.networkFailureSummary } : {}),
           ...(accessibilityFingerprints.length > 0 ? { accessibilityFindingFingerprints: accessibilityFingerprints } : {}),
         },
         status: actionError ? "error" : scopeRejection ? "rejected" : newState ? "ok" : "revisited",
@@ -569,36 +570,67 @@ function consoleErrorRecords(records: object[]): Record<string, unknown>[] {
   return records.map((record) => record as Record<string, unknown>).filter((record) => record.type === "error" || record.level === "error").slice(0, 100)
 }
 
-function adaptiveOracleEvidence(consoleRecords: Record<string, unknown>[], networkRecords: object[], pageErrors: string[], contract: BrowserAdaptiveExplorationContract, networkPolicy?: BrowserPreviewNetworkPolicy): { fingerprints: string[]; networkFailures: BrowserAdaptiveNetworkFailure[]; errorCount: number } {
+function adaptiveOracleEvidence(consoleRecords: Record<string, unknown>[], networkRecords: object[], pageErrors: string[], contract: BrowserAdaptiveExplorationContract, networkPolicy?: BrowserPreviewNetworkPolicy): { fingerprints: string[]; networkFailures: BrowserAdaptiveNetworkFailure[]; networkFailureSummary?: BrowserAdaptiveTransition["observations"]["networkFailureSummary"]; errorCount: number } {
   const failures = networkRecords.map((record) => record as Record<string, unknown>).filter((record) => record.type === "requestfailed")
-  const networkFailures = failures.map((record): BrowserAdaptiveNetworkFailure => {
+  const classifiedFailures = failures.map((record): BrowserAdaptiveNetworkFailure & { expectedBlock: boolean } => {
     const url = typeof record.url === "string" ? record.url : ""
     const decision = networkPolicy ? browserPreviewNetworkDecision(url, networkPolicy) : { url, urlClassification: "invalid" as const, policyDecision: "unknown" as const, policyReason: "network-policy-unavailable" }
     const failure = networkFailureMessage(record)
     const expectedBlock = decision.policyDecision === "blocked" && /ERR_BLOCKED_BY_CLIENT|blockedbyclient/i.test(failure)
-    return { ...decision, ...(failure ? { failure } : {}), oracleFinding: !expectedBlock || contract.oraclePolicy.policyBlocks === "finding" }
+    return { ...decision, ...(failure ? { failure } : {}), oracleFinding: !expectedBlock || contract.oraclePolicy.policyBlocks === "finding", expectedBlock }
   })
-  const expectedBlockUrls = networkFailures.filter((failure) => !failure.oracleFinding).map((failure) => failure.url)
-  const findingConsoleRecords = consoleRecords.filter((record) => {
-    if (!/ERR_BLOCKED_BY_CLIENT/i.test(recordMessage(record))) return true
-    const location = objectRecord(record.location)
-    const match = typeof location.url === "string" && location.url
-      ? expectedBlockUrls.indexOf(location.url)
-      : expectedBlockUrls.length > 0 ? 0 : -1
-    if (match < 0) return true
-    expectedBlockUrls.splice(match, 1)
-    return false
+  const unmatchedFailures = new Set(classifiedFailures.map((_failure, index) => index))
+  const consoleMessages = consoleRecords.flatMap((record) => {
+    const message = recordMessage(record)
+    const locationUrl = objectRecord(record.location).url
+    const match = classifiedFailures.findIndex((failure, index) => unmatchedFailures.has(index)
+      && (typeof locationUrl === "string" && locationUrl ? failure.url === locationUrl : failureTokenMatches(message, failure.failure)))
+    if (match < 0) return [message]
+    unmatchedFailures.delete(match)
+    const failure = classifiedFailures[match]!
+    return failure.expectedBlock && contract.oraclePolicy.policyBlocks === "evidence" ? [] : [message]
   })
-  const consoleUrls = new Set(findingConsoleRecords.map((record) => objectRecord(record.location).url).filter((url): url is string => typeof url === "string"))
-  const networkMessages = networkFailures.filter((failure) => failure.oracleFinding && !consoleUrls.has(failure.url)).map((failure) => stableJson({ type: "requestfailed", url: failure.url, failure: failure.failure, urlClassification: failure.urlClassification, policyDecision: failure.policyDecision, policyReason: failure.policyReason }))
-  const messages = [...findingConsoleRecords.map(recordMessage), ...pageErrors, ...networkMessages]
+  const networkMessages = classifiedFailures.filter((failure, index) => unmatchedFailures.has(index) && failure.oracleFinding).map(networkFailureOracleMessage)
+  const messages = [...consoleMessages, ...pageErrors, ...networkMessages]
+  const networkFailureSummary = classifiedFailures.length > 0 ? boundedNetworkFailureEvidence(classifiedFailures, contract) : undefined
   return {
     fingerprints: [...new Set(messages.map((message) => browserAdaptiveDigest("oracle", message)))].sort(),
-    networkFailures,
-    errorCount: findingConsoleRecords.length + pageErrors.length + networkMessages.length,
+    networkFailures: networkFailureSummary?.failures ?? [],
+    networkFailureSummary: networkFailureSummary?.summary,
+    errorCount: consoleMessages.length + pageErrors.length + networkMessages.length,
   }
 }
 
+function boundedNetworkFailureEvidence(failures: Array<BrowserAdaptiveNetworkFailure & { expectedBlock: boolean }>, contract: BrowserAdaptiveExplorationContract): { failures: BrowserAdaptiveNetworkFailure[]; summary: NonNullable<BrowserAdaptiveTransition["observations"]["networkFailureSummary"]> } {
+  const ordered = failures.map(({ expectedBlock: _expectedBlock, ...failure }) => failure).sort((left, right) => stableJson(left).localeCompare(stableJson(right)))
+  const maximum = Math.min(contract.budgets.maxErrors, contract.descriptorLimits.maxDiagnostics)
+  const byteBudget = Math.floor(contract.budgets.maxArtifactBytes / 8)
+  const retained: BrowserAdaptiveNetworkFailure[] = []
+  for (const failure of ordered.slice(0, maximum)) {
+    if (Buffer.byteLength(stableJson([...retained, failure])) > byteBudget) break
+    retained.push(failure)
+  }
+  return {
+    failures: retained,
+    summary: {
+      total: failures.length,
+      retained: retained.length,
+      policyBlocks: failures.filter((failure) => failure.expectedBlock).length,
+      oracleFindings: failures.filter((failure) => failure.oracleFinding).length,
+      truncated: retained.length < failures.length,
+    },
+  }
+}
+
+function failureTokenMatches(message: string, failure: string | undefined): boolean {
+  const token = failure?.match(/ERR_[A-Z0-9_]+/i)?.[0]
+  return Boolean(token && message.toUpperCase().includes(token.toUpperCase()))
+}
+
+function networkFailureOracleMessage(failure: BrowserAdaptiveNetworkFailure): string {
+  return stableJson({ type: "requestfailed", url: failure.url, failure: failure.failure, urlClassification: failure.urlClassification, policyDecision: failure.policyDecision, policyReason: failure.policyReason })
+}
+
 function networkFailureMessage(record: Record<string, unknown>): string {
   const failure = objectRecord(record.failure)
   return typeof failure.errorText === "string" ? failure.errorText : typeof record.failure === "string" ? record.failure : ""
diff --git a/tests/browser-adaptive-exploration.test.ts b/tests/browser-adaptive-exploration.test.ts
@@ -5,6 +5,7 @@ import { chromium, type Page } from "playwright"
 
 import {
   BROWSER_ADAPTIVE_EXPLORATION_SCHEMA,
+  browserAdaptiveDigest,
   browserAdaptiveExplorationContract,
   planBrowserAdaptiveStateActions,
 } from "../packages/runtime-core/src/browser-adaptive-exploration.js"
@@ -506,6 +507,28 @@ test("callers can opt policy blocks back into deterministic findings and replay"
   assert.equal(finding.replay.expectedFingerprint, finding.fingerprint)
 })
 
+test("URL-less policy-block console records correlate before promotion and preserve canonical fingerprints", async () => {
+  const evidenceOnly = await runNetworkOracleFixture("block", "blocked", {}, true)
+  assert.equal(evidenceOnly.result.findings.length, 0)
+  assert.deepEqual(evidenceOnly.result.transitions[0]?.observations.oracleFingerprints, [])
+  assert.equal(evidenceOnly.result.transitions[0]?.observations.networkFailureSummary?.policyBlocks, 1)
+
+  const input = { oraclePolicy: { policyBlocks: "finding" } }
+  const attributed = await runNetworkOracleFixture("block", "blocked", input)
+  const urlLess = await runNetworkOracleFixture("block", "blocked", input, true)
+  const message = urlLess.result.transitions[0]?.observations.consoleErrors[0]
+  assert(message)
+  assert.equal(urlLess.result.transitions[0]?.observations.oracleFingerprints.length, 1)
+  assert.equal(urlLess.result.findings[0]?.fingerprint, browserAdaptiveDigest("oracle", message))
+  assert.equal(urlLess.result.findings[0]?.fingerprint, attributed.result.findings[0]?.fingerprint, "removing console location must not change the historical console fingerprint")
+
+  const mixed = await runNetworkOracleFixture("block", "mixed", {}, true)
+  assert.equal(mixed.result.findings.length, 1)
+  assert.equal(mixed.result.transitions[0]?.observations.oracleFingerprints.length, 1)
+  assert.equal(mixed.result.transitions[0]?.observations.networkFailureSummary?.policyBlocks, 1)
+  assert.equal(mixed.result.transitions[0]?.observations.networkFailureSummary?.oracleFindings, 1)
+})
+
 test("allow and record policies do not explain unexpected same-origin request failures", async () => {
   for (const mode of ["allow", "record"] as const) {
     const run = await runNetworkOracleFixture(mode, "unexpected")
@@ -536,6 +559,31 @@ test("page exceptions remain findings alongside policy-aware network classificat
   assert.deepEqual(run.result.findings[0]?.replay.actions, run.result.findings[0]?.minimizedPath)
 })
 
+test("policy-block floods retain bounded deterministic evidence without stopping later findings", async () => {
+  const [first, second] = await runPolicyBlockFloodFixture()
+  for (const result of [first, second]) {
+    const flood = result.transitions.find((transition) => transition.observations.networkFailureSummary?.total === 80)
+    assert(flood)
+    assert.deepEqual(flood.observations.networkFailureSummary, { total: 80, retained: 2, policyBlocks: 80, oracleFindings: 0, truncated: true })
+    assert.equal(flood.observations.networkFailures?.length, 2)
+    assert.deepEqual(flood.observations.oracleFingerprints, [])
+    assert.equal(result.status, "findings")
+    assert.equal(result.summary.errors, 1, "policy blocks must not consume the product error budget")
+    assert.notEqual(result.summary.budgetExhausted, "maxArtifactBytes")
+    assert.equal(result.findings.length, 1)
+    assert.equal(result.findings[0]?.originalPath.length, 2)
+    assert.deepEqual(result.findings[0]?.minimizedPath, result.findings[0]?.originalPath)
+    assert.deepEqual(result.findings[0]?.replay.actions, result.findings[0]?.minimizedPath)
+  }
+  const stable = (result: typeof first) => ({
+    status: result.status,
+    summary: result.summary,
+    transitions: result.transitions.map((transition) => ({ action: transition.action.id, observations: transition.observations, status: transition.status })),
+    findings: result.findings,
+  })
+  assert.deepEqual(stable(second), stable(first))
+})
+
 test("loops, budgets, cancellation, frames, and partial evidence remain bounded", async () => {
   const loopFixture = `<!doctype html><style>button,input,a,iframe { display:block;width:100px;height:30px }</style><button id="toggle">Toggle</button><form id="first"><input name="query"></form><form id="second"><input name="query"></form><a id="external" href="https://example.test/outside">External</a><iframe id="same" srcdoc="<style>button{width:100px;height:30px}</style><button id='framed'>Framed</button>"></iframe><script>toggle.onclick=()=>document.body.classList.toggle('on')</script>`
   const bounded = await runFixture(loopFixture, {
@@ -651,7 +699,7 @@ async function runRoutedFixture(topology: ReturnType<typeof browserPreviewTopolo
   }
 }
 
-async function runNetworkOracleFixture(mode: "allow" | "block" | "record", behavior: "blocked" | "unexpected" | "mixed" | "exception", input: Record<string, unknown> = {}) {
+async function runNetworkOracleFixture(mode: "allow" | "block" | "record", behavior: "blocked" | "unexpected" | "mixed" | "exception", input: Record<string, unknown> = {}, urlLessConsole = false) {
   const server = createServer((request, response) => {
     const path = new URL(request.url ?? "/", "http://preview.invalid").pathname
     if (path === "/failed.png") {
@@ -677,7 +725,8 @@ async function runNetworkOracleFixture(mode: "allow" | "block" | "record", behav
   const consoleMessages: Record<string, unknown>[] = []
   const errors: Record<string, unknown>[] = []
   const network: Record<string, unknown>[] = []
-  attachBrowserCaptureListeners({ captureConsole: true, captureErrors: true, captureNetwork: true, captureWebSocket: false, consoleMessages, errors, network, page })
+  attachBrowserCaptureListeners({ captureConsole: !urlLessConsole, captureErrors: true, captureNetwork: true, captureWebSocket: false, consoleMessages, errors, network, page })
+  if (urlLessConsole) page.on("console", (message) => consoleMessages.push({ type: message.type(), text: message.text() }))
   await page.addInitScript("globalThis.__name = value => value")
   await page.goto(startUrl, { waitUntil: "load" })
   const contract = browserAdaptiveExplorationContract({
@@ -699,6 +748,59 @@ async function runNetworkOracleFixture(mode: "allow" | "block" | "record", behav
   }
 }
 
+async function runPolicyBlockFloodFixture() {
+  const server = createServer((_request, response) => {
+    response.setHeader("content-type", "text/html")
+    response.end(`<!doctype html><style>button{display:block;width:180px;height:30px}</style><button id="trigger">Load assets</button><script>
+      document.querySelector('#trigger').addEventListener('click', () => {
+        for (let index = 0; index < 80; index += 1) {
+          const image = document.createElement('img');
+          image.src = 'http://assets.example.invalid/tile-' + String(index).padStart(3, '0') + '.png';
+          document.body.append(image);
+        }
+        const fail = document.createElement('button');
+        fail.id = 'fail';
+        fail.textContent = 'Reveal defect';
+        fail.addEventListener('click', () => setTimeout(() => { throw new Error('post-flood defect'); }, 0));
+        document.body.append(fail);
+      });
+    </script>`)
+  })
+  const startUrl = await listenLocalHttpServer(server)
+  const topology = browserPreviewTopology(["network-policy=block"], undefined, startUrl)
+  const run = async () => {
+    const browser = await chromium.launch({ headless: true })
+    const context = await browser.newContext()
+    await routeBrowserPreviewContextNetwork(context, topology.networkPolicy, startUrl)
+    const page = await context.newPage()
+    const consoleMessages: Record<string, unknown>[] = []
+    const errors: Record<string, unknown>[] = []
+    const network: Record<string, unknown>[] = []
+    attachBrowserCaptureListeners({ captureConsole: true, captureErrors: true, captureNetwork: true, captureWebSocket: false, consoleMessages, errors, network, page })
+    await page.addInitScript("globalThis.__name = value => value")
+    await page.goto(startUrl, { waitUntil: "load" })
+    const contract = browserAdaptiveExplorationContract({
+      seed: "policy-block-flood",
+      startUrl,
+      failOnFinding: false,
+      actionFamilies: ["click"],
+      budgets: { maxActions: 20, maxStates: 6, maxTransitions: 10, maxDurationMs: 20_000, maxArtifactBytes: 40_000, maxErrors: 2 },
+      descriptorLimits: { maxPerState: 10, maxDiagnostics: 3, maxTextLength: 500 },
+      stabilization: { pollIntervalMs: 25, quietWindowMs: 100, maxWaitMs: 1_500, maxMutationRecords: 100 },
+    })
+    try {
+      return await exploreAdaptiveBrowserStateMachine({ page, baseUrl: startUrl, contract, observations: { consoleMessages, errors, network }, navigationScope: topology.navigationScope, networkPolicy: topology.networkPolicy })
+    } finally {
+      await browser.close()
+    }
+  }
+  try {
+    return [await run(), await run()] as const
+  } finally {
+    await closeHttpServer(server)
+  }
+}
+
 function stableAdaptiveEvidence(result: Awaited<ReturnType<typeof runRoutedFixture>>) {
   return {
     states: result.states.map((state) => ({ digest: state.digest, url: state.url, descriptors: state.descriptors.map((descriptor) => descriptor.id) })),
