fix: tighten adaptive failure correlation

Author: chubes4

packages/runtime-playground/src/browser-adaptive-explorer.ts

@@ -45,6 +45,11 @@ interface FrontierEntry {
   path: BrowserAdaptiveAction[]
 }
 
+interface AdaptiveNetworkFailureRetention {
+  remainingCount: number
+  remainingBytes: number
+}
+
 export async function exploreAdaptiveBrowserStateMachine({
   page,
   baseUrl,
@@ -79,6 +84,10 @@ export async function exploreAdaptiveBrowserStateMachine({
   let revisits = 0
   let keyboardActions = 0
   let exhausted: BrowserAdaptiveExplorationResult["summary"]["budgetExhausted"]
+  const networkFailureRetention: AdaptiveNetworkFailureRetention = {
+    remainingCount: Math.min(contract.budgets.maxErrors, contract.descriptorLimits.maxDiagnostics),
+    remainingBytes: Math.floor(contract.budgets.maxArtifactBytes / 8),
+  }
 
   const initial = await captureAdaptiveState(page, contract, 0, navigationScope)
   appendDiagnostics(diagnostics, initial.diagnostics, contract.descriptorLimits.maxDiagnostics)
@@ -144,7 +153,7 @@ export async function exploreAdaptiveBrowserStateMachine({
       const newConsoleRecords = consoleErrorRecords(observations.consoleMessages.slice(beforeConsole))
       const newConsoleErrors = newConsoleRecords.map(recordMessage)
       const newPageErrors = errorMessages(observations.errors.slice(beforeErrors))
-      const oracleEvidence = adaptiveOracleEvidence(newConsoleRecords, observations.network.slice(beforeNetwork), newPageErrors, contract, networkPolicy)
+      const oracleEvidence = adaptiveOracleEvidence(newConsoleRecords, observations.network.slice(beforeNetwork), newPageErrors, contract, networkPolicy, networkFailureRetention)
       const fingerprints = [...oracleEvidence.fingerprints]
       const existing = states.get(stabilized.state.digest)
       const newState = !existing
@@ -570,7 +579,7 @@ function consoleErrorRecords(records: object[]): Record<string, unknown>[] {
   return records.map((record) => record as Record<string, unknown>).filter((record) => record.type === "error" || record.level === "error").slice(0, 100)
 }
 
-function adaptiveOracleEvidence(consoleRecords: Record<string, unknown>[], networkRecords: object[], pageErrors: string[], contract: BrowserAdaptiveExplorationContract, networkPolicy?: BrowserPreviewNetworkPolicy): { fingerprints: string[]; networkFailures: BrowserAdaptiveNetworkFailure[]; networkFailureSummary?: BrowserAdaptiveTransition["observations"]["networkFailureSummary"]; errorCount: number } {
+function adaptiveOracleEvidence(consoleRecords: Record<string, unknown>[], networkRecords: object[], pageErrors: string[], contract: BrowserAdaptiveExplorationContract, networkPolicy?: BrowserPreviewNetworkPolicy, retention?: AdaptiveNetworkFailureRetention): { fingerprints: string[]; networkFailures: BrowserAdaptiveNetworkFailure[]; networkFailureSummary?: BrowserAdaptiveTransition["observations"]["networkFailureSummary"]; errorCount: number } {
   const failures = networkRecords.map((record) => record as Record<string, unknown>).filter((record) => record.type === "requestfailed")
   const classifiedFailures = failures.map((record): BrowserAdaptiveNetworkFailure & { expectedBlock: boolean } => {
     const url = typeof record.url === "string" ? record.url : ""
@@ -584,15 +593,15 @@ function adaptiveOracleEvidence(consoleRecords: Record<string, unknown>[], netwo
     const message = recordMessage(record)
     const locationUrl = objectRecord(record.location).url
     const match = classifiedFailures.findIndex((failure, index) => unmatchedFailures.has(index)
-      && (typeof locationUrl === "string" && locationUrl ? failure.url === locationUrl : failureTokenMatches(message, failure.failure)))
+      && (typeof locationUrl === "string" && locationUrl ? failure.url === locationUrl && failureTokenMatches(message, failure.failure) : failureTokenMatches(message, failure.failure)))
     if (match < 0) return [message]
     unmatchedFailures.delete(match)
     const failure = classifiedFailures[match]!
     return failure.expectedBlock && contract.oraclePolicy.policyBlocks === "evidence" ? [] : [message]
   })
   const networkMessages = classifiedFailures.filter((failure, index) => unmatchedFailures.has(index) && failure.oracleFinding).map(networkFailureOracleMessage)
   const messages = [...consoleMessages, ...pageErrors, ...networkMessages]
-  const networkFailureSummary = classifiedFailures.length > 0 ? boundedNetworkFailureEvidence(classifiedFailures, contract) : undefined
+  const networkFailureSummary = classifiedFailures.length > 0 ? boundedNetworkFailureEvidence(classifiedFailures, contract, retention) : undefined
   return {
     fingerprints: [...new Set(messages.map((message) => browserAdaptiveDigest("oracle", message)))].sort(),
     networkFailures: networkFailureSummary?.failures ?? [],
@@ -601,14 +610,19 @@ function adaptiveOracleEvidence(consoleRecords: Record<string, unknown>[], netwo
   }
 }
 
-function boundedNetworkFailureEvidence(failures: Array<BrowserAdaptiveNetworkFailure & { expectedBlock: boolean }>, contract: BrowserAdaptiveExplorationContract): { failures: BrowserAdaptiveNetworkFailure[]; summary: NonNullable<BrowserAdaptiveTransition["observations"]["networkFailureSummary"]> } {
+function boundedNetworkFailureEvidence(failures: Array<BrowserAdaptiveNetworkFailure & { expectedBlock: boolean }>, contract: BrowserAdaptiveExplorationContract, cumulativeRetention?: AdaptiveNetworkFailureRetention): { failures: BrowserAdaptiveNetworkFailure[]; summary: NonNullable<BrowserAdaptiveTransition["observations"]["networkFailureSummary"]> } {
   const ordered = failures.map(({ expectedBlock: _expectedBlock, ...failure }) => failure).sort((left, right) => stableJson(left).localeCompare(stableJson(right)))
-  const maximum = Math.min(contract.budgets.maxErrors, contract.descriptorLimits.maxDiagnostics)
-  const byteBudget = Math.floor(contract.budgets.maxArtifactBytes / 8)
+  const retention = cumulativeRetention ?? {
+    remainingCount: Math.min(contract.budgets.maxErrors, contract.descriptorLimits.maxDiagnostics),
+    remainingBytes: Math.floor(contract.budgets.maxArtifactBytes / 8),
+  }
   const retained: BrowserAdaptiveNetworkFailure[] = []
-  for (const failure of ordered.slice(0, maximum)) {
-    if (Buffer.byteLength(stableJson([...retained, failure])) > byteBudget) break
+  for (const failure of ordered.slice(0, retention.remainingCount)) {
+    const bytes = Buffer.byteLength(stableJson(failure)) + 1
+    if (bytes > retention.remainingBytes) break
     retained.push(failure)
+    retention.remainingCount -= 1
+    retention.remainingBytes -= bytes
   }
   return {
     failures: retained,

tests/browser-adaptive-exploration.test.ts

@@ -529,6 +529,23 @@ test("URL-less policy-block console records correlate before promotion and prese
   assert.equal(mixed.result.transitions[0]?.observations.networkFailureSummary?.oracleFindings, 1)
 })
 
+test("same-URL product errors require a matching failure token before policy correlation", async () => {
+  const productMessage = "same-URL product defect"
+  const productFingerprint = browserAdaptiveDigest("oracle", productMessage)
+  const evidence = await runNetworkOracleFixture("block", "blocked", {}, false, true)
+  assert.deepEqual(evidence.result.transitions[0]?.observations.oracleFingerprints, [productFingerprint])
+  assert.equal(evidence.result.findings[0]?.fingerprint, productFingerprint)
+
+  const finding = await runNetworkOracleFixture("block", "blocked", { oraclePolicy: { policyBlocks: "finding" } }, false, true)
+  const fingerprints = finding.result.transitions[0]?.observations.oracleFingerprints ?? []
+  const blockMessage = finding.result.transitions[0]?.observations.consoleErrors.find((message) => message.includes("ERR_BLOCKED_BY_CLIENT"))
+  assert(blockMessage)
+  assert.equal(fingerprints.length, 2)
+  assert(fingerprints.includes(productFingerprint))
+  assert(fingerprints.includes(browserAdaptiveDigest("oracle", blockMessage)))
+  assert.equal(finding.result.transitions[0]?.observations.networkFailureSummary?.policyBlocks, 1)
+})
+
 test("allow and record policies do not explain unexpected same-origin request failures", async () => {
   for (const mode of ["allow", "record"] as const) {
     const run = await runNetworkOracleFixture(mode, "unexpected")
@@ -562,16 +579,20 @@ test("page exceptions remain findings alongside policy-aware network classificat
 test("policy-block floods retain bounded deterministic evidence without stopping later findings", async () => {
   const [first, second] = await runPolicyBlockFloodFixture()
   for (const result of [first, second]) {
-    const flood = result.transitions.find((transition) => transition.observations.networkFailureSummary?.total === 80)
-    assert(flood)
-    assert.deepEqual(flood.observations.networkFailureSummary, { total: 80, retained: 2, policyBlocks: 80, oracleFindings: 0, truncated: true })
-    assert.equal(flood.observations.networkFailures?.length, 2)
-    assert.deepEqual(flood.observations.oracleFingerprints, [])
+    const floods = result.transitions.filter((transition) => transition.observations.networkFailureSummary?.total === 80)
+    assert.equal(floods.length, 3)
+    assert.equal(floods.reduce((total, transition) => total + (transition.observations.networkFailures?.length ?? 0), 0), 2, "retained failures are bounded across the artifact")
+    assert.deepEqual(floods.map((transition) => transition.observations.networkFailureSummary), [
+      { total: 80, retained: 2, policyBlocks: 80, oracleFindings: 0, truncated: true },
+      { total: 80, retained: 0, policyBlocks: 80, oracleFindings: 0, truncated: true },
+      { total: 80, retained: 0, policyBlocks: 80, oracleFindings: 0, truncated: true },
+    ])
+    assert(floods.every((transition) => transition.observations.oracleFingerprints.length === 0))
     assert.equal(result.status, "findings")
     assert.equal(result.summary.errors, 1, "policy blocks must not consume the product error budget")
     assert.notEqual(result.summary.budgetExhausted, "maxArtifactBytes")
     assert.equal(result.findings.length, 1)
-    assert.equal(result.findings[0]?.originalPath.length, 2)
+    assert.equal(result.findings[0]?.originalPath.length, 4)
     assert.deepEqual(result.findings[0]?.minimizedPath, result.findings[0]?.originalPath)
     assert.deepEqual(result.findings[0]?.replay.actions, result.findings[0]?.minimizedPath)
   }
@@ -699,7 +720,7 @@ async function runRoutedFixture(topology: ReturnType<typeof browserPreviewTopolo
   }
 }
 
-async function runNetworkOracleFixture(mode: "allow" | "block" | "record", behavior: "blocked" | "unexpected" | "mixed" | "exception", input: Record<string, unknown> = {}, urlLessConsole = false) {
+async function runNetworkOracleFixture(mode: "allow" | "block" | "record", behavior: "blocked" | "unexpected" | "mixed" | "exception", input: Record<string, unknown> = {}, urlLessConsole = false, sameUrlProductError = false) {
   const server = createServer((request, response) => {
     const path = new URL(request.url ?? "/", "http://preview.invalid").pathname
     if (path === "/failed.png") {
@@ -725,8 +746,12 @@ async function runNetworkOracleFixture(mode: "allow" | "block" | "record", behav
   const consoleMessages: Record<string, unknown>[] = []
   const errors: Record<string, unknown>[] = []
   const network: Record<string, unknown>[] = []
-  attachBrowserCaptureListeners({ captureConsole: !urlLessConsole, captureErrors: true, captureNetwork: true, captureWebSocket: false, consoleMessages, errors, network, page })
-  if (urlLessConsole) page.on("console", (message) => consoleMessages.push({ type: message.type(), text: message.text() }))
+  attachBrowserCaptureListeners({ captureConsole: !urlLessConsole && !sameUrlProductError, captureErrors: true, captureNetwork: true, captureWebSocket: false, consoleMessages, errors, network, page })
+  if (urlLessConsole || sameUrlProductError) page.on("console", (message) => {
+    const location = message.location()
+    if (sameUrlProductError && message.text().includes("ERR_BLOCKED_BY_CLIENT")) consoleMessages.push({ type: "error", text: "same-URL product defect", location })
+    consoleMessages.push({ type: message.type(), text: message.text(), ...(urlLessConsole ? {} : { location }) })
+  })
   await page.addInitScript("globalThis.__name = value => value")
   await page.goto(startUrl, { waitUntil: "load" })
   const contract = browserAdaptiveExplorationContract({
@@ -753,11 +778,16 @@ async function runPolicyBlockFloodFixture() {
     response.setHeader("content-type", "text/html")
     response.end(`<!doctype html><style>button{display:block;width:180px;height:30px}</style><button id="trigger">Load assets</button><script>
       document.querySelector('#trigger').addEventListener('click', () => {
+        const stage = Number(document.querySelector('#trigger').dataset.stage || '0') + 1;
+        document.querySelector('#trigger').dataset.stage = String(stage);
+        document.querySelector('#trigger').textContent = 'Load assets stage ' + String(stage);
         for (let index = 0; index < 80; index += 1) {
           const image = document.createElement('img');
-          image.src = 'http://assets.example.invalid/tile-' + String(index).padStart(3, '0') + '.png';
+          image.src = 'http://assets.example.invalid/tile-' + String(stage) + '-' + String(index).padStart(3, '0') + '.png';
           document.body.append(image);
         }
+        if (stage < 3) return;
+        document.querySelector('#trigger').disabled = true;
         const fail = document.createElement('button');
         fail.id = 'fail';
         fail.textContent = 'Reveal defect';
@@ -784,7 +814,7 @@ async function runPolicyBlockFloodFixture() {
       startUrl,
       failOnFinding: false,
       actionFamilies: ["click"],
-      budgets: { maxActions: 20, maxStates: 6, maxTransitions: 10, maxDurationMs: 20_000, maxArtifactBytes: 40_000, maxErrors: 2 },
+      budgets: { maxActions: 40, maxStates: 8, maxTransitions: 20, maxDurationMs: 30_000, maxArtifactBytes: 40_000, maxErrors: 2 },
       descriptorLimits: { maxPerState: 10, maxDiagnostics: 3, maxTextLength: 500 },
       stabilization: { pollIntervalMs: 25, quietWindowMs: 100, maxWaitMs: 1_500, maxMutationRecords: 100 },
     })