Merge pull request #2030 from Automattic/feat/2025-adaptive-browser-exploration

Add adaptive browser state graph exploration

Author: chubes4

examples/recipes/adversarial-adaptive-browser.json

@@ -0,0 +1,77 @@
+{
+  "schema": "wp-codebox/workspace-recipe/v1",
+  "runtime": {
+    "backend": "wordpress"
+  },
+  "workflow": {
+    "steps": [
+      {
+        "command": "wordpress.run-php",
+        "args": ["code=update_option( 'wp_codebox_adaptive_fixture', 0 );"]
+      }
+    ]
+  },
+  "adversarialCampaigns": [
+    {
+      "schema": "wp-codebox/adversarial-recipe-campaign/v1",
+      "id": "adaptive-browser",
+      "seed": "adaptive-browser-v1",
+      "corpus": [
+        {
+          "id": "adaptive-homepage",
+          "actions": [
+            {
+              "type": "explore-browser"
+            }
+          ]
+        }
+      ],
+      "caseTemplates": [
+        {
+          "id": "explore-browser",
+          "phases": {
+            "action": [
+              {
+                "command": "wordpress.browser-actions",
+                "args": [
+                  "adaptive-exploration-json={\"schema\":\"wp-codebox/browser-adaptive-exploration/v1\",\"seed\":\"adaptive-browser-v1\",\"startUrl\":\"/\",\"budgets\":{\"maxActions\":24,\"maxStates\":16,\"maxTransitions\":32,\"maxDurationMs\":60000,\"maxArtifactBytes\":2097152,\"maxErrors\":10},\"actionFamilies\":[\"click\",\"fill\",\"select\",\"submit\",\"keyboard\",\"back\",\"reload\",\"repeat\",\"double-submit\"],\"resetPolicy\":{\"mode\":\"start-url\"}}",
+                  "capture=steps,console,errors,network"
+                ]
+              }
+            ]
+          }
+        }
+      ],
+      "mutators": ["sequence"],
+      "oracles": [
+        {
+          "id": "runtime-status",
+          "severity": "high"
+        }
+      ],
+      "concurrency": 1,
+      "budgets": {
+        "maxCases": 4,
+        "maxActionsPerCase": 4,
+        "maxInputBytes": 65536,
+        "maxCaseTimeMs": 90000,
+        "maxWallTimeMs": 360000,
+        "maxArtifactBytes": 10485760
+      },
+      "resetPolicy": {
+        "mode": "checkpoint-per-case",
+        "checkpointName": "adaptive-browser-baseline"
+      },
+      "requiredCapabilities": ["adversarial-campaign", "artifact-export", "command:wordpress.browser-actions"],
+      "novelty": {
+        "retainSignals": true
+      },
+      "shrinking": {
+        "enabled": true
+      },
+      "replay": {
+        "selection": "findings"
+      }
+    }
+  ]
+}

packages/cli/src/adversarial-recipe.ts

@@ -5,6 +5,7 @@ import { isAbsolute, join, relative } from "node:path"
 import {
   ADVERSARIAL_ORACLE_SCHEMA,
   adversarialCampaign,
+  adversarialFindingFingerprint,
   artifactManifestFile,
   refreshArtifactManifestFileSha256s,
   runAdversarialCampaign,
@@ -129,17 +130,31 @@ async function runRecipeAdversarialReplay(
     const started = Date.now()
     const observation = await execute(plan, controller.signal)
     const signals = [...new Set(observation.signals ?? [])].sort()
+    const replayMismatch = Boolean(replay.expectedStateDigest && observation.stateDigest !== replay.expectedStateDigest)
+    const replayFingerprint = adversarialFindingFingerprint({
+      oracleIds: observation.status === "passed" ? [] : ["runtime-status"],
+      status: observation.status,
+      diagnosticCodes: (observation.diagnostics ?? []).map((diagnostic) => diagnostic.code).sort(),
+      stateDigest: observation.stateDigest,
+      matrix: plan.matrix,
+    })
+    const fingerprintMismatch = Boolean(replay.expectedFingerprint && replayFingerprint !== replay.expectedFingerprint)
+    const reproduced = observation.status === "passed" && !replayMismatch && !fingerprintMismatch
     return {
       schema: "wp-codebox/adversarial-campaign-result/v1",
       campaignId: campaign.id,
       seed: replay.seed,
-      status: observation.status === "passed" ? "passed" : "findings",
-      summary: { generated: 1, executed: 1, retained: 1, findings: observation.status === "passed" ? 0 : 1, duplicates: 0, timedOut: observation.status === "timed-out" ? 1 : 0 },
+      status: reproduced ? "passed" : "findings",
+      summary: { generated: 1, executed: 1, retained: 1, findings: reproduced ? 0 : 1, duplicates: 0, timedOut: observation.status === "timed-out" ? 1 : 0 },
       corpus: [{ id: replay.caseId, actions: replay.actions, input: replay.input, signals }],
       findings: [],
       schedule: replay.schedule,
       noveltySignals: signals,
-      diagnostics: [{ code: "adversarial-replay-completed", message: `Replayed ${replay.caseId} through the recipe fuzz lifecycle.` }],
+      diagnostics: replayMismatch
+        ? [{ code: "adversarial-replay-state-mismatch", message: `Replay ${replay.caseId} did not reproduce the recorded state digest.` }]
+        : fingerprintMismatch
+          ? [{ code: "adversarial-replay-fingerprint-mismatch", message: `Replay ${replay.caseId} did not reproduce the recorded finding fingerprint.` }]
+          : [{ code: "adversarial-replay-completed", message: `Replayed ${replay.caseId} through the recipe fuzz lifecycle.` }],
       resourceUsage: { wallTimeMs: Date.now() - started, artifactBytes: (observation.artifacts ?? []).reduce((sum, artifact) => sum + (artifact.bytes ?? 0), 0) },
     }
   } finally {

packages/runtime-core/src/adversarial-campaign.ts

@@ -124,6 +124,8 @@ export interface AdversarialReplay {
   schedule: AdversarialScheduleEntry[]
   provenance?: Record<string, unknown>
   command: string
+  expectedFingerprint?: string
+  expectedStateDigest?: string
 }
 
 export interface AdversarialScheduleEntry {
@@ -239,8 +241,8 @@ export async function runAdversarialCampaign(campaignInput: AdversarialCampaign,
       const failedOracles = oracleResults.filter((result) => result.failed)
       if (failedOracles.length > 0 || observation.status !== "passed") {
         if (options.signal?.aborted) { incomplete = true; diagnostics.push({ code: "campaign-interrupted", message: "Campaign stopped before finding minimization after an interruption request." }); break }
-        const minimized = options.minimize === false ? normalizeCorpusEntry({ id: plan.caseId, actions: plan.actions, input: plan.input }) : await minimizeAdversarialCase(campaign, plan, failedOracles, options)
-        const fingerprint = adversarialFindingFingerprint({ oracleIds: failedOracles.map((result) => result.oracleId), status: observation.status, diagnosticCodes: (observation.diagnostics ?? []).map((item) => item.code).sort(), matrix: plan.matrix })
+        const fingerprint = observationFingerprint(plan, observation, failedOracles)
+        const minimized = options.minimize === false ? normalizeCorpusEntry({ id: plan.caseId, actions: plan.actions, input: plan.input }) : await minimizeAdversarialCase(campaign, plan, failedOracles, fingerprint, options)
         const existing = findings.get(fingerprint)
         if (existing) existing.duplicates += 1
         else findings.set(fingerprint, createFinding(campaign, plan, minimized, observation, failedOracles, schedule, options, fingerprint))
@@ -335,14 +337,15 @@ function mutateCorpusEntry(campaign: AdversarialCampaign, source: AdversarialCor
   return { id: `${source.id}-mutation-${iteration}`, caseId: `${campaign.id}-${iteration}`, corpusId: source.id, iteration, workerId, matrix, actions, input, mutation, metadata: source.metadata }
 }
 
-async function minimizeAdversarialCase(campaign: AdversarialCampaign, plan: AdversarialCasePlan, originalOracles: AdversarialOracleResult[], options: AdversarialCampaignRunnerOptions): Promise<AdversarialCorpusEntry> {
+async function minimizeAdversarialCase(campaign: AdversarialCampaign, plan: AdversarialCasePlan, originalOracles: AdversarialOracleResult[], expectedFingerprint: string, options: AdversarialCampaignRunnerOptions): Promise<AdversarialCorpusEntry> {
   const oracleIds = new Set(originalOracles.filter((item) => item.failed).map((item) => item.oracleId))
   const preserves = async (candidate: AdversarialCorpusEntry): Promise<boolean> => {
     if (options.signal?.aborted) return false
     const candidatePlan = { ...plan, actions: candidate.actions, input: candidate.input }
     const observation = await executeBoundedCase(campaign, candidatePlan, options)
     const oracleResults = options.evaluate ? await options.evaluate(candidatePlan, observation, campaign.oracles) : defaultOracleResults(observation)
-    return observation.status !== "passed" || oracleResults.some((item) => item.failed && (oracleIds.size === 0 || oracleIds.has(item.oracleId)))
+    const preservesOracle = observation.status !== "passed" || oracleResults.some((item) => item.failed && (oracleIds.size === 0 || oracleIds.has(item.oracleId)))
+    return preservesOracle && observationFingerprint(candidatePlan, observation, oracleResults.filter((item) => item.failed)) === expectedFingerprint
   }
   let actions = [...plan.actions]
   let chunk = Math.max(1, Math.floor(actions.length / 2))
@@ -373,7 +376,7 @@ function createFinding(campaign: AdversarialCampaign, plan: AdversarialCasePlan,
     status: observation.status,
     minimized,
     original: { id: plan.caseId, actions: plan.actions, input: plan.input },
-    replay: stripUndefined({ schema: ADVERSARIAL_REPLAY_SCHEMA, campaignId: campaign.id, seed: campaign.seed, caseId: plan.caseId, corpusId: plan.corpusId, workerId: plan.workerId, iteration: plan.iteration, matrix: plan.matrix, actions: minimized.actions, input: minimized.input, faultSchedule: campaign.faults, schedule: [...schedule], provenance: campaign.provenance, command }),
+    replay: stripUndefined({ schema: ADVERSARIAL_REPLAY_SCHEMA, campaignId: campaign.id, seed: campaign.seed, caseId: plan.caseId, corpusId: plan.corpusId, workerId: plan.workerId, iteration: plan.iteration, matrix: plan.matrix, actions: minimized.actions, input: minimized.input, faultSchedule: campaign.faults, schedule: [...schedule], provenance: campaign.provenance, command, expectedFingerprint: fingerprint, expectedStateDigest: observation.stateDigest }),
     diagnostics: observation.diagnostics ?? [],
     artifactRefs: observation.artifacts ?? [],
     secretScan: { status: "passed", redactions: 0 },
@@ -386,6 +389,16 @@ function defaultOracleResults(observation: AdversarialExecutionObservation): Adv
   return observation.status === "passed" ? [] : [{ oracleId: "runtime-status", failed: true, code: observation.status, message: observation.diagnostics?.[0]?.message ?? `Runtime status was ${observation.status}.` }]
 }
 
+function observationFingerprint(plan: AdversarialCasePlan, observation: AdversarialExecutionObservation, oracles: AdversarialOracleResult[]): string {
+  return adversarialFindingFingerprint({
+    oracleIds: oracles.filter((result) => result.failed).map((result) => result.oracleId).sort(),
+    status: observation.status,
+    diagnosticCodes: (observation.diagnostics ?? []).map((item) => item.code).sort(),
+    stateDigest: observation.stateDigest,
+    matrix: plan.matrix,
+  })
+}
+
 function normalizeBudgets(input: Partial<AdversarialResourceBudget> | undefined): AdversarialResourceBudget {
   return {
     maxCases: boundedInteger(input?.maxCases, 100, 1, 100_000),