diff --git a/examples/recipes/adversarial-adaptive-browser.json b/examples/recipes/adversarial-adaptive-browser.json new file mode 100644 index 000000000..08e68ba50 --- /dev/null +++ b/examples/recipes/adversarial-adaptive-browser.json @@ -0,0 +1,77 @@ +{ + "schema": "wp-codebox/workspace-recipe/v1", + "runtime": { + "backend": "wordpress" + }, + "workflow": { + "steps": [ + { + "command": "wordpress.run-php", + "args": ["code=update_option( 'wp_codebox_adaptive_fixture', 0 );"] + } + ] + }, + "adversarialCampaigns": [ + { + "schema": "wp-codebox/adversarial-recipe-campaign/v1", + "id": "adaptive-browser", + "seed": "adaptive-browser-v1", + "corpus": [ + { + "id": "adaptive-homepage", + "actions": [ + { + "type": "explore-browser" + } + ] + } + ], + "caseTemplates": [ + { + "id": "explore-browser", + "phases": { + "action": [ + { + "command": "wordpress.browser-actions", + "args": [ + "adaptive-exploration-json={\"schema\":\"wp-codebox/browser-adaptive-exploration/v1\",\"seed\":\"adaptive-browser-v1\",\"startUrl\":\"/\",\"budgets\":{\"maxActions\":24,\"maxStates\":16,\"maxTransitions\":32,\"maxDurationMs\":60000,\"maxArtifactBytes\":2097152,\"maxErrors\":10},\"actionFamilies\":[\"click\",\"fill\",\"select\",\"submit\",\"keyboard\",\"back\",\"reload\",\"repeat\",\"double-submit\"],\"resetPolicy\":{\"mode\":\"start-url\"}}", + "capture=steps,console,errors,network" + ] + } + ] + } + } + ], + "mutators": ["sequence"], + "oracles": [ + { + "id": "runtime-status", + "severity": "high" + } + ], + "concurrency": 1, + "budgets": { + "maxCases": 4, + "maxActionsPerCase": 4, + "maxInputBytes": 65536, + "maxCaseTimeMs": 90000, + "maxWallTimeMs": 360000, + "maxArtifactBytes": 10485760 + }, + "resetPolicy": { + "mode": "checkpoint-per-case", + "checkpointName": "adaptive-browser-baseline" + }, + "requiredCapabilities": ["adversarial-campaign", "artifact-export", "command:wordpress.browser-actions"], + "novelty": { + "retainSignals": true + }, + "shrinking": { + "enabled": true + }, + "replay": { + "selection": "findings" + } + } + ] +} diff --git a/packages/cli/src/adversarial-recipe.ts b/packages/cli/src/adversarial-recipe.ts index b27de44a2..486853860 100644 --- a/packages/cli/src/adversarial-recipe.ts +++ b/packages/cli/src/adversarial-recipe.ts @@ -5,6 +5,7 @@ import { isAbsolute, join, relative } from "node:path" import { ADVERSARIAL_ORACLE_SCHEMA, adversarialCampaign, + adversarialFindingFingerprint, artifactManifestFile, refreshArtifactManifestFileSha256s, runAdversarialCampaign, @@ -129,17 +130,31 @@ async function runRecipeAdversarialReplay( const started = Date.now() const observation = await execute(plan, controller.signal) const signals = [...new Set(observation.signals ?? [])].sort() + const replayMismatch = Boolean(replay.expectedStateDigest && observation.stateDigest !== replay.expectedStateDigest) + const replayFingerprint = adversarialFindingFingerprint({ + oracleIds: observation.status === "passed" ? [] : ["runtime-status"], + status: observation.status, + diagnosticCodes: (observation.diagnostics ?? []).map((diagnostic) => diagnostic.code).sort(), + stateDigest: observation.stateDigest, + matrix: plan.matrix, + }) + const fingerprintMismatch = Boolean(replay.expectedFingerprint && replayFingerprint !== replay.expectedFingerprint) + const reproduced = observation.status === "passed" && !replayMismatch && !fingerprintMismatch return { schema: "wp-codebox/adversarial-campaign-result/v1", campaignId: campaign.id, seed: replay.seed, - status: observation.status === "passed" ? "passed" : "findings", - summary: { generated: 1, executed: 1, retained: 1, findings: observation.status === "passed" ? 0 : 1, duplicates: 0, timedOut: observation.status === "timed-out" ? 1 : 0 }, + status: reproduced ? "passed" : "findings", + summary: { generated: 1, executed: 1, retained: 1, findings: reproduced ? 0 : 1, duplicates: 0, timedOut: observation.status === "timed-out" ? 1 : 0 }, corpus: [{ id: replay.caseId, actions: replay.actions, input: replay.input, signals }], findings: [], schedule: replay.schedule, noveltySignals: signals, - diagnostics: [{ code: "adversarial-replay-completed", message: `Replayed ${replay.caseId} through the recipe fuzz lifecycle.` }], + diagnostics: replayMismatch + ? [{ code: "adversarial-replay-state-mismatch", message: `Replay ${replay.caseId} did not reproduce the recorded state digest.` }] + : fingerprintMismatch + ? [{ code: "adversarial-replay-fingerprint-mismatch", message: `Replay ${replay.caseId} did not reproduce the recorded finding fingerprint.` }] + : [{ code: "adversarial-replay-completed", message: `Replayed ${replay.caseId} through the recipe fuzz lifecycle.` }], resourceUsage: { wallTimeMs: Date.now() - started, artifactBytes: (observation.artifacts ?? []).reduce((sum, artifact) => sum + (artifact.bytes ?? 0), 0) }, } } finally { diff --git a/packages/runtime-core/src/adversarial-campaign.ts b/packages/runtime-core/src/adversarial-campaign.ts index a989f4620..4d7a3be15 100644 --- a/packages/runtime-core/src/adversarial-campaign.ts +++ b/packages/runtime-core/src/adversarial-campaign.ts @@ -124,6 +124,8 @@ export interface AdversarialReplay { schedule: AdversarialScheduleEntry[] provenance?: Record command: string + expectedFingerprint?: string + expectedStateDigest?: string } export interface AdversarialScheduleEntry { @@ -239,8 +241,8 @@ export async function runAdversarialCampaign(campaignInput: AdversarialCampaign, const failedOracles = oracleResults.filter((result) => result.failed) if (failedOracles.length > 0 || observation.status !== "passed") { if (options.signal?.aborted) { incomplete = true; diagnostics.push({ code: "campaign-interrupted", message: "Campaign stopped before finding minimization after an interruption request." }); break } - const minimized = options.minimize === false ? normalizeCorpusEntry({ id: plan.caseId, actions: plan.actions, input: plan.input }) : await minimizeAdversarialCase(campaign, plan, failedOracles, options) - const fingerprint = adversarialFindingFingerprint({ oracleIds: failedOracles.map((result) => result.oracleId), status: observation.status, diagnosticCodes: (observation.diagnostics ?? []).map((item) => item.code).sort(), matrix: plan.matrix }) + const fingerprint = observationFingerprint(plan, observation, failedOracles) + const minimized = options.minimize === false ? normalizeCorpusEntry({ id: plan.caseId, actions: plan.actions, input: plan.input }) : await minimizeAdversarialCase(campaign, plan, failedOracles, fingerprint, options) const existing = findings.get(fingerprint) if (existing) existing.duplicates += 1 else findings.set(fingerprint, createFinding(campaign, plan, minimized, observation, failedOracles, schedule, options, fingerprint)) @@ -335,14 +337,15 @@ function mutateCorpusEntry(campaign: AdversarialCampaign, source: AdversarialCor return { id: `${source.id}-mutation-${iteration}`, caseId: `${campaign.id}-${iteration}`, corpusId: source.id, iteration, workerId, matrix, actions, input, mutation, metadata: source.metadata } } -async function minimizeAdversarialCase(campaign: AdversarialCampaign, plan: AdversarialCasePlan, originalOracles: AdversarialOracleResult[], options: AdversarialCampaignRunnerOptions): Promise { +async function minimizeAdversarialCase(campaign: AdversarialCampaign, plan: AdversarialCasePlan, originalOracles: AdversarialOracleResult[], expectedFingerprint: string, options: AdversarialCampaignRunnerOptions): Promise { const oracleIds = new Set(originalOracles.filter((item) => item.failed).map((item) => item.oracleId)) const preserves = async (candidate: AdversarialCorpusEntry): Promise => { if (options.signal?.aborted) return false const candidatePlan = { ...plan, actions: candidate.actions, input: candidate.input } const observation = await executeBoundedCase(campaign, candidatePlan, options) const oracleResults = options.evaluate ? await options.evaluate(candidatePlan, observation, campaign.oracles) : defaultOracleResults(observation) - return observation.status !== "passed" || oracleResults.some((item) => item.failed && (oracleIds.size === 0 || oracleIds.has(item.oracleId))) + const preservesOracle = observation.status !== "passed" || oracleResults.some((item) => item.failed && (oracleIds.size === 0 || oracleIds.has(item.oracleId))) + return preservesOracle && observationFingerprint(candidatePlan, observation, oracleResults.filter((item) => item.failed)) === expectedFingerprint } let actions = [...plan.actions] let chunk = Math.max(1, Math.floor(actions.length / 2)) @@ -373,7 +376,7 @@ function createFinding(campaign: AdversarialCampaign, plan: AdversarialCasePlan, status: observation.status, minimized, original: { id: plan.caseId, actions: plan.actions, input: plan.input }, - replay: stripUndefined({ schema: ADVERSARIAL_REPLAY_SCHEMA, campaignId: campaign.id, seed: campaign.seed, caseId: plan.caseId, corpusId: plan.corpusId, workerId: plan.workerId, iteration: plan.iteration, matrix: plan.matrix, actions: minimized.actions, input: minimized.input, faultSchedule: campaign.faults, schedule: [...schedule], provenance: campaign.provenance, command }), + replay: stripUndefined({ schema: ADVERSARIAL_REPLAY_SCHEMA, campaignId: campaign.id, seed: campaign.seed, caseId: plan.caseId, corpusId: plan.corpusId, workerId: plan.workerId, iteration: plan.iteration, matrix: plan.matrix, actions: minimized.actions, input: minimized.input, faultSchedule: campaign.faults, schedule: [...schedule], provenance: campaign.provenance, command, expectedFingerprint: fingerprint, expectedStateDigest: observation.stateDigest }), diagnostics: observation.diagnostics ?? [], artifactRefs: observation.artifacts ?? [], secretScan: { status: "passed", redactions: 0 }, @@ -386,6 +389,16 @@ function defaultOracleResults(observation: AdversarialExecutionObservation): Adv return observation.status === "passed" ? [] : [{ oracleId: "runtime-status", failed: true, code: observation.status, message: observation.diagnostics?.[0]?.message ?? `Runtime status was ${observation.status}.` }] } +function observationFingerprint(plan: AdversarialCasePlan, observation: AdversarialExecutionObservation, oracles: AdversarialOracleResult[]): string { + return adversarialFindingFingerprint({ + oracleIds: oracles.filter((result) => result.failed).map((result) => result.oracleId).sort(), + status: observation.status, + diagnosticCodes: (observation.diagnostics ?? []).map((item) => item.code).sort(), + stateDigest: observation.stateDigest, + matrix: plan.matrix, + }) +} + function normalizeBudgets(input: Partial | undefined): AdversarialResourceBudget { return { maxCases: boundedInteger(input?.maxCases, 100, 1, 100_000), diff --git a/packages/runtime-core/src/browser-adaptive-exploration.ts b/packages/runtime-core/src/browser-adaptive-exploration.ts new file mode 100644 index 000000000..df9ed8f94 --- /dev/null +++ b/packages/runtime-core/src/browser-adaptive-exploration.ts @@ -0,0 +1,242 @@ +import { createHash } from "node:crypto" + +import type { BrowserActionCorpusDescriptor, BrowserInteractionStep, BrowserRandomWalkContext } from "./browser-interaction.js" +import { isPlainObject, stableJson, stripUndefined } from "./object-utils.js" + +export const BROWSER_ADAPTIVE_EXPLORATION_SCHEMA = "wp-codebox/browser-adaptive-exploration/v1" as const +export const BROWSER_ADAPTIVE_EXPLORATION_ARTIFACT_SCHEMA = "wp-codebox/browser-adaptive-exploration-artifact/v1" as const + +export const BROWSER_ADAPTIVE_ACTION_FAMILIES = ["click", "fill", "select", "submit", "keyboard", "back", "reload", "repeat", "double-submit"] as const +export type BrowserAdaptiveActionFamily = typeof BROWSER_ADAPTIVE_ACTION_FAMILIES[number] + +export interface BrowserAdaptiveExplorationContract { + schema: typeof BROWSER_ADAPTIVE_EXPLORATION_SCHEMA + context: BrowserRandomWalkContext + seed: string + startUrl: string + budgets: { + maxActions: number + maxStates: number + maxTransitions: number + maxDurationMs: number + maxArtifactBytes: number + maxErrors: number + } + actionFamilies: BrowserAdaptiveActionFamily[] + resetPolicy: { mode: "none" | "start-url" } + revisitPolicy: { maxStateVisits: number; maxActionVisits: number } + descriptorLimits: { maxPerState: number; maxDiagnostics: number; maxTextLength: number } + stabilization: { pollIntervalMs: number; quietWindowMs: number; maxWaitMs: number; maxMutationRecords: number } + failOnFinding: boolean + metadata?: Record +} + +export interface BrowserAdaptiveFrameIdentity { + id: string + parentId?: string + url: string + scope: "document" | "same-origin-frame" +} + +export interface BrowserAdaptiveState { + digest: string + url: string + historyLength: number + historyStateDigest: string + descriptorDigest: string + descriptors: BrowserActionCorpusDescriptor[] + frames: BrowserAdaptiveFrameIdentity[] + visits: number + depth: number + loadingIndicators: number +} + +export interface BrowserAdaptiveAction { + id: string + family: BrowserAdaptiveActionFamily + frameId: string + descriptorId?: string + descriptor?: BrowserActionCorpusDescriptor + steps: BrowserInteractionStep[] + input?: string | string[] +} + +export interface BrowserAdaptiveTransition { + id: string + sourceDigest: string + destinationDigest?: string + action: BrowserAdaptiveAction + sourceUrl: string + destinationUrl: string + history: { before: number; after: number; beforeStateDigest: string; afterStateDigest: string } + timing: { durationMs: number; stabilizationMs: number; polls: number } + novelty: { newState: boolean; newDescriptors: number; mutationRecords: number; mutationEvidenceTruncated: boolean } + observations: { + networkEvents: number + consoleErrors: string[] + pageErrors: string[] + loadingBefore: number + loadingAfter: number + oracleFingerprints: string[] + } + status: "ok" | "revisited" | "rejected" | "error" | "cancelled" + diagnostic?: { code: string; message: string } +} + +export interface BrowserAdaptiveFinding { + fingerprint: string + stateDigest?: string + transitionId: string + originalPath: BrowserAdaptiveAction[] + minimizedPath: BrowserAdaptiveAction[] + replay: { + schema: typeof BROWSER_ADAPTIVE_EXPLORATION_SCHEMA + seed: string + startUrl: string + expectedFingerprint: string + expectedStateDigest?: string + actions: BrowserAdaptiveAction[] + resetPolicy: BrowserAdaptiveExplorationContract["resetPolicy"] + } +} + +export interface BrowserAdaptiveExplorationResult { + schema: typeof BROWSER_ADAPTIVE_EXPLORATION_SCHEMA + status: "completed" | "findings" | "incomplete" + seed: string + startUrl: string + states: BrowserAdaptiveState[] + transitions: BrowserAdaptiveTransition[] + findings: BrowserAdaptiveFinding[] + diagnostics: Array<{ code: string; message: string; metadata?: Record }> + summary: { + actions: number + states: number + transitions: number + revisits: number + errors: number + findings: number + budgetExhausted?: keyof BrowserAdaptiveExplorationContract["budgets"] | "cancelled" | "frontier" + } + replay: { schema: typeof BROWSER_ADAPTIVE_EXPLORATION_SCHEMA; seed: string; startUrl: string; contract: BrowserAdaptiveExplorationContract } +} + +export interface BrowserAdaptiveExplorationArtifact { + schema: typeof BROWSER_ADAPTIVE_EXPLORATION_ARTIFACT_SCHEMA + contract: BrowserAdaptiveExplorationContract + result: BrowserAdaptiveExplorationResult + capturedAt: string +} + +export function browserAdaptiveExplorationContract(input: Record): BrowserAdaptiveExplorationContract { + const context: BrowserRandomWalkContext = input.context === "admin" || input.context === "editor" ? input.context : "browser" + const budgets = object(input.budgets) + const revisit = object(input.revisitPolicy ?? input.revisit_policy) + const descriptors = object(input.descriptorLimits ?? input.descriptor_limits) + const stabilization = object(input.stabilization) + const reset = object(input.resetPolicy ?? input.reset_policy) + const families = Array.isArray(input.actionFamilies ?? input.action_families) + ? (input.actionFamilies ?? input.action_families) as unknown[] + : [] + const actionFamilies = [...new Set(families.filter((value): value is BrowserAdaptiveActionFamily => (BROWSER_ADAPTIVE_ACTION_FAMILIES as readonly unknown[]).includes(value)))] + const startUrl = string(input.startUrl ?? input.start_url) ?? (context === "admin" ? "/wp-admin/" : context === "editor" ? "/wp-admin/post-new.php" : "/") + return stripUndefined({ + schema: BROWSER_ADAPTIVE_EXPLORATION_SCHEMA, + context, + seed: string(input.seed) ?? "browser-adaptive-exploration", + startUrl, + budgets: { + maxActions: integer(budgets.maxActions ?? budgets.max_actions, 32, 1, 500), + maxStates: integer(budgets.maxStates ?? budgets.max_states, 24, 1, 250), + maxTransitions: integer(budgets.maxTransitions ?? budgets.max_transitions, 64, 1, 1_000), + maxDurationMs: integer(budgets.maxDurationMs ?? budgets.max_duration_ms, 120_000, 100, 3_600_000), + maxArtifactBytes: integer(budgets.maxArtifactBytes ?? budgets.max_artifact_bytes, 5 * 1_048_576, 1_024, 100 * 1_048_576), + maxErrors: integer(budgets.maxErrors ?? budgets.max_errors, 20, 1, 1_000), + }, + actionFamilies: actionFamilies.length > 0 ? actionFamilies : [...BROWSER_ADAPTIVE_ACTION_FAMILIES], + resetPolicy: { mode: reset.mode === "none" ? "none" as const : "start-url" as const }, + revisitPolicy: { + maxStateVisits: integer(revisit.maxStateVisits ?? revisit.max_state_visits, 2, 1, 20), + maxActionVisits: integer(revisit.maxActionVisits ?? revisit.max_action_visits, 1, 1, 20), + }, + descriptorLimits: { + maxPerState: integer(descriptors.maxPerState ?? descriptors.max_per_state, 80, 1, 500), + maxDiagnostics: integer(descriptors.maxDiagnostics ?? descriptors.max_diagnostics, 20, 1, 200), + maxTextLength: integer(descriptors.maxTextLength ?? descriptors.max_text_length, 2_000, 64, 20_000), + }, + stabilization: { + pollIntervalMs: integer(stabilization.pollIntervalMs ?? stabilization.poll_interval_ms, 50, 10, 1_000), + quietWindowMs: integer(stabilization.quietWindowMs ?? stabilization.quiet_window_ms, 150, 20, 10_000), + maxWaitMs: integer(stabilization.maxWaitMs ?? stabilization.max_wait_ms, 3_000, 50, 60_000), + maxMutationRecords: integer(stabilization.maxMutationRecords ?? stabilization.max_mutation_records, 100, 1, 5_000), + }, + failOnFinding: input.failOnFinding !== false && input.fail_on_finding !== false, + metadata: isPlainObject(input.metadata) ? input.metadata : undefined, + }) +} + +export function browserAdaptiveDigest(kind: "state" | "descriptors" | "oracle" | "action", value: unknown): string { + return createHash("sha256").update(`wp-codebox/browser-adaptive-${kind}/v1\n`).update(stableJson(value)).digest("hex") +} + +export function orderBrowserAdaptiveActions(actions: readonly BrowserAdaptiveAction[], seed: string, stateDigest: string): BrowserAdaptiveAction[] { + return [...actions].sort((left, right) => browserAdaptiveDigest("action", `${seed}:${stateDigest}:${left.id}`).localeCompare(browserAdaptiveDigest("action", `${seed}:${stateDigest}:${right.id}`)) || left.id.localeCompare(right.id)) +} + +export function planBrowserAdaptiveStateActions(state: BrowserAdaptiveState, contract: BrowserAdaptiveExplorationContract): BrowserAdaptiveAction[] { + const actions: BrowserAdaptiveAction[] = [] + const add = (family: BrowserAdaptiveActionFamily, descriptor: BrowserActionCorpusDescriptor | undefined, steps: BrowserInteractionStep[], input?: string | string[]) => { + if (!contract.actionFamilies.includes(family)) return + const frameId = descriptor?.frameId ?? "document" + const id = `${family}:${frameId}:${descriptor?.id ?? state.digest}` + actions.push({ id, family, frameId, ...(descriptor ? { descriptorId: descriptor.id } : {}), steps, ...(input !== undefined ? { input } : {}) }) + } + for (const descriptor of state.descriptors) { + if (descriptor.disabled || descriptor.readonly) continue + const selector = descriptor.selector + if (descriptor.kind === "input" || descriptor.kind === "textarea") { + const value = generatedValue(contract.seed, descriptor) + add("fill", descriptor, [{ kind: "fill", selector, value }], value) + add("keyboard", descriptor, [{ kind: "press", selector, key: "Enter" }]) + if (descriptor.formId) add("submit", descriptor, [{ kind: "fill", selector, value }, { kind: "press", selector, key: "Enter" }], value) + add("repeat", descriptor, [{ kind: "fill", selector, value }, { kind: "fill", selector, value }], value) + } else if (descriptor.kind === "select") { + const values = descriptor.optionValues?.filter(Boolean) ?? [] + if (values.length > 0) { + const value = values[parseInt(browserAdaptiveDigest("action", `${contract.seed}:${descriptor.id}`).slice(0, 8), 16) % values.length] as string + add("select", descriptor, [{ kind: "select", selector, value }], value) + } + } else { + add("click", descriptor, [{ kind: "click", selector }]) + add("repeat", descriptor, [{ kind: "click", selector }, { kind: "click", selector }]) + if (descriptor.type === "submit" && descriptor.formId) { + add("submit", descriptor, [{ kind: "click", selector }]) + add("double-submit", descriptor, [{ kind: "click", selector }, { kind: "click", selector }]) + } + } + } + add("back", undefined, []) + add("reload", undefined, []) + return actions +} + +function object(value: unknown): Record { + return isPlainObject(value) ? value : {} +} + +function string(value: unknown): string | undefined { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined +} + +function integer(value: unknown, fallback: number, minimum: number, maximum: number): number { + const numeric = typeof value === "number" ? value : Number(value) + return Number.isFinite(numeric) ? Math.max(minimum, Math.min(maximum, Math.floor(numeric))) : fallback +} + +function generatedValue(seed: string, descriptor: BrowserActionCorpusDescriptor): string { + const suffix = browserAdaptiveDigest("action", `${seed}:${descriptor.frameId}:${descriptor.id}`).slice(0, 10) + if (descriptor.type === "email") return `adaptive-${suffix}@example.test` + if (descriptor.type === "number") return String(parseInt(suffix.slice(0, 6), 16) % 1_000) + if (descriptor.type === "url") return `https://example.test/${suffix}` + return `adaptive-${suffix}` +} diff --git a/packages/runtime-core/src/browser-interaction.ts b/packages/runtime-core/src/browser-interaction.ts index 9ccf6c0ba..8a685abd3 100644 --- a/packages/runtime-core/src/browser-interaction.ts +++ b/packages/runtime-core/src/browser-interaction.ts @@ -160,6 +160,8 @@ export interface BrowserActionCorpusDescriptor { optionValues?: string[] disabled?: boolean readonly?: boolean + /** Adaptive exploration frame identity. Omitted for the main document and one-shot corpus discovery. */ + frameId?: string } export interface BrowserActionCorpusContract { diff --git a/packages/runtime-core/src/command-registry.ts b/packages/runtime-core/src/command-registry.ts index 19514f9ba..76bb374e0 100644 --- a/packages/runtime-core/src/command-registry.ts +++ b/packages/runtime-core/src/command-registry.ts @@ -133,7 +133,7 @@ const browserProbeValidation: CommandValidationDescriptor = { const browserActionsValidation: CommandValidationDescriptor = { requiredAnyArgs: [ - { names: ["steps-json", "url"], code: "missing-steps", message: "wordpress.browser-actions requires steps-json= or url=." }, + { names: ["steps-json", "url", "adaptive-exploration-json"], code: "missing-steps", message: "wordpress.browser-actions requires steps-json=, url=, or adaptive-exploration-json=." }, ], argRules: [ { name: "step-timeout", kind: "duration", code: "invalid-duration", message: "wordpress.browser-actions step-timeout must look like 500ms or 2s." }, @@ -1177,6 +1177,7 @@ export const commandRegistry = [ { name: "url", description: "Initial preview path or absolute URL to visit when the script omits an initial navigate step.", format: "path or URL" }, { name: "steps-json", description: "Ordered interaction script: navigate, click, fill, type, press, drag, hover, select, waitFor, evaluate, expect, screenshot, and capture steps. waitFor and screenshot steps support generic painted-readiness waits: painted, frame-painted:, and frame-url-painted:.", format: "JSON array (inline or @)" }, { name: "action-corpus-json", description: "Optional wp-codebox/browser-action-corpus/v1 object. The runtime loads the start URL, discovers visible links, buttons, inputs, textareas, and selects, creates deterministic seeded fill/click/select steps from stable descriptors, and writes replayable corpus artifacts.", format: "JSON object" }, + { name: "adaptive-exploration-json", description: "Optional standalone wp-codebox/browser-adaptive-exploration/v1 mode. Repeatedly stabilizes and rediscovers unique controls while building a bounded deterministic state graph with replay and minimized finding evidence; cannot be combined with url, steps-json, or action-corpus-json.", format: "JSON object" }, { name: "step-timeout", description: "Per-step timeout applied to each interaction step.", format: "duration, e.g. 5s or 500ms" }, { name: "timeout", description: "Total-script timeout bounding the whole interaction run.", format: "duration, e.g. 30s or 1500ms" }, { name: "auth", description: "Optional in-memory browser authentication mode. Use wordpress-admin to bootstrap WordPress admin cookies from PHP without writing token-bearing storage-state artifacts.", format: "wordpress-admin" }, @@ -1185,7 +1186,7 @@ export const commandRegistry = [ { name: "capture", description: "Comma-separated artifacts to capture after interactions.", format: "steps,console,errors,html,network,screenshot,dom-snapshot" }, { name: "max-dom-snapshot-elements", description: "Maximum visible elements captured in each screenshot sidecar DOM/style snapshot; defaults to 160.", format: "positive integer" }, ], - outputShape: "JSON summary plus files/browser/steps.jsonl, action-summary.json (with assertions pass/fail), optional action-corpus.json replay artifacts, named screenshots, sidecar DOM/style snapshots, and optional console/errors/network/html/screenshot artifacts.", + outputShape: "JSON summary plus files/browser/steps.jsonl, action-summary.json (with assertions pass/fail), optional action-corpus.json or adaptive-exploration.json replay artifacts, named screenshots, sidecar DOM/style snapshots, and optional console/errors/network/html/screenshot artifacts.", policyRequirement: "Runtime policy commands must include wordpress.browser-actions. The evaluate step additionally requires wordpress.browser-actions.evaluate.", validation: browserActionsValidation, recipe: true, diff --git a/packages/runtime-core/src/contracts.ts b/packages/runtime-core/src/contracts.ts index 57dce0ef5..b09149656 100644 --- a/packages/runtime-core/src/contracts.ts +++ b/packages/runtime-core/src/contracts.ts @@ -1,6 +1,7 @@ /** Inspectable Codebox contract metadata for CLI and orchestrator consumers. */ export * from "./browser-probe-contract.js" export * from "./browser-multi-actor-scenario-contracts.js" +export * from "./browser-adaptive-exploration.js" export * from "./command-registry.js" export * from "./fuzz-fixture-plan-contracts.js" export * from "./fuzz-coverage-plan-contracts.js" diff --git a/packages/runtime-core/src/index.ts b/packages/runtime-core/src/index.ts index d3beb9db3..805f21401 100644 --- a/packages/runtime-core/src/index.ts +++ b/packages/runtime-core/src/index.ts @@ -48,6 +48,7 @@ export * from "./command-diagnostics.js" export * from "./command-agent-run.js" export * from "./php-worker-runner.js" export * from "./browser-interaction.js" +export * from "./browser-adaptive-exploration.js" export * from "./browser-multi-actor-scenario-contracts.js" export * from "./browser-probe-contract.js" export * from "./browser-playground-session-run.js" diff --git a/packages/runtime-core/src/public.ts b/packages/runtime-core/src/public.ts index 139acab4d..a095f69a2 100644 --- a/packages/runtime-core/src/public.ts +++ b/packages/runtime-core/src/public.ts @@ -24,6 +24,7 @@ export * from "./artifact-test-results.js" export * from "./browser-artifact-lifecycle.js" export * from "./browser-callback-contracts.js" export * from "./browser-interaction.js" +export * from "./browser-adaptive-exploration.js" export * from "./browser-multi-actor-scenario-contracts.js" export * from "./browser-probe-contract.js" export * from "./browser-playground-session-run.js" diff --git a/packages/runtime-playground/src/browser-action-discovery.ts b/packages/runtime-playground/src/browser-action-discovery.ts new file mode 100644 index 000000000..4c73ee4a0 --- /dev/null +++ b/packages/runtime-playground/src/browser-action-discovery.ts @@ -0,0 +1,145 @@ +import type { BrowserActionCorpusDescriptor } from "@automattic/wp-codebox-core" +import type { Frame, Page } from "playwright" + +export async function discoverBrowserActionCorpusDescriptors(page: Page | Frame): Promise<{ + descriptors: BrowserActionCorpusDescriptor[] + diagnostics: Array<{ code: string; message: string; metadata?: Record }> +}> { + return await page.evaluate(() => { + const MAX_REJECTION_DIAGNOSTICS = 20 + const descriptors: BrowserActionCorpusDescriptor[] = [] + const rejected: Array<{ kind: string; tag: string; label: string }> = [] + const cssEscape = (value: string) => { + const escapeFn = (globalThis as typeof globalThis & { CSS?: { escape?: (raw: string) => string } }).CSS?.escape + return escapeFn ? escapeFn(value) : value.replace(/[^a-zA-Z0-9_-]/g, "\\$&") + } + const text = (value: string | null | undefined) => (value || "").replace(/\s+/g, " ").trim().slice(0, 120) + const attributeSelector = (name: string, value: string) => `[${name}=${cssEscape(value)}]` + const uniquelySelects = (selector: string, element: Element) => { + try { + const matches = document.querySelectorAll(selector) + return matches.length === 1 && matches[0] === element + } catch { + return false + } + } + const selectorFor = (element: Element): string | undefined => { + const tag = element.tagName.toLowerCase() + const candidates: string[] = [] + const seen = new Set() + const addCandidate = (selector: string) => { + if (!seen.has(selector)) { + seen.add(selector) + candidates.push(selector) + } + } + const id = element.getAttribute("id") + if (id) addCandidate(`#${cssEscape(id)}`) + + const attributes = new Map(["aria-label", "aria-labelledby", "name", "type", "value", "title", "href", "role"].flatMap((name) => { + const value = element.getAttribute(name) + return value ? [[name, value] as const] : [] + })) + const addAttributeCombination = (...names: string[]) => { + if (names.every((name) => attributes.has(name))) { + addCandidate(`${tag}${names.map((name) => attributeSelector(name, attributes.get(name)!)).join("")}`) + } + } + for (const names of [ + ["aria-label"], ["aria-labelledby"], + ["name", "type", "value"], ["name", "type"], ["name", "value"], ["name"], + ["href"], ["title"], + ["role", "type", "value"], ["role", "type"], ["role"], + ["type", "value"], ["value"], ["type"], + ]) { + addAttributeCombination(...names) + } + + const form = (element as HTMLInputElement).form + const formId = form?.getAttribute("id") + if (form && formId) { + const formSelector = `#${cssEscape(formId)}` + if (uniquelySelects(formSelector, form)) { + for (const candidate of [...candidates]) addCandidate(`${formSelector} ${candidate}`) + } + } + for (const candidate of candidates) { + if (uniquelySelects(candidate, element)) return candidate + } + + const parts: string[] = [] + let current: Element | null = element + while (current) { + let part = current.tagName.toLowerCase() + const parent: Element | null = current.parentElement + if (parent) { + const sameTagSiblings = Array.from(parent.children).filter((child) => child.tagName === current?.tagName) + if (sameTagSiblings.length > 1) part += `:nth-of-type(${sameTagSiblings.indexOf(current) + 1})` + } + parts.unshift(part) + const candidate = parts.join(" > ") + if (uniquelySelects(candidate, element)) return candidate + current = parent + } + return undefined + } + const labelFor = (element: Element) => { + const labelledBy = element.getAttribute("aria-labelledby") + if (labelledBy) { + const labelled = labelledBy.split(/\s+/).map((id) => document.getElementById(id)?.textContent).filter(Boolean).join(" ") + if (text(labelled)) return text(labelled) + } + const aria = text(element.getAttribute("aria-label")) + if (aria) return aria + const id = element.getAttribute("id") + const label = id ? document.querySelector(`label[for="${cssEscape(id)}"]`) : null + if (label && text(label.textContent)) return text(label.textContent) + return text(element.textContent) + } + const descriptorId = (kind: string, selector: string, element: Element) => `${kind}:${selector}:${element.getAttribute("name") || ""}:${labelFor(element)}` + const visible = (element: Element) => { + const htmlElement = element as HTMLElement + const style = window.getComputedStyle(htmlElement) + const rect = htmlElement.getBoundingClientRect() + return rect.width > 0 && rect.height > 0 && style.visibility !== "hidden" && style.display !== "none" + } + document.querySelectorAll("a[href], button, input, textarea, select").forEach((element) => { + if (!visible(element)) return + const tag = element.tagName.toLowerCase() + const input = element as HTMLInputElement + const kind = tag === "a" ? "link" : tag === "button" ? "button" : tag === "textarea" ? "textarea" : tag === "select" ? "select" : "input" + const selector = selectorFor(element) + if (!selector) { + rejected.push({ kind, tag, label: labelFor(element) }) + return + } + descriptors.push({ + id: descriptorId(kind, selector, element), + kind, + selector, + label: labelFor(element), + name: element.getAttribute("name") || undefined, + role: element.getAttribute("role") || undefined, + type: input.type || element.getAttribute("type") || undefined, + formId: (element as HTMLInputElement).form?.id || undefined, + href: tag === "a" ? (element as HTMLAnchorElement).href : undefined, + disabled: Boolean((element as HTMLButtonElement).disabled), + readonly: Boolean(input.readOnly), + optionValues: tag === "select" ? Array.from((element as HTMLSelectElement).options).map((option) => option.value).filter(Boolean) : undefined, + }) + }) + const diagnostics: Array<{ code: string; message: string; metadata?: Record }> = rejected.slice(0, MAX_REJECTION_DIAGNOSTICS).map((item) => ({ + code: "browser_action_corpus_selector_not_unique", + message: "An actionable control was rejected because discovery could not produce a selector resolving uniquely to that element.", + metadata: item, + })) + if (rejected.length > MAX_REJECTION_DIAGNOSTICS) { + diagnostics[MAX_REJECTION_DIAGNOSTICS - 1] = { + code: "browser_action_corpus_selector_rejections_truncated", + message: "Additional actionable controls without unique selectors were omitted from diagnostics.", + metadata: { rejected: rejected.length, retainedDiagnostics: MAX_REJECTION_DIAGNOSTICS - 1 }, + } + } + return { descriptors, diagnostics } + }) +} diff --git a/packages/runtime-playground/src/browser-actions-runner.ts b/packages/runtime-playground/src/browser-actions-runner.ts index 21903b631..c176ac744 100644 --- a/packages/runtime-playground/src/browser-actions-runner.ts +++ b/packages/runtime-playground/src/browser-actions-runner.ts @@ -1,5 +1,5 @@ import { readFile } from "node:fs/promises" -import { BROWSER_ACTION_CORPUS_SCHEMA, BROWSER_MULTI_ACTOR_SCENARIO_SCHEMA, BROWSER_TOOL_VERIFIER_RESULT_SCHEMA, HostToolRegistry, assertRuntimeCommandAllowed, browserActionCorpusArtifact, browserActionCorpusContract, browserInteractionScriptUsesEvaluate, browserToolVerifierInputSummary, createHostToolRegistry, executeHostTool, resolveCommandPath, validateBrowserInteractionScript, type BrowserActionCorpusArtifact, type BrowserActionCorpusContract, type BrowserActionCorpusDescriptor, type BrowserInteractionStep, type BrowserMultiActorScenario, type BrowserToolVerifierResult, type ExecutionSpec, type HostToolDefinition, type JsonValue, type RuntimeCreateSpec } from "@automattic/wp-codebox-core" +import { BROWSER_ACTION_CORPUS_SCHEMA, BROWSER_ADAPTIVE_EXPLORATION_SCHEMA, BROWSER_MULTI_ACTOR_SCENARIO_SCHEMA, BROWSER_TOOL_VERIFIER_RESULT_SCHEMA, HostToolRegistry, assertRuntimeCommandAllowed, browserActionCorpusArtifact, browserActionCorpusContract, browserAdaptiveExplorationContract, browserInteractionScriptUsesEvaluate, browserToolVerifierInputSummary, createHostToolRegistry, executeHostTool, resolveCommandPath, validateBrowserInteractionScript, type BrowserActionCorpusArtifact, type BrowserActionCorpusContract, type BrowserAdaptiveExplorationArtifact, type BrowserAdaptiveExplorationContract, type BrowserInteractionStep, type BrowserMultiActorScenario, type BrowserToolVerifierResult, type ExecutionSpec, type HostToolDefinition, type JsonValue, type RuntimeCreateSpec } from "@automattic/wp-codebox-core" import { now, sha256 } from "@automattic/wp-codebox-core/internals" import { browserInteractionStepsFromArgs, browserStepTimeoutMs, durationStringMs, sanitizeScreenshotName } from "./browser-actions.js" import { BrowserArtifactSession } from "./browser-artifact-session.js" @@ -21,6 +21,10 @@ import { argValue, commaListArg, durationArg, viewportArg } from "./commands.js" import type { PlaygroundRunResponse } from "./playground-command-errors.js" import type { PlaygroundCliServer } from "./preview-server.js" import type { Page } from "playwright" +import { discoverBrowserActionCorpusDescriptors } from "./browser-action-discovery.js" +import { exploreAdaptiveBrowserStateMachine } from "./browser-adaptive-explorer.js" + +export { discoverBrowserActionCorpusDescriptors } from "./browser-action-discovery.js" const BROWSER_STEP_DEFAULT_TIMEOUT_MS = 15_000 const BROWSER_SCRIPT_DEFAULT_TIMEOUT_MS = 120_000 @@ -41,6 +45,7 @@ export interface BrowserActionsRunPlan { storageStateImport?: BrowserStorageStateImport maxDomSnapshotElements: number actionCorpus?: BrowserActionCorpusContract + adaptiveExploration?: BrowserAdaptiveExplorationContract } interface BrowserRunPlan { @@ -70,9 +75,12 @@ export async function runBrowserActionsCommand({ const args = spec.args ?? [] const runPlan = plan ?? await browserActionsRunPlanFromArgs(args, artifactRoot) const steps = [...runPlan.steps] - const initialUrl = runPlan.initialUrl + if (runPlan.adaptiveExploration && (runPlan.initialUrl || runPlan.actionCorpus || steps.length > 0)) { + throw new Error("wordpress.browser-actions adaptive-exploration-json is a standalone additive mode and cannot be combined with url, steps-json, or action-corpus-json") + } + const initialUrl = runPlan.initialUrl ?? runPlan.adaptiveExploration?.startUrl if (steps.length === 0 && !initialUrl) { - throw new Error("wordpress.browser-actions requires steps-json= or url=") + throw new Error("wordpress.browser-actions requires steps-json=, url=, or adaptive-exploration-json=") } if (initialUrl && steps[0]?.kind !== "navigate") { @@ -131,6 +139,8 @@ export async function runBrowserActionsCommand({ let wordpressDiagnosticsReady = false let actionCorpusArtifact: BrowserActionCorpusArtifact | undefined let actionCorpusSummary: BrowserArtifact["summary"]["actionCorpus"] | undefined + let adaptiveExplorationArtifact: BrowserAdaptiveExplorationArtifact | undefined + let adaptiveExplorationSummary: BrowserArtifact["summary"]["adaptiveExploration"] | undefined try { const context = browserPreviewNeedsContextRouting(networkPolicy) || !!storageStateImport ? await browser.newContext({ @@ -163,8 +173,8 @@ export async function runBrowserActionsCommand({ wordpressDiagnosticsReady = await installBrowserWordPressDiagnostics(runPlaygroundCommand, server) viewport = await browserProbeViewport(page) attachBrowserCaptureListeners({ - captureConsole: capture.has("console"), - captureErrors: capture.has("errors"), + captureConsole: capture.has("console") || Boolean(runPlan.adaptiveExploration), + captureErrors: capture.has("errors") || Boolean(runPlan.adaptiveExploration), captureNetwork: true, captureWebSocket: capture.has("websocket"), consoleMessages, @@ -175,9 +185,10 @@ export async function runBrowserActionsCommand({ webSockets, }) - if (runPlan.actionCorpus) { - if (!steps.some((step) => step.kind === "navigate") && runPlan.actionCorpus.startUrl) { - steps.unshift({ kind: "navigate", url: runPlan.actionCorpus.startUrl, waitFor: "domcontentloaded" }) + if (runPlan.actionCorpus || runPlan.adaptiveExploration) { + const generatedStartUrl = runPlan.actionCorpus?.startUrl ?? runPlan.adaptiveExploration?.startUrl + if (!steps.some((step) => step.kind === "navigate") && generatedStartUrl) { + steps.unshift({ kind: "navigate", url: generatedStartUrl, waitFor: "domcontentloaded" }) } if (steps[0]?.kind === "navigate") { const navigateStep = steps.shift()! @@ -198,24 +209,58 @@ export async function runBrowserActionsCommand({ throw error } } - const discovery = await discoverBrowserActionCorpusDescriptors(page) - actionCorpusArtifact = browserActionCorpusArtifact(runPlan.actionCorpus, discovery.descriptors, now()) - actionCorpusArtifact.plan.diagnostics.push(...discovery.diagnostics) - await artifactSession.writeJson("actionCorpus", "action-corpus.json", actionCorpusArtifact) - const corpusSteps = actionCorpusArtifact.plan.steps - steps.unshift(...corpusSteps) - actionCorpusSummary = { - schema: BROWSER_ACTION_CORPUS_SCHEMA, - seed: actionCorpusArtifact.contract.seed, - descriptorsDiscovered: actionCorpusArtifact.plan.observations.descriptorsDiscovered, - descriptorsSelected: actionCorpusArtifact.plan.observations.descriptorsSelected, - stepsPlanned: actionCorpusArtifact.plan.observations.stepsPlanned, - artifact: "files/browser/action-corpus.json", + if (runPlan.actionCorpus) { + const discovery = await discoverBrowserActionCorpusDescriptors(page) + actionCorpusArtifact = browserActionCorpusArtifact(runPlan.actionCorpus, discovery.descriptors, now()) + actionCorpusArtifact.plan.diagnostics.push(...discovery.diagnostics) + await artifactSession.writeJson("actionCorpus", "action-corpus.json", actionCorpusArtifact) + const corpusSteps = actionCorpusArtifact.plan.steps + steps.unshift(...corpusSteps) + actionCorpusSummary = { + schema: BROWSER_ACTION_CORPUS_SCHEMA, + seed: actionCorpusArtifact.contract.seed, + descriptorsDiscovered: actionCorpusArtifact.plan.observations.descriptorsDiscovered, + descriptorsSelected: actionCorpusArtifact.plan.observations.descriptorsSelected, + stepsPlanned: actionCorpusArtifact.plan.observations.stepsPlanned, + artifact: "files/browser/action-corpus.json", + } + } + if (runPlan.adaptiveExploration) { + const adaptiveContract = { + ...runPlan.adaptiveExploration, + budgets: { + ...runPlan.adaptiveExploration.budgets, + maxDurationMs: Math.min(runPlan.adaptiveExploration.budgets.maxDurationMs, livenessRemainingWallTimeMs(startedAtMs, totalTimeoutMs)), + }, + } + const result = await exploreAdaptiveBrowserStateMachine({ + page, + baseUrl: preview.effectiveOrigin, + contract: adaptiveContract, + observations: { consoleMessages, errors, network }, + signal: spec.signal, + }) + adaptiveExplorationArtifact = { schema: "wp-codebox/browser-adaptive-exploration-artifact/v1", contract: adaptiveContract, result, capturedAt: now() } + await artifactSession.writeJson("adaptiveExploration", "adaptive-exploration.json", adaptiveExplorationArtifact) + adaptiveExplorationSummary = { + schema: BROWSER_ADAPTIVE_EXPLORATION_SCHEMA, + seed: result.seed, + status: result.status, + states: result.summary.states, + transitions: result.summary.transitions, + findings: result.summary.findings, + artifact: "files/browser/adaptive-exploration.json", + } + finalUrl = page.url() + if (result.findings.length > 0 && adaptiveContract.failOnFinding) { + pendingError = new Error(`Adaptive browser exploration found ${result.findings.length} reproducible oracle failure(s).`) + } } } const stepIndexOffset = stepRecords.length for (const [loopIndex, step] of steps.entries()) { + if (pendingError) break const index = loopIndex + stepIndexOffset const recordStartedAt = now() const recordStartedAtMs = Date.now() @@ -397,6 +442,7 @@ export async function runBrowserActionsCommand({ ...(domSnapshots.length > 0 ? { domSnapshots: domSnapshots.map((snapshot) => snapshot.snapshot) } : {}), ...(verifierResults.length > 0 ? { verifierResults: verifierResults.map((result) => result.artifact) } : {}), ...(actionCorpusArtifact ? { actionCorpus: "files/browser/action-corpus.json" } : {}), + ...(adaptiveExplorationArtifact ? { adaptiveExploration: "files/browser/adaptive-exploration.json" } : {}), ...(wordpressDiagnostics ? { wordpressDiagnostics: "files/browser/wordpress-diagnostics.json" } : {}), summary: "files/browser/action-summary.json", }, @@ -413,6 +459,7 @@ export async function runBrowserActionsCommand({ ...(domSnapshots.length > 0 ? { domSnapshots } : {}), ...(verifierResults.length > 0 ? { verifierResults } : {}), ...(actionCorpusSummary ? { actionCorpus: actionCorpusSummary } : {}), + ...(adaptiveExplorationSummary ? { adaptiveExploration: adaptiveExplorationSummary } : {}), liveness: { wallTimeoutMs: totalTimeoutMs, networkSettleTimeoutMs: livenessPolicy.networkSettleTimeoutMs }, networkEvents: network.length, ...(capture.has("websocket") ? { webSockets: browserProbeWebSocketSummary(webSockets) } : {}), @@ -450,6 +497,7 @@ export async function runBrowserActionsCommand({ ...(wordpressDiagnosticsSummary ? { wordpressDiagnostics: wordpressDiagnosticsSummary } : {}), ...(verifierResults.length > 0 ? { verifierResults } : {}), ...(actionCorpusArtifact ? { actionCorpus: actionCorpusArtifact.plan } : {}), + ...(adaptiveExplorationArtifact ? { adaptiveExploration: adaptiveExplorationArtifact.result } : {}), viewport, summary: artifact.summary, }) @@ -596,6 +644,7 @@ async function browserActionsRunPlanFromArgs(args: string[], artifactRoot: strin storageStateImport: await browserStorageStateImportFromArgs(args, "wordpress.browser-actions", artifactRoot), maxDomSnapshotElements: positiveIntegerArg(args, "max-dom-snapshot-elements", 160), actionCorpus: browserActionCorpusFromArgs(args), + adaptiveExploration: browserAdaptiveExplorationFromArgs(args), } } @@ -606,148 +655,11 @@ function browserActionCorpusFromArgs(args: string[]): BrowserActionCorpusContrac return browserActionCorpusContract(parsed) } -export async function discoverBrowserActionCorpusDescriptors(page: Page): Promise<{ - descriptors: BrowserActionCorpusDescriptor[] - diagnostics: Array<{ code: string; message: string; metadata?: Record }> -}> { - return await page.evaluate(() => { - const MAX_REJECTION_DIAGNOSTICS = 20 - const descriptors: BrowserActionCorpusDescriptor[] = [] - const rejected: Array<{ kind: string; tag: string; label: string }> = [] - const cssEscape = (value: string) => { - const escapeFn = (globalThis as typeof globalThis & { CSS?: { escape?: (raw: string) => string } }).CSS?.escape - return escapeFn ? escapeFn(value) : value.replace(/[^a-zA-Z0-9_-]/g, "\\$&") - } - const text = (value: string | null | undefined) => (value || "").replace(/\s+/g, " ").trim().slice(0, 120) - const attributeSelector = (name: string, value: string) => `[${name}=${cssEscape(value)}]` - const uniquelySelects = (selector: string, element: Element) => { - try { - const matches = document.querySelectorAll(selector) - return matches.length === 1 && matches[0] === element - } catch { - return false - } - } - const selectorFor = (element: Element): string | undefined => { - const tag = element.tagName.toLowerCase() - const candidates: string[] = [] - const seen = new Set() - const addCandidate = (selector: string) => { - if (!seen.has(selector)) { - seen.add(selector) - candidates.push(selector) - } - } - const id = element.getAttribute("id") - if (id) addCandidate(`#${cssEscape(id)}`) - - const attributes = new Map(["aria-label", "aria-labelledby", "name", "type", "value", "title", "href", "role"].flatMap((name) => { - const value = element.getAttribute(name) - return value ? [[name, value] as const] : [] - })) - const addAttributeCombination = (...names: string[]) => { - if (names.every((name) => attributes.has(name))) { - addCandidate(`${tag}${names.map((name) => attributeSelector(name, attributes.get(name)!)).join("")}`) - } - } - for (const names of [ - ["aria-label"], ["aria-labelledby"], - ["name", "type", "value"], ["name", "type"], ["name", "value"], ["name"], - ["href"], ["title"], - ["role", "type", "value"], ["role", "type"], ["role"], - ["type", "value"], ["value"], ["type"], - ]) { - addAttributeCombination(...names) - } - - const form = (element as HTMLInputElement).form - const formId = form?.getAttribute("id") - if (form && formId) { - const formSelector = `#${cssEscape(formId)}` - if (uniquelySelects(formSelector, form)) { - for (const candidate of [...candidates]) addCandidate(`${formSelector} ${candidate}`) - } - } - for (const candidate of candidates) { - if (uniquelySelects(candidate, element)) return candidate - } - - const parts: string[] = [] - let current: Element | null = element - while (current) { - let part = current.tagName.toLowerCase() - const parent: Element | null = current.parentElement - if (parent) { - const sameTagSiblings = Array.from(parent.children).filter((child) => child.tagName === current?.tagName) - if (sameTagSiblings.length > 1) part += `:nth-of-type(${sameTagSiblings.indexOf(current) + 1})` - } - parts.unshift(part) - const candidate = parts.join(" > ") - if (uniquelySelects(candidate, element)) return candidate - current = parent - } - return undefined - } - const labelFor = (element: Element) => { - const labelledBy = element.getAttribute("aria-labelledby") - if (labelledBy) { - const labelled = labelledBy.split(/\s+/).map((id) => document.getElementById(id)?.textContent).filter(Boolean).join(" ") - if (text(labelled)) return text(labelled) - } - const aria = text(element.getAttribute("aria-label")) - if (aria) return aria - const id = element.getAttribute("id") - const label = id ? document.querySelector(`label[for="${cssEscape(id)}"]`) : null - if (label && text(label.textContent)) return text(label.textContent) - return text(element.textContent) - } - const descriptorId = (kind: string, selector: string, element: Element) => `${kind}:${selector}:${element.getAttribute("name") || ""}:${labelFor(element)}` - const visible = (element: Element) => { - const htmlElement = element as HTMLElement - const style = window.getComputedStyle(htmlElement) - const rect = htmlElement.getBoundingClientRect() - return rect.width > 0 && rect.height > 0 && style.visibility !== "hidden" && style.display !== "none" - } - document.querySelectorAll("a[href], button, input, textarea, select").forEach((element) => { - if (!visible(element)) return - const tag = element.tagName.toLowerCase() - const input = element as HTMLInputElement - const kind = tag === "a" ? "link" : tag === "button" ? "button" : tag === "textarea" ? "textarea" : tag === "select" ? "select" : "input" - const selector = selectorFor(element) - if (!selector) { - rejected.push({ kind, tag, label: labelFor(element) }) - return - } - const descriptor: BrowserActionCorpusDescriptor = { - id: descriptorId(kind, selector, element), - kind, - selector, - label: labelFor(element), - name: element.getAttribute("name") || undefined, - role: element.getAttribute("role") || undefined, - type: input.type || element.getAttribute("type") || undefined, - formId: (element as HTMLInputElement).form?.id || undefined, - href: tag === "a" ? (element as HTMLAnchorElement).href : undefined, - disabled: Boolean((element as HTMLButtonElement).disabled), - readonly: Boolean(input.readOnly), - optionValues: tag === "select" ? Array.from((element as HTMLSelectElement).options).map((option) => option.value).filter(Boolean) : undefined, - } - descriptors.push(descriptor) - }) - const diagnostics: Array<{ code: string; message: string; metadata?: Record }> = rejected.slice(0, MAX_REJECTION_DIAGNOSTICS).map((item) => ({ - code: "browser_action_corpus_selector_not_unique", - message: "An actionable control was rejected because discovery could not produce a selector resolving uniquely to that element.", - metadata: item, - })) - if (rejected.length > MAX_REJECTION_DIAGNOSTICS) { - diagnostics[MAX_REJECTION_DIAGNOSTICS - 1] = { - code: "browser_action_corpus_selector_rejections_truncated", - message: "Additional actionable controls without unique selectors were omitted from diagnostics.", - metadata: { rejected: rejected.length, retainedDiagnostics: MAX_REJECTION_DIAGNOSTICS - 1 }, - } - } - return { descriptors, diagnostics } - }) +function browserAdaptiveExplorationFromArgs(args: string[]): BrowserAdaptiveExplorationContract | undefined { + const raw = argValue(args, "adaptive-exploration-json") + if (typeof raw !== "string" || raw.trim().length === 0) return undefined + const parsed = JSON.parse(raw) as Record + return browserAdaptiveExplorationContract(parsed) } async function captureBrowserActionDomSnapshot({ diff --git a/packages/runtime-playground/src/browser-adaptive-explorer.ts b/packages/runtime-playground/src/browser-adaptive-explorer.ts new file mode 100644 index 000000000..bf75aa906 --- /dev/null +++ b/packages/runtime-playground/src/browser-adaptive-explorer.ts @@ -0,0 +1,503 @@ +import { + BROWSER_ADAPTIVE_EXPLORATION_SCHEMA, + browserAdaptiveDigest, + orderBrowserAdaptiveActions, + planBrowserAdaptiveStateActions, + type BrowserActionCorpusDescriptor, + type BrowserAdaptiveAction, + type BrowserAdaptiveExplorationContract, + type BrowserAdaptiveExplorationResult, + type BrowserAdaptiveFinding, + type BrowserAdaptiveFrameIdentity, + type BrowserAdaptiveState, + type BrowserAdaptiveTransition, +} from "@automattic/wp-codebox-core" +import { stableJson } from "@automattic/wp-codebox-core/internals" +import type { Frame, Page } from "playwright" + +import { discoverBrowserActionCorpusDescriptors } from "./browser-action-discovery.js" + +interface AdaptiveObservationSources { + consoleMessages: object[] + errors: object[] + network: object[] +} + +interface CapturedAdaptiveState { + state: BrowserAdaptiveState + diagnostics: BrowserAdaptiveExplorationResult["diagnostics"] + loading: number +} + +interface StabilizationResult extends CapturedAdaptiveState { + waitedMs: number + polls: number + mutationRecords: number + mutationEvidenceTruncated: boolean +} + +interface FrontierEntry { + state: BrowserAdaptiveState + path: BrowserAdaptiveAction[] +} + +export async function exploreAdaptiveBrowserStateMachine({ + page, + baseUrl, + contract, + observations, + signal, + now = Date.now, +}: { + page: Page + baseUrl: string + contract: BrowserAdaptiveExplorationContract + observations: AdaptiveObservationSources + signal?: AbortSignal + now?: () => number +}): Promise { + const started = now() + const states = new Map() + const transitions: BrowserAdaptiveTransition[] = [] + const findings: BrowserAdaptiveFinding[] = [] + const diagnostics: BrowserAdaptiveExplorationResult["diagnostics"] = [] + const actionVisits = new Map() + let actions = 0 + let errors = 0 + let revisits = 0 + let exhausted: BrowserAdaptiveExplorationResult["summary"]["budgetExhausted"] + + const initial = await captureAdaptiveState(page, contract, 0) + appendDiagnostics(diagnostics, initial.diagnostics, contract.descriptorLimits.maxDiagnostics) + states.set(initial.state.digest, initial.state) + const frontier: FrontierEntry[] = [{ state: initial.state, path: [] }] + if (artifactBytes(states, transitions, diagnostics, findings) > contract.budgets.maxArtifactBytes) exhausted = "maxArtifactBytes" + + while (frontier.length > 0 && !exhausted && findings.length === 0) { + if (signal?.aborted) { exhausted = "cancelled"; break } + if (now() - started >= contract.budgets.maxDurationMs) { exhausted = "maxDurationMs"; break } + const source = frontier.shift() as FrontierEntry + const candidates = orderBrowserAdaptiveActions(planBrowserAdaptiveStateActions(source.state, contract), contract.seed, source.state.digest) + for (const action of candidates) { + if (signal?.aborted) { exhausted = "cancelled"; break } + if (actions >= contract.budgets.maxActions) { exhausted = "maxActions"; break } + if (transitions.length >= contract.budgets.maxTransitions) { exhausted = "maxTransitions"; break } + if (now() - started >= contract.budgets.maxDurationMs) { exhausted = "maxDurationMs"; break } + const actionVisitKey = `${source.state.digest}:${action.id}` + const visits = actionVisits.get(actionVisitKey) ?? 0 + if (visits >= contract.revisitPolicy.maxActionVisits) continue + actionVisits.set(actionVisitKey, visits + 1) + + if (contract.resetPolicy.mode === "start-url" && (transitions.length > 0 || source.path.length > 0)) { + if (actions + source.path.length >= contract.budgets.maxActions) { exhausted = "maxActions"; break } + const restored = await restoreAdaptivePath(page, baseUrl, contract, source.path, signal) + actions += restored.executed + appendDiagnostics(diagnostics, restored.diagnostics, contract.descriptorLimits.maxDiagnostics) + if (!restored.state || restored.state.digest !== source.state.digest) { + const rejected = rejectedTransition(source.state, action, page.url(), "browser_adaptive_source_state_not_reproduced", "The declared start URL and replay path did not reproduce the queued source state.") + if (artifactBytes(states, [...transitions, rejected], diagnostics, findings) > contract.budgets.maxArtifactBytes) { exhausted = "maxArtifactBytes"; break } + transitions.push(rejected) + continue + } + } + + const beforeConsole = observations.consoleMessages.length + const beforeErrors = observations.errors.length + const beforeNetwork = observations.network.length + const transitionStarted = now() + await installMutationObservers(page, contract) + let actionError: string | undefined + try { + await executeAdaptiveAction(page, action, contract.stabilization.maxWaitMs) + } catch (error) { + actionError = error instanceof Error ? error.message : String(error) + } + actions += 1 + const stabilized = await stabilizeAdaptiveState(page, contract, source.path.length + 1, now) + appendDiagnostics(diagnostics, stabilized.diagnostics, contract.descriptorLimits.maxDiagnostics) + const newConsoleErrors = consoleErrorMessages(observations.consoleMessages.slice(beforeConsole)) + const newPageErrors = errorMessages(observations.errors.slice(beforeErrors)) + const fingerprints = [...new Set([...newConsoleErrors, ...newPageErrors].map((message) => browserAdaptiveDigest("oracle", message)))].sort() + const existing = states.get(stabilized.state.digest) + const newState = !existing + if (newState && states.size < contract.budgets.maxStates) { + states.set(stabilized.state.digest, stabilized.state) + } else if (newState) { + exhausted = "maxStates" + } else { + revisits += 1 + existing.visits += 1 + } + const destination = existing ?? stabilized.state + const transition: BrowserAdaptiveTransition = { + id: `transition-${transitions.length}`, + sourceDigest: source.state.digest, + destinationDigest: destination.digest, + action, + sourceUrl: source.state.url, + destinationUrl: destination.url, + history: { before: source.state.historyLength, after: destination.historyLength, beforeStateDigest: source.state.historyStateDigest, afterStateDigest: destination.historyStateDigest }, + timing: { durationMs: Math.max(0, now() - transitionStarted), stabilizationMs: stabilized.waitedMs, polls: stabilized.polls }, + novelty: { + newState, + newDescriptors: destination.descriptors.filter((descriptor) => !source.state.descriptors.some((candidate) => candidate.id === descriptor.id && candidate.frameId === descriptor.frameId)).length, + mutationRecords: stabilized.mutationRecords, + mutationEvidenceTruncated: stabilized.mutationEvidenceTruncated, + }, + observations: { + networkEvents: observations.network.length - beforeNetwork, + consoleErrors: newConsoleErrors.slice(0, contract.budgets.maxErrors), + pageErrors: newPageErrors.slice(0, contract.budgets.maxErrors), + loadingBefore: source.state.loadingIndicators, + loadingAfter: stabilized.loading, + oracleFingerprints: fingerprints, + }, + status: actionError ? "error" : newState ? "ok" : "revisited", + ...(actionError ? { diagnostic: { code: "browser_adaptive_action_error", message: actionError } } : {}), + } + errors += newConsoleErrors.length + newPageErrors.length + (actionError ? 1 : 0) + if (artifactBytes(states, [...transitions, transition], diagnostics, findings) > contract.budgets.maxArtifactBytes) { + if (newState) states.delete(stabilized.state.digest) + exhausted = "maxArtifactBytes" + break + } + transitions.push(transition) + + const path = [...source.path, action] + if (fingerprints.length > 0) { + const fingerprint = fingerprints[0] as string + const finding: BrowserAdaptiveFinding = { + fingerprint, + stateDigest: destination.digest, + transitionId: transition.id, + originalPath: path, + minimizedPath: path, + replay: { + schema: BROWSER_ADAPTIVE_EXPLORATION_SCHEMA, + seed: contract.seed, + startUrl: contract.startUrl, + expectedFingerprint: fingerprint, + expectedStateDigest: destination.digest, + actions: path, + resetPolicy: contract.resetPolicy, + }, + } + if (artifactBytes(states, transitions, diagnostics, [finding]) > contract.budgets.maxArtifactBytes) { + exhausted = "maxArtifactBytes" + break + } + findings.push(finding) + break + } + if (errors >= contract.budgets.maxErrors) { exhausted = "maxErrors"; break } + if (newState && states.size <= contract.budgets.maxStates) frontier.push({ state: destination, path }) + if (!newState && destination.visits < contract.revisitPolicy.maxStateVisits) frontier.push({ state: destination, path }) + if (contract.resetPolicy.mode === "none") break + } + } + + if (findings.length > 0 && !signal?.aborted) { + for (const finding of findings) { + const minimized = await minimizeAdaptiveFinding(page, baseUrl, contract, finding, observations, Math.max(0, contract.budgets.maxActions - actions), started + contract.budgets.maxDurationMs, signal, now) + actions += minimized.executed + finding.minimizedPath = minimized.path + finding.replay.actions = finding.minimizedPath + if (minimized.exhausted && !exhausted) exhausted = minimized.exhausted + } + } + if (!exhausted && findings.length === 0 && frontier.length > 0) exhausted = "frontier" + if (exhausted) appendTerminalDiagnostic(diagnostics, { code: exhausted === "cancelled" ? "browser_adaptive_cancelled" : "browser_adaptive_budget_exhausted", message: exhausted === "cancelled" ? "Adaptive exploration stopped scheduling actions after cancellation and retained partial evidence." : `Adaptive exploration stopped at the ${exhausted} bound.`, metadata: { budget: exhausted } }, contract.descriptorLimits.maxDiagnostics) + + return { + schema: BROWSER_ADAPTIVE_EXPLORATION_SCHEMA, + status: exhausted ? "incomplete" : findings.length > 0 ? "findings" : "completed", + seed: contract.seed, + startUrl: contract.startUrl, + states: [...states.values()], + transitions, + findings, + diagnostics: diagnostics.slice(0, contract.descriptorLimits.maxDiagnostics), + summary: { actions, states: states.size, transitions: transitions.length, revisits, errors, findings: findings.length, ...(exhausted ? { budgetExhausted: exhausted } : {}) }, + replay: { schema: BROWSER_ADAPTIVE_EXPLORATION_SCHEMA, seed: contract.seed, startUrl: contract.startUrl, contract }, + } +} + +async function captureAdaptiveState(page: Page, contract: BrowserAdaptiveExplorationContract, depth: number): Promise { + const diagnostics: BrowserAdaptiveExplorationResult["diagnostics"] = [] + const mainOrigin = origin(page.url()) + const frames = frameIdentities(page, contract.descriptorLimits.maxPerState) + const descriptors: BrowserActionCorpusDescriptor[] = [] + const semanticFrames: Array> = [] + let loading = 0 + for (const identity of frames) { + const frame = frameById(page, identity.id) + if (!frame) continue + const frameOrigin = origin(frame.url()) + if (identity.scope === "same-origin-frame" && frameOrigin && mainOrigin && frameOrigin !== mainOrigin) { + diagnostics.push({ code: "browser_adaptive_frame_inaccessible", message: "A cross-origin frame was not explored.", metadata: { frameId: identity.id, url: frame.url() } }) + continue + } + try { + const discovery = await discoverBrowserActionCorpusDescriptors(frame) + diagnostics.push(...discovery.diagnostics.map((diagnostic) => ({ ...diagnostic, metadata: { ...diagnostic.metadata, frameId: identity.id } }))) + const scopedDescriptors = discovery.descriptors.filter((descriptor) => { + const hrefOrigin = descriptor.href ? origin(descriptor.href) : undefined + if (!hrefOrigin || !mainOrigin || hrefOrigin === mainOrigin) return true + diagnostics.push({ code: "browser_adaptive_cross_origin_action_rejected", message: "A link leaving the exploration origin was excluded from the adaptive action frontier.", metadata: { frameId: identity.id, descriptorId: descriptor.id, href: descriptor.href } }) + return false + }) + descriptors.push(...scopedDescriptors.slice(0, contract.descriptorLimits.maxPerState).map((descriptor) => ({ ...descriptor, frameId: identity.id }))) + const semantic = await frame.evaluate(({ maxElements, maxTextLength }) => { + const text = (document.body?.innerText || "").replace(/\s+/g, " ").trim().slice(0, maxTextLength) + const loadingSelectors = "[aria-busy='true'], [role='progressbar'], .loading, .spinner, [data-loading='true']" + const visible = (element: Element) => { + const rect = element.getBoundingClientRect() + const style = getComputedStyle(element) + return rect.width > 0 && rect.height > 0 && style.display !== "none" && style.visibility !== "hidden" + } + const dom = Array.from(document.body?.querySelectorAll("*") ?? []).filter(visible).slice(0, maxElements).map((element) => { + const input = element as HTMLInputElement + return { + tag: element.tagName.toLowerCase(), + id: element.getAttribute("id") || undefined, + class: element.getAttribute("class") || undefined, + role: element.getAttribute("role") || undefined, + ariaHidden: element.getAttribute("aria-hidden") || undefined, + text: (element.textContent || "").replace(/\s+/g, " ").trim().slice(0, 120), + value: "value" in input ? String(input.value).slice(0, 120) : undefined, + checked: "checked" in input ? Boolean(input.checked) : undefined, + } + }) + return { + title: document.title || "", + text, + dom, + historyLength: history.length, + historyState: (() => { try { const serialized = JSON.stringify(history.state); return typeof serialized === "string" ? serialized.slice(0, maxTextLength) : "null" } catch { return "[unserializable]" } })(), + loading: Array.from(document.querySelectorAll(loadingSelectors)).filter(visible).length, + shadowRoots: Array.from(document.querySelectorAll("*")).filter((element) => Boolean(element.shadowRoot)).length, + } + }, { maxElements: contract.descriptorLimits.maxPerState, maxTextLength: contract.descriptorLimits.maxTextLength }) + loading += semantic.loading + if (semantic.shadowRoots > 0) diagnostics.push({ code: "browser_adaptive_shadow_roots_not_explored", message: "Open shadow roots were observed but are outside the current actionable descriptor scope.", metadata: { frameId: identity.id, observed: semantic.shadowRoots } }) + semanticFrames.push({ id: identity.id, url: frame.url(), title: semantic.title, text: semantic.text, dom: semantic.dom, historyLength: semantic.historyLength, historyState: semantic.historyState, loading: semantic.loading }) + } catch (error) { + diagnostics.push({ code: "browser_adaptive_frame_inaccessible", message: "A frame could not be inspected within the bounded discovery pass.", metadata: { frameId: identity.id, url: frame.url(), reason: error instanceof Error ? error.message : String(error) } }) + } + } + const boundedDescriptors: BrowserActionCorpusDescriptor[] = [] + const descriptorByteBudget = Math.max(1_024, Math.floor(contract.budgets.maxArtifactBytes / 4)) + for (const descriptor of descriptors.slice(0, contract.descriptorLimits.maxPerState)) { + if (Buffer.byteLength(stableJson([...boundedDescriptors, descriptor])) > descriptorByteBudget) break + boundedDescriptors.push(descriptor) + } + if (descriptors.length > boundedDescriptors.length) diagnostics.push({ code: "browser_adaptive_descriptors_truncated", message: "Actionable descriptors were truncated at the per-state bound.", metadata: { discovered: descriptors.length, retained: boundedDescriptors.length } }) + const descriptorDigest = browserAdaptiveDigest("descriptors", boundedDescriptors.map(stableDescriptor)) + const digest = browserAdaptiveDigest("state", { frames: semanticFrames.map(({ historyLength: _historyLength, ...frame }) => frame), descriptorDigest }) + const mainSemantic = semanticFrames.find((frame) => frame.id === "document") + return { + state: { + digest, + url: page.url(), + historyLength: typeof mainSemantic?.historyLength === "number" ? mainSemantic.historyLength : 0, + historyStateDigest: browserAdaptiveDigest("state", mainSemantic?.historyState ?? "null"), + descriptorDigest, + descriptors: boundedDescriptors, + frames: frames.filter((frame) => semanticFrames.some((semantic) => semantic.id === frame.id)), + visits: 1, + depth, + loadingIndicators: loading, + }, + diagnostics, + loading, + } +} + +async function executeAdaptiveAction(page: Page, action: BrowserAdaptiveAction, timeout: number): Promise { + if (action.family === "back") { await page.goBack({ waitUntil: "domcontentloaded", timeout }); return } + if (action.family === "reload") { await page.reload({ waitUntil: "domcontentloaded", timeout }); return } + const frame = frameById(page, action.frameId) + if (!frame) throw new Error(`Adaptive action frame is no longer available: ${action.frameId}`) + for (const step of action.steps) { + if (!step.selector) throw new Error(`Adaptive ${step.kind} action requires a unique selector.`) + const locator = frame.locator(step.selector) + const count = await locator.count() + if (count !== 1) throw new Error(`Adaptive selector must resolve exactly once, resolved ${count}: ${step.selector}`) + if (step.kind === "click") await locator.click({ timeout }) + else if (step.kind === "fill") await locator.fill(String(step.value ?? ""), { timeout }) + else if (step.kind === "select") await locator.selectOption(Array.isArray(step.values) ? step.values : String(step.value ?? ""), { timeout }) + else if (step.kind === "press") await locator.press(String(step.key ?? ""), { timeout }) + else throw new Error(`Unsupported adaptive interaction step: ${step.kind}`) + } +} + +async function stabilizeAdaptiveState(page: Page, contract: BrowserAdaptiveExplorationContract, depth: number, now: () => number): Promise { + const started = now() + let polls = 0 + let quietSince = started + let previous: CapturedAdaptiveState | undefined + while (now() - started < contract.stabilization.maxWaitMs) { + await page.waitForTimeout(contract.stabilization.pollIntervalMs) + const current = await captureAdaptiveState(page, contract, depth) + polls += 1 + if (!previous || current.state.digest !== previous.state.digest) quietSince = now() + previous = current + if (current.loading === 0 && now() - quietSince >= contract.stabilization.quietWindowMs) break + } + const state = previous ?? await captureAdaptiveState(page, contract, depth) + const mutations = await readMutationObservers(page, contract) + return { ...state, waitedMs: Math.max(0, now() - started), polls, mutationRecords: mutations.count, mutationEvidenceTruncated: mutations.truncated } +} + +async function installMutationObservers(page: Page, contract: BrowserAdaptiveExplorationContract): Promise { + await Promise.all(frameIdentities(page, contract.descriptorLimits.maxPerState).map(async (identity) => { + const frame = frameById(page, identity.id) + if (!frame) return + await frame.evaluate(({ maximum }) => { + const target = globalThis as typeof globalThis & { __wpCodeboxAdaptiveMutations?: { count: number; truncated: boolean; observer: MutationObserver } } + target.__wpCodeboxAdaptiveMutations?.observer.disconnect() + const evidence = { count: 0, truncated: false, observer: undefined as unknown as MutationObserver } + evidence.observer = new MutationObserver((records) => { + evidence.count += records.length + if (evidence.count > maximum) { evidence.count = maximum; evidence.truncated = true } + }) + if (document.documentElement) evidence.observer.observe(document.documentElement, { childList: true, subtree: true, attributes: true, characterData: true }) + target.__wpCodeboxAdaptiveMutations = evidence + }, { maximum: contract.stabilization.maxMutationRecords }).catch(() => undefined) + })) +} + +async function readMutationObservers(page: Page, contract: BrowserAdaptiveExplorationContract): Promise<{ count: number; truncated: boolean }> { + let count = 0 + let truncated = false + for (const identity of frameIdentities(page, contract.descriptorLimits.maxPerState)) { + const frame = frameById(page, identity.id) + if (!frame) continue + const result = await frame.evaluate(() => { + const target = globalThis as typeof globalThis & { __wpCodeboxAdaptiveMutations?: { count: number; truncated: boolean; observer: MutationObserver } } + const evidence = target.__wpCodeboxAdaptiveMutations + evidence?.observer.disconnect() + return evidence ? { count: evidence.count, truncated: evidence.truncated } : { count: 0, truncated: false } + }).catch(() => ({ count: 0, truncated: false })) + count = Math.min(contract.stabilization.maxMutationRecords, count + result.count) + truncated ||= result.truncated || count >= contract.stabilization.maxMutationRecords + } + return { count, truncated } +} + +async function restoreAdaptivePath(page: Page, baseUrl: string, contract: BrowserAdaptiveExplorationContract, path: BrowserAdaptiveAction[], signal?: AbortSignal): Promise<{ state?: BrowserAdaptiveState; executed: number; diagnostics: BrowserAdaptiveExplorationResult["diagnostics"] }> { + const diagnostics: BrowserAdaptiveExplorationResult["diagnostics"] = [] + let executed = 0 + try { + await page.goto(resolveUrl(contract.startUrl, baseUrl), { waitUntil: "domcontentloaded", timeout: contract.stabilization.maxWaitMs }) + await stabilizeAdaptiveState(page, contract, 0, Date.now) + for (const action of path) { + if (signal?.aborted) return { executed, diagnostics } + executed += 1 + await executeAdaptiveAction(page, action, contract.stabilization.maxWaitMs) + await stabilizeAdaptiveState(page, contract, executed, Date.now) + } + return { state: (await captureAdaptiveState(page, contract, path.length)).state, executed, diagnostics } + } catch (error) { + diagnostics.push({ code: "browser_adaptive_reset_replay_failed", message: "The start-URL reset path could not reproduce a queued state.", metadata: { reason: error instanceof Error ? error.message : String(error) } }) + return { executed, diagnostics } + } +} + +async function minimizeAdaptiveFinding(page: Page, baseUrl: string, contract: BrowserAdaptiveExplorationContract, finding: BrowserAdaptiveFinding, observations: AdaptiveObservationSources, maximumActions: number, deadline: number, signal?: AbortSignal, now: () => number = Date.now): Promise<{ path: BrowserAdaptiveAction[]; executed: number; exhausted?: "maxActions" | "maxDurationMs" | "cancelled" }> { + let current = [...finding.originalPath] + let chunk = Math.max(1, Math.floor(current.length / 2)) + let executed = 0 + let exhausted: "maxActions" | "maxDurationMs" | "cancelled" | undefined + while (current.length > 1 && chunk >= 1 && !signal?.aborted) { + let reduced = false + for (let start = 0; start < current.length; start += chunk) { + const candidate = [...current.slice(0, start), ...current.slice(start + chunk)] + if (candidate.length === 0) continue + if (now() >= deadline) { exhausted = "maxDurationMs"; break } + if (executed + candidate.length > maximumActions) { exhausted = "maxActions"; break } + const beforeConsole = observations.consoleMessages.length + const beforeErrors = observations.errors.length + const replay = await restoreAdaptivePath(page, baseUrl, contract, candidate, signal) + executed += replay.executed + const fingerprints = [...consoleErrorMessages(observations.consoleMessages.slice(beforeConsole)), ...errorMessages(observations.errors.slice(beforeErrors))].map((message) => browserAdaptiveDigest("oracle", message)) + if (replay.state && fingerprints.includes(finding.fingerprint) && (!finding.stateDigest || replay.state.digest === finding.stateDigest)) { + current = candidate + reduced = true + break + } + } + if (exhausted) break + if (!reduced) chunk = Math.floor(chunk / 2) + } + if (signal?.aborted) exhausted = "cancelled" + return { path: current, executed, ...(exhausted ? { exhausted } : {}) } +} + +function frameIdentities(page: Page, maximum: number): BrowserAdaptiveFrameIdentity[] { + const identities: BrowserAdaptiveFrameIdentity[] = [{ id: "document", url: page.url(), scope: "document" }] + const walk = (parent: Frame, parentId: string) => { + parent.childFrames().forEach((frame, index) => { + if (identities.length >= maximum) return + const id = parentId === "document" ? `frame:${index}` : `${parentId}.${index}` + identities.push({ id, parentId, url: frame.url(), scope: "same-origin-frame" }) + walk(frame, id) + }) + } + walk(page.mainFrame(), "document") + return identities +} + +function frameById(page: Page, id: string): Frame | undefined { + if (id === "document") return page.mainFrame() + const indexes = id.replace(/^frame:/, "").split(".").map(Number) + let frame = page.mainFrame() + for (const index of indexes) { + const next = frame.childFrames()[index] + if (!next) return undefined + frame = next + } + return frame +} + +function stableDescriptor(descriptor: BrowserActionCorpusDescriptor): Record { + return { id: descriptor.id, frameId: descriptor.frameId, kind: descriptor.kind, selector: descriptor.selector, label: descriptor.label, name: descriptor.name, role: descriptor.role, type: descriptor.type, formId: descriptor.formId, href: descriptor.href, optionValues: descriptor.optionValues } +} + +function rejectedTransition(state: BrowserAdaptiveState, action: BrowserAdaptiveAction, destinationUrl: string, code: string, message: string): BrowserAdaptiveTransition { + return { id: `transition-rejected-${browserAdaptiveDigest("action", `${state.digest}:${action.id}`).slice(0, 12)}`, sourceDigest: state.digest, action, sourceUrl: state.url, destinationUrl, history: { before: state.historyLength, after: state.historyLength, beforeStateDigest: state.historyStateDigest, afterStateDigest: state.historyStateDigest }, timing: { durationMs: 0, stabilizationMs: 0, polls: 0 }, novelty: { newState: false, newDescriptors: 0, mutationRecords: 0, mutationEvidenceTruncated: false }, observations: { networkEvents: 0, consoleErrors: [], pageErrors: [], loadingBefore: 0, loadingAfter: 0, oracleFingerprints: [] }, status: "rejected", diagnostic: { code, message } } +} + +function consoleErrorMessages(records: object[]): string[] { + return records.map((record) => record as Record).filter((record) => record.type === "error" || record.level === "error").map(recordMessage).filter(Boolean).slice(0, 100) +} + +function errorMessages(records: object[]): string[] { + return records.map((record) => recordMessage(record as Record)).filter(Boolean).slice(0, 100) +} + +function recordMessage(record: Record): string { + return typeof record.message === "string" ? record.message : typeof record.text === "string" ? record.text : stableJson(record).slice(0, 1_000) +} + +function artifactBytes(states: Map, transitions: BrowserAdaptiveTransition[], diagnostics: BrowserAdaptiveExplorationResult["diagnostics"], findings: BrowserAdaptiveFinding[]): number { + return Buffer.byteLength(stableJson({ states: [...states.values()], transitions, diagnostics, findings })) +} + +function appendDiagnostics(target: BrowserAdaptiveExplorationResult["diagnostics"], incoming: BrowserAdaptiveExplorationResult["diagnostics"], maximum: number): void { + const remaining = Math.max(0, maximum - target.length) + if (remaining > 0) target.push(...incoming.slice(0, remaining)) +} + +function appendTerminalDiagnostic(target: BrowserAdaptiveExplorationResult["diagnostics"], diagnostic: BrowserAdaptiveExplorationResult["diagnostics"][number], maximum: number): void { + if (target.length < maximum) target.push(diagnostic) + else target[Math.max(0, maximum - 1)] = diagnostic +} + +function origin(url: string): string | undefined { + try { return new URL(url).origin } catch { return undefined } +} + +function resolveUrl(pathOrUrl: string, baseUrl: string): string { + try { return new URL(pathOrUrl).toString() } catch { return new URL(pathOrUrl, baseUrl).toString() } +} diff --git a/packages/runtime-playground/src/browser-artifacts.ts b/packages/runtime-playground/src/browser-artifacts.ts index a5136096e..b95a71e6a 100644 --- a/packages/runtime-playground/src/browser-artifacts.ts +++ b/packages/runtime-playground/src/browser-artifacts.ts @@ -84,6 +84,7 @@ export interface BrowserArtifactFiles { domSnapshots?: string[] verifierResults?: string[] actionCorpus?: string + adaptiveExploration?: string sourceScreenshot?: string | string[] candidateScreenshot?: string | string[] sourceDomSnapshot?: string | string[] @@ -167,6 +168,15 @@ export interface BrowserArtifactSummary { stepsPlanned: number artifact: string } + adaptiveExploration?: { + schema: "wp-codebox/browser-adaptive-exploration/v1" + seed: string + status: "completed" | "findings" | "incomplete" + states: number + transitions: number + findings: number + artifact: string + } networkPolicy?: BrowserProbeNetworkPolicySummary previewProxy?: PlaygroundPreviewProxyDiagnostics lifecycle?: BrowserProbeLifecycleSummary @@ -1107,6 +1117,7 @@ const BROWSER_ARTIFACT_FILE_MANIFEST: Record diagnostic.code === "campaign-interrupted")) }) + +test("minimization preserves the exact oracle and state fingerprint", async () => { + const exactState = adversarialCampaign({ + id: "exact-state", + seed: "exact-state-seed", + corpus: [{ id: "journey", actions: [{ type: "open" }, { type: "trigger" }], input: { value: "fixture" } }], + mutationKinds: ["scalar"], + budgets: { maxCases: 1, maxActionsPerCase: 4, maxCaseTimeMs: 1_000, maxWallTimeMs: 5_000 }, + oracles: [{ schema: "wp-codebox/adversarial-oracle/v1", id: "fixture-defect", severity: "high" }], + }) + const result = await runAdversarialCampaign(exactState, { + execute: async (plan) => ({ + status: "failed", + stateDigest: plan.actions.some((action) => action.type === "open") ? "modal-open" : "modal-closed", + diagnostics: [{ code: "fixture-defect", message: "same visible error in a different state" }], + }), + evaluate: async () => [{ oracleId: "fixture-defect", failed: true }], + }) + const finding = result.findings[0]! + assert.equal(finding.replay.expectedStateDigest, "modal-open") + assert.equal(finding.replay.expectedFingerprint, finding.fingerprint) + assert(finding.minimized.actions.some((action) => action.type === "open"), "a candidate in the wrong state must not be accepted") +}) diff --git a/tests/adversarial-recipe-orchestration.test.ts b/tests/adversarial-recipe-orchestration.test.ts index b0ba3ddf8..5eef909c9 100644 --- a/tests/adversarial-recipe-orchestration.test.ts +++ b/tests/adversarial-recipe-orchestration.test.ts @@ -74,6 +74,7 @@ assert.deepEqual(first[0]?.result.findings, second[0]?.result.findings) assert.equal(first[0]?.capabilities.optional[0]?.available, false, "optional fidelity must be explicit") assert(executions.length > 0, "generated cases must execute through runtime commands") assert(checkpointOperations.includes("create:baseline") && checkpointOperations.includes("restore:baseline"), "campaign cases must use the existing checkpoint reset path") +assert(checkpointOperations.filter((operation) => operation === "restore:baseline").length >= (first[0]?.result.summary.generated ?? 0) + (second[0]?.result.summary.generated ?? 0), "every campaign and minimization execution must restore the declared baseline before running") const unsupportedRecipe = structuredClone(recipe) unsupportedRecipe.adversarialCampaigns![0]!.requiredCapabilities = ["missing-adapter"] diff --git a/tests/browser-adaptive-exploration.test.ts b/tests/browser-adaptive-exploration.test.ts new file mode 100644 index 000000000..33120eb32 --- /dev/null +++ b/tests/browser-adaptive-exploration.test.ts @@ -0,0 +1,201 @@ +import assert from "node:assert/strict" +import test from "node:test" +import { chromium, type Page } from "playwright" + +import { + BROWSER_ADAPTIVE_EXPLORATION_SCHEMA, + browserAdaptiveExplorationContract, + planBrowserAdaptiveStateActions, +} from "../packages/runtime-core/src/browser-adaptive-exploration.js" +import { getCommandDefinition } from "../packages/runtime-core/src/command-registry.js" +import { exploreAdaptiveBrowserStateMachine } from "../packages/runtime-playground/src/browser-adaptive-explorer.js" + +const modalFixture = ` + + +` + +const delayedRouteFixture = ` + + +` + +test("adaptive browser exploration is an additive public browser-actions mode", () => { + const definition = getCommandDefinition("wordpress.browser-actions") + const argument = definition?.acceptedArgs.find((candidate) => candidate.name === "adaptive-exploration-json") + assert.equal(argument?.format, "JSON object") + assert.match(argument?.description ?? "", /bounded deterministic state graph/) + assert(definition?.acceptedArgs.some((candidate) => candidate.name === "action-corpus-json"), "one-shot corpus remains public") + assert(definition?.acceptedArgs.some((candidate) => candidate.name === "steps-json"), "authored steps remain public") +}) + +test("adaptive contract normalizes every safety and exploration bound", () => { + const contract = browserAdaptiveExplorationContract({ + seed: "bounded", + startUrl: "/fixture", + actionFamilies: ["click", "fill", "double-submit", "unknown"], + budgets: { maxActions: 0, maxStates: 9999, maxTransitions: 7, maxDurationMs: 1, maxArtifactBytes: 2, maxErrors: 3 }, + revisitPolicy: { maxStateVisits: 0, maxActionVisits: 99 }, + descriptorLimits: { maxPerState: 4, maxDiagnostics: 5, maxTextLength: 1 }, + stabilization: { pollIntervalMs: 1, quietWindowMs: 1, maxWaitMs: 1, maxMutationRecords: 0 }, + }) + assert.equal(contract.schema, BROWSER_ADAPTIVE_EXPLORATION_SCHEMA) + assert.deepEqual(contract.actionFamilies, ["click", "fill", "double-submit"]) + assert.equal(contract.budgets.maxActions, 1) + assert.equal(contract.budgets.maxStates, 250) + assert.equal(contract.budgets.maxArtifactBytes, 1024) + assert.equal(contract.revisitPolicy.maxStateVisits, 1) + assert.equal(contract.descriptorLimits.maxTextLength, 64) + assert.equal(contract.stabilization.pollIntervalMs, 10) + + const allFamilies = browserAdaptiveExplorationContract({ seed: "families", startUrl: "/fixture" }) + const planned = planBrowserAdaptiveStateActions({ + digest: "state", + url: "/fixture", + historyLength: 1, + historyStateDigest: "history", + descriptorDigest: "descriptors", + descriptors: [ + { id: "input", kind: "input", selector: "#field", type: "text", formId: "form", frameId: "document" }, + { id: "select", kind: "select", selector: "#choice", optionValues: ["one"], formId: "form", frameId: "document" }, + { id: "submit", kind: "button", selector: "#submit", type: "submit", formId: "form", frameId: "document" }, + { id: "link", kind: "link", selector: "#link", frameId: "document" }, + ], + frames: [{ id: "document", url: "/fixture", scope: "document" }], + visits: 1, + depth: 0, + loadingIndicators: 0, + }, allFamilies) + assert.deepEqual(new Set(planned.map((action) => action.family)), new Set(["click", "fill", "select", "submit", "keyboard", "back", "reload", "repeat", "double-submit"])) +}) + +test("rediscovery finds, minimizes, and replays a defect revealed by a dynamic modal", async () => { + const run = await runFixture(modalFixture, { + seed: "modal-seed", + budgets: { maxActions: 80, maxStates: 24, maxTransitions: 80, maxDurationMs: 30_000, maxArtifactBytes: 2_000_000, maxErrors: 10 }, + actionFamilies: ["click", "fill", "submit", "repeat", "double-submit"], + }) + assert.equal(run.result.status, "findings") + assert.equal(run.result.findings.length, 1) + const finding = run.result.findings[0]! + assert(run.result.states.some((state) => state.descriptors.some((descriptor) => descriptor.selector === "#modal-save")), "modal controls must be discovered after interaction") + assert(run.result.transitions.some((transition) => transition.novelty.newDescriptors >= 2 && transition.novelty.mutationRecords > 0), "modal transition must retain bounded mutation novelty") + assert(finding.originalPath.length >= 2, "the failure requires opening the modal before its generated final action") + assert(finding.minimizedPath.length <= finding.originalPath.length) + assert.deepEqual(finding.replay.actions, finding.minimizedPath) + assert.equal(finding.replay.expectedFingerprint, finding.fingerprint) + assert.equal(finding.replay.expectedStateDigest, finding.stateDigest) +}) + +test("bounded stabilization captures delayed conditional fields and route state", async () => { + const run = await runFixture(delayedRouteFixture, { + seed: "delayed-route", + failOnFinding: false, + budgets: { maxActions: 20, maxStates: 12, maxTransitions: 20, maxDurationMs: 20_000 }, + actionFamilies: ["click", "fill"], + }) + const transition = run.result.transitions.find((candidate) => candidate.destinationUrl.endsWith("#configured")) + assert(transition, "route transition should be observed after bounded stabilization") + assert(transition.novelty.newDescriptors >= 1) + assert(transition.timing.polls >= 2) + const destination = run.result.states.find((state) => state.digest === transition.destinationDigest) + assert(destination?.descriptors.some((descriptor) => descriptor.selector === "#conditional-field")) +}) + +test("identical seed and DOM produce deterministic state and action graph identity", async () => { + const input = { + seed: "deterministic-graph", + failOnFinding: false, + budgets: { maxActions: 8, maxStates: 8, maxTransitions: 8, maxDurationMs: 15_000 }, + actionFamilies: ["click", "fill"], + } + const first = await runFixture(delayedRouteFixture, input) + const second = await runFixture(delayedRouteFixture, input) + const stableGraph = (result: typeof first.result) => ({ + states: result.states.map((state) => ({ digest: state.digest, descriptorDigest: state.descriptorDigest, descriptors: state.descriptors.map((descriptor) => descriptor.id) })), + transitions: result.transitions.map((transition) => ({ source: transition.sourceDigest, destination: transition.destinationDigest, action: transition.action.id, status: transition.status, newDescriptors: transition.novelty.newDescriptors })), + }) + assert.deepEqual(stableGraph(second.result), stableGraph(first.result)) +}) + +test("loops, budgets, cancellation, frames, and partial evidence remain bounded", async () => { + const loopFixture = `
External` + const bounded = await runFixture(loopFixture, { + seed: "loop-bound", + failOnFinding: false, + budgets: { maxActions: 4, maxStates: 3, maxTransitions: 4, maxDurationMs: 15_000, maxArtifactBytes: 200_000 }, + revisitPolicy: { maxStateVisits: 1, maxActionVisits: 1 }, + actionFamilies: ["click", "reload"], + }) + assert(bounded.result.summary.actions <= 4) + assert(bounded.result.summary.transitions <= 4) + assert(bounded.result.summary.states <= 3) + assert(bounded.result.states.some((state) => state.frames.some((frame) => frame.id === "frame:0"))) + assert(bounded.result.states.some((state) => state.descriptors.some((descriptor) => descriptor.frameId === "frame:0" && descriptor.selector === "#framed"))) + const repeated = bounded.result.states[0]!.descriptors.filter((descriptor) => descriptor.name === "query") + assert.equal(repeated.length, 2) + assert.equal(new Set(repeated.map((descriptor) => descriptor.selector)).size, 2, "repeated controls retain #2027 unique selectors") + assert(bounded.result.diagnostics.some((diagnostic) => diagnostic.code === "browser_adaptive_cross_origin_action_rejected")) + + const controller = new AbortController() + controller.abort("fixture cancellation") + const cancelled = await runFixture(loopFixture, { seed: "cancelled", failOnFinding: false }, controller.signal) + assert.equal(cancelled.result.status, "incomplete") + assert.equal(cancelled.result.summary.budgetExhausted, "cancelled") + assert.equal(cancelled.result.transitions.length, 0) + assert(cancelled.result.states.length >= 1, "partial initial-state evidence is retained") +}) + +test("no-reset exploration remains a truthful linear state chain", async () => { + const linear = await runFixture(``, { + seed: "linear-no-reset", + failOnFinding: false, + resetPolicy: { mode: "none" }, + revisitPolicy: { maxStateVisits: 2, maxActionVisits: 1 }, + budgets: { maxActions: 3, maxStates: 4, maxTransitions: 3, maxDurationMs: 10_000 }, + actionFamilies: ["click"], + }) + assert(linear.result.transitions.length >= 2) + for (let index = 1; index < linear.result.transitions.length; index += 1) { + assert.equal(linear.result.transitions[index]!.sourceDigest, linear.result.transitions[index - 1]!.destinationDigest) + } +}) + +async function runFixture(html: string, input: Record, signal?: AbortSignal) { + const browser = await chromium.launch({ headless: true }) + const page = await browser.newPage() + const consoleMessages: object[] = [] + const errors: object[] = [] + const network: object[] = [] + page.on("console", (message) => consoleMessages.push({ type: message.type(), text: message.text() })) + page.on("pageerror", (error) => errors.push({ message: error.message })) + page.on("request", (request) => network.push({ url: request.url(), method: request.method() })) + await page.addInitScript("globalThis.__name = value => value") + const startUrl = `data:text/html;charset=utf-8,${encodeURIComponent(html)}` + await page.goto(startUrl, { waitUntil: "load" }) + const contract = browserAdaptiveExplorationContract({ startUrl, stabilization: { pollIntervalMs: 25, quietWindowMs: 100, maxWaitMs: 1500, maxMutationRecords: 40 }, ...input }) + try { + const result = await exploreAdaptiveBrowserStateMachine({ page, baseUrl: startUrl, contract, observations: { consoleMessages, errors, network }, signal }) + return { result, contract } + } finally { + await browser.close() + } +} diff --git a/tests/public-api-contract.test.ts b/tests/public-api-contract.test.ts index 36e1bb498..7ecb30c7b 100644 --- a/tests/public-api-contract.test.ts +++ b/tests/public-api-contract.test.ts @@ -159,6 +159,7 @@ assert.deepEqual(barrelExportModules(publicBarrel), [ "./browser-artifact-lifecycle.js", "./browser-callback-contracts.js", "./browser-interaction.js", + "./browser-adaptive-exploration.js", "./browser-multi-actor-scenario-contracts.js", "./browser-probe-contract.js", "./browser-playground-session-run.js", @@ -242,6 +243,7 @@ assert.deepEqual(barrelExportModules(publicBarrel), [ assert.deepEqual(barrelExportModules(contractsBarrel), [ "./browser-probe-contract.js", "./browser-multi-actor-scenario-contracts.js", + "./browser-adaptive-exploration.js", "./command-registry.js", "./fuzz-fixture-plan-contracts.js", "./fuzz-coverage-plan-contracts.js", diff --git a/tests/recipe-validation-descriptors.test.ts b/tests/recipe-validation-descriptors.test.ts index ee192c66d..bbd814c42 100644 --- a/tests/recipe-validation-descriptors.test.ts +++ b/tests/recipe-validation-descriptors.test.ts @@ -27,7 +27,7 @@ await withTempDir("wp-codebox-recipe-validation-descriptors-", async (recipeDire { code: "invalid-capture", path: "$.workflow.steps[0].args", message: "wordpress.browser-probe capture does not support: bogus" }, ]) assert.deepEqual(issues.filter((issue) => issue.path === "$.workflow.steps[1].args"), [ - { code: "missing-steps", path: "$.workflow.steps[1].args", message: "wordpress.browser-actions requires steps-json= or url=." }, + { code: "missing-steps", path: "$.workflow.steps[1].args", message: "wordpress.browser-actions requires steps-json=, url=, or adaptive-exploration-json=." }, { code: "invalid-duration", path: "$.workflow.steps[1].args", message: "wordpress.browser-actions timeout must look like 500ms or 2s." }, { code: "invalid-capture", path: "$.workflow.steps[1].args", message: "wordpress.browser-actions capture does not support: bogus" }, ])