From 2cef6895f8df4e35914e4230ffac4602a11d002e Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Sat, 25 Jul 2026 15:45:40 +0000 Subject: [PATCH] feat: add browser accessibility fuzzing oracles --- README.md | 2 +- docs/adversarial-runtime.md | 38 +++ .../recipes/adversarial-adaptive-browser.json | 2 +- package.json | 1 + .../runtime-core/src/adversarial-browser.ts | 3 + .../runtime-core/src/browser-accessibility.ts | 167 +++++++++ .../src/browser-adaptive-exploration.ts | 18 +- packages/runtime-core/src/command-registry.ts | 2 +- packages/runtime-core/src/contracts.ts | 1 + packages/runtime-core/src/index.ts | 1 + packages/runtime-core/src/public.ts | 1 + .../src/browser-accessibility-collector.ts | 319 ++++++++++++++++++ .../src/browser-actions-runner.ts | 98 +++++- .../src/browser-adaptive-explorer.ts | 119 ++++++- .../src/browser-dom-snapshot.ts | 6 +- tests/adversarial-browser.test.ts | 11 + tests/browser-accessibility-oracles.test.ts | 287 ++++++++++++++++ 17 files changed, 1048 insertions(+), 28 deletions(-) create mode 100644 packages/runtime-core/src/browser-accessibility.ts create mode 100644 packages/runtime-playground/src/browser-accessibility-collector.ts create mode 100644 tests/browser-accessibility-oracles.test.ts diff --git a/README.md b/README.md index 17863a4e6..5c948b111 100644 --- a/README.md +++ b/README.md @@ -725,7 +725,7 @@ Supported runtime commands today: - `wordpress.server-page-load`: load `surface=admin|frontend` plus `path=` or `url=` through the live preview HTTP server without a browser. Its performance observation is marked `source=server-http` and `kind=server-page-load`. - `wordpress.browser-page-load`: load `surface=admin|frontend` plus `path=` or `url=` in Playwright by wrapping `wordpress.browser-probe`. - `wordpress.browser-probe`: boot the live preview, visit `url=` with Playwright, and capture generic browser replay/audit evidence under `files/browser/`. -- `wordpress.browser-actions`: boot the live preview, drive it with an ordered interaction script (`steps-json`), assert browser behavior, and capture replay/audit evidence under `files/browser/`. +- `wordpress.browser-actions`: boot the live preview, drive it with an ordered interaction script (`steps-json`) or bounded adaptive exploration, optionally classify accessibility/keyboard/focus failures, and capture replay/audit evidence under `files/browser/`. `wordpress.run-php` loads `/wordpress/wp-load.php` by default. Use `--arg bootstrap=none` for raw PHP. diff --git a/docs/adversarial-runtime.md b/docs/adversarial-runtime.md index 175bb2729..abaa699f1 100644 --- a/docs/adversarial-runtime.md +++ b/docs/adversarial-runtime.md @@ -16,6 +16,10 @@ and replay surfaces; they do not replace those surfaces. does not assume a browser, HTTP library, application, or transport. - `adversarial-browser.ts` owns DOM-derived journey planning, hostile generic input generation, journey minimization, and user-facing oracle contracts. +- `browser-accessibility.ts` owns provider-neutral accessibility configuration, + normalized findings, capability status, and stable privacy-safe fingerprints. + The Playground adapter collects bounded DOM, focus, keyboard, geometry, and + redacted ARIA-tree evidence through the existing adaptive browser runner. - `adversarial-artifacts.ts` writes bounded manifested finding and replay bundles, removes declared secrets and machine-specific paths, and seals corpus/finding identity with a deterministic content digest. @@ -166,6 +170,40 @@ controls are repeated to expose duplicate side effects. Generic oracles cover: Journey minimization uses deterministic subset replay to retain the shortest sequence that preserves the oracle failure. +### Accessibility and keyboard exploration + +Add `accessibility` to `adaptive-exploration-json` to scan the initial state, +each novel state, and the final state. The contract accepts `ruleTags`, +`includeScopes`, `excludeScopes`, `impactThreshold`, `cadence`, required or +optional `rules`, `focus`, and `accessibilityTree` capabilities, plus explicit +scan, violation, focus-history, tree-size, and keyboard-action budgets. + +```json +{ + "schema": "wp-codebox/browser-adaptive-exploration/v1", + "seed": "accessible-settings", + "startUrl": "/settings/", + "accessibility": { + "ruleTags": ["accessible-name", "keyboard-reachable", "focus-visible", "dialog-focus", "aria-state"], + "impactThreshold": "serious", + "cadence": ["initial", "novel-state", "final"], + "capabilities": { "rules": "required", "focus": "required", "accessibilityTree": "optional" }, + "budgets": { "maxScans": 24, "maxViolationsPerScan": 25, "maxFocusTransitions": 100, "maxTreeChars": 20000, "maxKeyboardActions": 8 } + } +} +``` + +The collector classifies unnamed controls, keyboard-unreachable custom +controls, unexpected tab stops, hidden/inert/clipped/offscreen focus, focus loss, +dialog entry/containment/restoration failures, and observable ARIA state drift. +Every finding records its oracle, rule, impact, structural target, state and +transition context, screenshot, correlated bounded DOM snapshot, and stable +fingerprint. ARIA-tree names are redacted and target evidence excludes text. +Unavailable tree capabilities produce an explicit `unsupported` scan. Required +capabilities make the exploration incomplete; optional capabilities allow the +campaign to continue without turning the unsupported check into a pass. +Existing caller-supplied `accessibilityViolations` remain supported. + ## Evidence and safety `writeAdversarialEvidenceBundle()` writes a manifest, campaign result, one diff --git a/examples/recipes/adversarial-adaptive-browser.json b/examples/recipes/adversarial-adaptive-browser.json index 08e68ba50..bc51f9c2c 100644 --- a/examples/recipes/adversarial-adaptive-browser.json +++ b/examples/recipes/adversarial-adaptive-browser.json @@ -34,7 +34,7 @@ { "command": "wordpress.browser-actions", "args": [ - "adaptive-exploration-json={\"schema\":\"wp-codebox/browser-adaptive-exploration/v1\",\"seed\":\"adaptive-browser-v1\",\"startUrl\":\"/\",\"budgets\":{\"maxActions\":24,\"maxStates\":16,\"maxTransitions\":32,\"maxDurationMs\":60000,\"maxArtifactBytes\":2097152,\"maxErrors\":10},\"actionFamilies\":[\"click\",\"fill\",\"select\",\"submit\",\"keyboard\",\"back\",\"reload\",\"repeat\",\"double-submit\"],\"resetPolicy\":{\"mode\":\"start-url\"}}", + "adaptive-exploration-json={\"schema\":\"wp-codebox/browser-adaptive-exploration/v1\",\"seed\":\"adaptive-browser-v1\",\"startUrl\":\"/\",\"budgets\":{\"maxActions\":24,\"maxStates\":16,\"maxTransitions\":32,\"maxDurationMs\":60000,\"maxArtifactBytes\":2097152,\"maxErrors\":10},\"actionFamilies\":[\"click\",\"fill\",\"select\",\"submit\",\"keyboard\",\"back\",\"reload\",\"repeat\",\"double-submit\"],\"resetPolicy\":{\"mode\":\"start-url\"},\"accessibility\":{\"ruleTags\":[\"accessible-name\",\"keyboard-reachable\",\"focus-visible\",\"dialog-focus\",\"aria-state\"],\"impactThreshold\":\"serious\",\"cadence\":[\"initial\",\"novel-state\",\"final\"],\"capabilities\":{\"rules\":\"required\",\"focus\":\"required\",\"accessibilityTree\":\"optional\"},\"budgets\":{\"maxScans\":24,\"maxViolationsPerScan\":25,\"maxFocusTransitions\":100,\"maxTreeChars\":20000,\"maxKeyboardActions\":8}}}", "capture=steps,console,errors,network" ] } diff --git a/package.json b/package.json index 54f0d9473..56c51af15 100644 --- a/package.json +++ b/package.json @@ -232,6 +232,7 @@ "test:fuzz-run-recipe": "tsx tests/fuzz-run-recipe.test.ts", "test:fuzz-suite-runner": "tsx tests/fuzz-suite-runner.test.ts", "test:adversarial-runtime": "tsx --test tests/transport-faults.test.ts tests/adversarial-campaign.test.ts tests/adversarial-browser.test.ts tests/browser-clock-control.test.ts tests/adversarial-recipe-orchestration.test.ts tests/wordpress-adversarial-adapter.test.ts", + "test:browser-accessibility-oracles": "tsx --test tests/browser-accessibility-oracles.test.ts tests/browser-adaptive-exploration.test.ts tests/adversarial-browser.test.ts", "test:wordpress-adversarial-adapter": "tsx tests/wordpress-adversarial-adapter.test.ts", "test:playground-fuzz-suite-public": "tsx tests/playground-fuzz-suite-public.test.ts", "test:nested-fuzz-suite-recipe-command": "tsx tests/nested-fuzz-suite-recipe-command.test.ts", diff --git a/packages/runtime-core/src/adversarial-browser.ts b/packages/runtime-core/src/adversarial-browser.ts index e90656317..4e9b62296 100644 --- a/packages/runtime-core/src/adversarial-browser.ts +++ b/packages/runtime-core/src/adversarial-browser.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto" import type { BrowserActionCorpusDescriptor, BrowserInteractionStep } from "./browser-interaction.js" +import type { BrowserAccessibilityFinding } from "./browser-accessibility.js" export const ADVERSARIAL_BROWSER_PLAN_SCHEMA = "wp-codebox/adversarial-browser-plan/v1" as const export const ADVERSARIAL_BROWSER_ORACLE_RESULT_SCHEMA = "wp-codebox/adversarial-browser-oracle-result/v1" as const @@ -26,6 +27,7 @@ export interface AdversarialBrowserObservation { loadingIndicators?: Array<{ id: string; visibleForMs: number }> boxes?: Array<{ id: string; x: number; y: number; width: number; height: number; viewportWidth: number; viewportHeight: number; focused?: boolean; visible?: boolean; clipped?: boolean }> accessibilityViolations?: Array<{ rule: string; target: string; impact?: string }> + accessibilityFindings?: BrowserAccessibilityFinding[] effects?: Array<{ id: string; count: number; expectedMaximum?: number }> } @@ -91,6 +93,7 @@ export function evaluateAdversarialBrowserOracles(observation: AdversarialBrowse if (box.focused && box.visible === false) failures.push({ oracle: "layout", code: "browser-invisible-focus", message: `Focused element ${box.id} is not visible.`, target: box.id }) } for (const violation of observation.accessibilityViolations ?? []) failures.push({ oracle: "accessibility", code: `browser-a11y-${violation.rule}`, message: `Accessibility rule ${violation.rule} failed.`, target: violation.target }) + for (const finding of observation.accessibilityFindings ?? []) failures.push({ oracle: finding.oracle === "accessibility" ? "accessibility" : finding.oracle === "focus" ? "layout" : "dead-control", code: finding.code, message: `${finding.classification} (${finding.impact}).`, target: finding.target.locator }) for (const effect of observation.effects ?? []) { if (effect.count > (effect.expectedMaximum ?? 1)) failures.push({ oracle: "duplicate-effect", code: "browser-duplicate-effect", message: `Effect ${effect.id} occurred ${effect.count} times.`, target: effect.id }) } diff --git a/packages/runtime-core/src/browser-accessibility.ts b/packages/runtime-core/src/browser-accessibility.ts new file mode 100644 index 000000000..92ece08ab --- /dev/null +++ b/packages/runtime-core/src/browser-accessibility.ts @@ -0,0 +1,167 @@ +import { createHash } from "node:crypto" + +import type { BrowserAdaptiveAction } from "./browser-adaptive-exploration.js" +import { isPlainObject, stableJson, stripUndefined } from "./object-utils.js" + +export const BROWSER_ACCESSIBILITY_SCHEMA = "wp-codebox/browser-accessibility/v1" as const +export const BROWSER_ACCESSIBILITY_RULES = [ + "accessible-name", + "keyboard-reachable", + "tab-order", + "focus-visible", + "focus-loss", + "dialog-focus", + "aria-state", +] as const + +export type BrowserAccessibilityRule = typeof BROWSER_ACCESSIBILITY_RULES[number] +export type BrowserAccessibilityImpact = "minor" | "moderate" | "serious" | "critical" +export type BrowserAccessibilityScanPhase = "initial" | "novel-state" | "final" | "replay" + +export interface BrowserAccessibilityContract { + schema: typeof BROWSER_ACCESSIBILITY_SCHEMA + ruleTags: BrowserAccessibilityRule[] + includeScopes: string[] + excludeScopes: string[] + impactThreshold: BrowserAccessibilityImpact + cadence: Array<"initial" | "novel-state" | "final"> + capabilities: { + rules: "required" | "optional" + focus: "required" | "optional" + accessibilityTree: "required" | "optional" | "disabled" + } + budgets: { + maxScans: number + maxViolationsPerScan: number + maxTargetsPerViolation: number + maxFocusTransitions: number + maxTreeChars: number + maxKeyboardActions: number + maxFrames: number + } +} + +export interface BrowserAccessibilityTargetEvidence { + locator: string + frameId: string + tag: string + role?: string + states?: Partial> + box?: { x: number; y: number; width: number; height: number; viewportWidth: number; viewportHeight: number } +} + +export interface BrowserAccessibilityFinding { + oracle: "accessibility" | "keyboard" | "focus" + rule: BrowserAccessibilityRule + code: string + impact: BrowserAccessibilityImpact + classification: string + fingerprint: string + target: BrowserAccessibilityTargetEvidence + stateDigest?: string + transitionId?: string + actionId?: string + expected?: string + actual?: string +} + +export interface BrowserAccessibilityFocusTransition { + index: number + phase: BrowserAccessibilityScanPhase + actionId?: string + from?: string + to: string + visible: boolean + insideDialog: boolean +} + +export interface BrowserAccessibilityScan { + index: number + phase: BrowserAccessibilityScanPhase + status: "passed" | "findings" | "unsupported" | "inconclusive" + stateDigest?: string + transitionId?: string + actionId?: string + findings: BrowserAccessibilityFinding[] + accessibilityTree?: { status: "captured" | "unsupported" | "inconclusive"; snapshot?: string; reason?: string; truncated?: boolean } + diagnostics: Array<{ code: string; message: string }> + artifacts?: { screenshot?: string; domSnapshot?: string } +} + +export interface BrowserAccessibilityEvidence { + schema: typeof BROWSER_ACCESSIBILITY_SCHEMA + collector: { name: string; version: string; capabilities: { rules: "supported" | "unsupported"; focus: "supported" | "unsupported"; accessibilityTree: "supported" | "unsupported" } } + contract: BrowserAccessibilityContract + scans: BrowserAccessibilityScan[] + focusHistory: BrowserAccessibilityFocusTransition[] + diagnostics: Array<{ code: string; message: string }> + summary: { scans: number; findings: number; passed: number; unsupported: number; inconclusive: number; truncated: boolean } +} + +export interface BrowserAccessibilityCollector { + beforeAction(action: BrowserAdaptiveAction): Promise + scan(input: { phase: BrowserAccessibilityScanPhase; stateDigest?: string; transitionId?: string; action?: BrowserAdaptiveAction; record?: boolean }): Promise + reset(): Promise + evidence(): BrowserAccessibilityEvidence +} + +export function browserAccessibilityContract(input: unknown): BrowserAccessibilityContract | undefined { + if (!isPlainObject(input) || input.enabled === false) return undefined + const budgets = object(input.budgets) + const capabilities = object(input.capabilities) + const requestedRules = array(input.ruleTags ?? input.rule_tags).filter((value): value is BrowserAccessibilityRule => (BROWSER_ACCESSIBILITY_RULES as readonly unknown[]).includes(value)) + const requestedCadence = array(input.cadence).filter((value): value is "initial" | "novel-state" | "final" => value === "initial" || value === "novel-state" || value === "final") + return { + schema: BROWSER_ACCESSIBILITY_SCHEMA, + ruleTags: requestedRules.length > 0 ? [...new Set(requestedRules)] : [...BROWSER_ACCESSIBILITY_RULES], + includeScopes: boundedStrings(input.includeScopes ?? input.include_scopes, 20), + excludeScopes: boundedStrings(input.excludeScopes ?? input.exclude_scopes, 20), + impactThreshold: impact(input.impactThreshold ?? input.impact_threshold), + cadence: requestedCadence.length > 0 ? [...new Set(requestedCadence)] : ["initial", "novel-state", "final"], + capabilities: { + rules: requirement(capabilities.rules), + focus: requirement(capabilities.focus), + accessibilityTree: capabilities.accessibilityTree === "disabled" || capabilities.accessibility_tree === "disabled" ? "disabled" : requirement(capabilities.accessibilityTree ?? capabilities.accessibility_tree), + }, + budgets: { + maxScans: integer(budgets.maxScans ?? budgets.max_scans, 32, 1, 500), + maxViolationsPerScan: integer(budgets.maxViolationsPerScan ?? budgets.max_violations_per_scan, 25, 1, 200), + maxTargetsPerViolation: integer(budgets.maxTargetsPerViolation ?? budgets.max_targets_per_violation, 3, 1, 20), + maxFocusTransitions: integer(budgets.maxFocusTransitions ?? budgets.max_focus_transitions, 100, 1, 1_000), + maxTreeChars: integer(budgets.maxTreeChars ?? budgets.max_tree_chars, 20_000, 256, 200_000), + maxKeyboardActions: integer(budgets.maxKeyboardActions ?? budgets.max_keyboard_actions, 12, 0, 50), + maxFrames: integer(budgets.maxFrames ?? budgets.max_frames, 16, 1, 100), + }, + } +} + +export function browserAccessibilityFindingFingerprint(finding: Omit): string { + const stable = stripUndefined({ + oracle: finding.oracle, + rule: finding.rule, + code: finding.code, + impact: finding.impact, + classification: finding.classification, + target: { + locator: normalizeLocator(finding.target.locator), + frameId: finding.target.frameId, + tag: finding.target.tag, + role: finding.target.role, + states: finding.target.states, + }, + expected: finding.expected, + actual: finding.actual, + }) + return createHash("sha256").update("wp-codebox/browser-accessibility-finding/v1\n").update(stableJson(stable)).digest("hex") +} + +function normalizeLocator(value: string): string { + return value.replace(/([_:-](?:[a-z]{0,4})?)[a-f0-9]{8,}/gi, "$1").slice(0, 240) +} + +function object(value: unknown): Record { return isPlainObject(value) ? value : {} } +function array(value: unknown): unknown[] { return Array.isArray(value) ? value : [] } +function boundedStrings(value: unknown, maximum: number): string[] { return array(value).filter((item): item is string => typeof item === "string" && item.trim().length > 0).slice(0, maximum).map((item) => item.trim().slice(0, 240)) } +function requirement(value: unknown): "required" | "optional" { return value === "required" ? "required" : "optional" } +function impact(value: unknown): BrowserAccessibilityImpact { return value === "minor" || value === "moderate" || value === "critical" ? value : "serious" } +function integer(value: unknown, fallback: number, minimum: number, maximum: number): number { const numeric = Number(value); return Number.isFinite(numeric) ? Math.max(minimum, Math.min(maximum, Math.floor(numeric))) : fallback } diff --git a/packages/runtime-core/src/browser-adaptive-exploration.ts b/packages/runtime-core/src/browser-adaptive-exploration.ts index df9ed8f94..ac7574199 100644 --- a/packages/runtime-core/src/browser-adaptive-exploration.ts +++ b/packages/runtime-core/src/browser-adaptive-exploration.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto" import type { BrowserActionCorpusDescriptor, BrowserInteractionStep, BrowserRandomWalkContext } from "./browser-interaction.js" +import { browserAccessibilityContract, type BrowserAccessibilityContract, type BrowserAccessibilityEvidence } from "./browser-accessibility.js" import { isPlainObject, stableJson, stripUndefined } from "./object-utils.js" export const BROWSER_ADAPTIVE_EXPLORATION_SCHEMA = "wp-codebox/browser-adaptive-exploration/v1" as const @@ -28,6 +29,7 @@ export interface BrowserAdaptiveExplorationContract { descriptorLimits: { maxPerState: number; maxDiagnostics: number; maxTextLength: number } stabilization: { pollIntervalMs: number; quietWindowMs: number; maxWaitMs: number; maxMutationRecords: number } failOnFinding: boolean + accessibility?: BrowserAccessibilityContract metadata?: Record } @@ -78,6 +80,7 @@ export interface BrowserAdaptiveTransition { loadingBefore: number loadingAfter: number oracleFingerprints: string[] + accessibilityFindingFingerprints?: string[] } status: "ok" | "revisited" | "rejected" | "error" | "cancelled" diagnostic?: { code: string; message: string } @@ -108,6 +111,7 @@ export interface BrowserAdaptiveExplorationResult { states: BrowserAdaptiveState[] transitions: BrowserAdaptiveTransition[] findings: BrowserAdaptiveFinding[] + accessibility?: BrowserAccessibilityEvidence diagnostics: Array<{ code: string; message: string; metadata?: Record }> summary: { actions: number @@ -116,7 +120,7 @@ export interface BrowserAdaptiveExplorationResult { revisits: number errors: number findings: number - budgetExhausted?: keyof BrowserAdaptiveExplorationContract["budgets"] | "cancelled" | "frontier" + budgetExhausted?: keyof BrowserAdaptiveExplorationContract["budgets"] | "maxKeyboardActions" | "cancelled" | "frontier" } replay: { schema: typeof BROWSER_ADAPTIVE_EXPLORATION_SCHEMA; seed: string; startUrl: string; contract: BrowserAdaptiveExplorationContract } } @@ -135,6 +139,7 @@ export function browserAdaptiveExplorationContract(input: Record 0 ? actionFamilies : [...BROWSER_ADAPTIVE_ACTION_FAMILIES], @@ -171,6 +176,7 @@ export function browserAdaptiveExplorationContract(input: Record ({ kind: "press" as const, key })) + actions.push({ id: `keyboard:document:${keys.join(">")}`, family: "keyboard", frameId: "document", steps }) + } + } add("back", undefined, []) add("reload", undefined, []) return actions diff --git a/packages/runtime-core/src/command-registry.ts b/packages/runtime-core/src/command-registry.ts index 76bb374e0..3f6e06ce3 100644 --- a/packages/runtime-core/src/command-registry.ts +++ b/packages/runtime-core/src/command-registry.ts @@ -1177,7 +1177,7 @@ export const commandRegistry = [ { name: "url", description: "Initial preview path or absolute URL to visit when the script omits an initial navigate step.", format: "path or URL" }, { name: "steps-json", description: "Ordered interaction script: navigate, click, fill, type, press, drag, hover, select, waitFor, evaluate, expect, screenshot, and capture steps. waitFor and screenshot steps support generic painted-readiness waits: painted, frame-painted:, and frame-url-painted:.", format: "JSON array (inline or @)" }, { name: "action-corpus-json", description: "Optional wp-codebox/browser-action-corpus/v1 object. The runtime loads the start URL, discovers visible links, buttons, inputs, textareas, and selects, creates deterministic seeded fill/click/select steps from stable descriptors, and writes replayable corpus artifacts.", format: "JSON object" }, - { name: "adaptive-exploration-json", description: "Optional standalone wp-codebox/browser-adaptive-exploration/v1 mode. Repeatedly stabilizes and rediscovers unique controls while building a bounded deterministic state graph with replay and minimized finding evidence; cannot be combined with url, steps-json, or action-corpus-json.", format: "JSON object" }, + { name: "adaptive-exploration-json", description: "Optional standalone wp-codebox/browser-adaptive-exploration/v1 mode. Repeatedly stabilizes and rediscovers unique controls while building a bounded deterministic state graph with replay, minimized finding evidence, and optional accessibility/keyboard/focus oracles; cannot be combined with url, steps-json, or action-corpus-json.", format: "JSON object" }, { name: "step-timeout", description: "Per-step timeout applied to each interaction step.", format: "duration, e.g. 5s or 500ms" }, { name: "timeout", description: "Total-script timeout bounding the whole interaction run.", format: "duration, e.g. 30s or 1500ms" }, { name: "auth", description: "Optional in-memory browser authentication mode. Use wordpress-admin to bootstrap WordPress admin cookies from PHP without writing token-bearing storage-state artifacts.", format: "wordpress-admin" }, diff --git a/packages/runtime-core/src/contracts.ts b/packages/runtime-core/src/contracts.ts index b09149656..38d19eb0d 100644 --- a/packages/runtime-core/src/contracts.ts +++ b/packages/runtime-core/src/contracts.ts @@ -1,5 +1,6 @@ /** Inspectable Codebox contract metadata for CLI and orchestrator consumers. */ export * from "./browser-probe-contract.js" +export * from "./browser-accessibility.js" export * from "./browser-multi-actor-scenario-contracts.js" export * from "./browser-adaptive-exploration.js" export * from "./command-registry.js" diff --git a/packages/runtime-core/src/index.ts b/packages/runtime-core/src/index.ts index 805f21401..0257cae12 100644 --- a/packages/runtime-core/src/index.ts +++ b/packages/runtime-core/src/index.ts @@ -48,6 +48,7 @@ export * from "./command-diagnostics.js" export * from "./command-agent-run.js" export * from "./php-worker-runner.js" export * from "./browser-interaction.js" +export * from "./browser-accessibility.js" export * from "./browser-adaptive-exploration.js" export * from "./browser-multi-actor-scenario-contracts.js" export * from "./browser-probe-contract.js" diff --git a/packages/runtime-core/src/public.ts b/packages/runtime-core/src/public.ts index a095f69a2..ee4f27da1 100644 --- a/packages/runtime-core/src/public.ts +++ b/packages/runtime-core/src/public.ts @@ -24,6 +24,7 @@ export * from "./artifact-test-results.js" export * from "./browser-artifact-lifecycle.js" export * from "./browser-callback-contracts.js" export * from "./browser-interaction.js" +export * from "./browser-accessibility.js" export * from "./browser-adaptive-exploration.js" export * from "./browser-multi-actor-scenario-contracts.js" export * from "./browser-probe-contract.js" diff --git a/packages/runtime-playground/src/browser-accessibility-collector.ts b/packages/runtime-playground/src/browser-accessibility-collector.ts new file mode 100644 index 000000000..6ae5532c5 --- /dev/null +++ b/packages/runtime-playground/src/browser-accessibility-collector.ts @@ -0,0 +1,319 @@ +import { + BROWSER_ACCESSIBILITY_SCHEMA, + browserAccessibilityFindingFingerprint, + type BrowserAccessibilityCollector, + type BrowserAccessibilityContract, + type BrowserAccessibilityEvidence, + type BrowserAccessibilityFinding, + type BrowserAccessibilityScan, + type BrowserAccessibilityScanPhase, + type BrowserAdaptiveAction, +} from "@automattic/wp-codebox-core" +import type { Frame, Page } from "playwright" + +interface PageAccessibilityState { + url: string + focusedDocument?: boolean + active: ElementEvidence + dialogs: ElementEvidence[] + findings: Array> + diagnostics: Array<{ code: string; message: string }> +} + +interface ElementEvidence { + locator: string + frameId: string + tag: string + role?: string + visible: boolean + insideDialog: boolean + states?: BrowserAccessibilityFinding["target"]["states"] + box?: BrowserAccessibilityFinding["target"]["box"] +} + +export function createBrowserAccessibilityCollector(page: Page, contract: BrowserAccessibilityContract): BrowserAccessibilityCollector { + const scans: BrowserAccessibilityScan[] = [] + const focusHistory: BrowserAccessibilityEvidence["focusHistory"] = [] + const diagnostics: BrowserAccessibilityEvidence["diagnostics"] = [] + const dialogTriggers = new Map() + let before: PageAccessibilityState | undefined + let previousActive: string | undefined + let currentAction: BrowserAdaptiveAction | undefined + let truncated = false + let scanAttempts = 0 + + return { + async beforeAction(action) { + currentAction = action + before = await inspectPage(page, contract) + }, + async scan(input) { + if (scanAttempts >= contract.budgets.maxScans) { + truncated = true + if (!diagnostics.some((item) => item.code === "browser_accessibility_scan_budget_exhausted")) diagnostics.push({ code: "browser_accessibility_scan_budget_exhausted", message: "The accessibility scan budget was exhausted." }) + return { index: scanAttempts, phase: input.phase, status: "inconclusive", stateDigest: input.stateDigest, transitionId: input.transitionId, actionId: input.action?.id, findings: [], diagnostics: [{ code: "browser_accessibility_scan_budget_exhausted", message: "The accessibility scan budget was exhausted." }] } + } + scanAttempts += 1 + const state = await inspectPage(page, contract) + const findings = [...state.findings] + const action = input.action ?? currentAction + + if (contract.ruleTags.includes("focus-loss") && action && state.url === before?.url && state.active.locator === "body" && before.active.locator !== "body") { + findings.push(finding("focus", "focus-loss", "browser-focus-lost", "serious", "focus-lost-to-document", state.active, "an interaction target", "document body")) + } + + if (contract.ruleTags.includes("dialog-focus")) { + const beforeDialogs = new Set((before?.dialogs ?? []).map((dialog) => dialog.locator)) + const currentDialogs = new Set(state.dialogs.map((dialog) => dialog.locator)) + for (const dialog of state.dialogs) { + if (!beforeDialogs.has(dialog.locator) && action) dialogTriggers.set(dialog.locator, before?.active.locator ?? "body") + if (!state.active.insideDialog) { + findings.push(finding("focus", "dialog-focus", "browser-dialog-focus-entry", "serious", "dialog-focus-not-contained", dialog, "focus inside the open dialog", state.active.locator)) + } + } + for (const dialog of before?.dialogs ?? []) { + if (currentDialogs.has(dialog.locator)) continue + const trigger = dialogTriggers.get(dialog.locator) + if (trigger && state.active.locator !== trigger) { + findings.push(finding("focus", "dialog-focus", "browser-dialog-focus-restoration", "serious", "dialog-focus-not-restored", state.active, trigger, state.active.locator)) + } + dialogTriggers.delete(dialog.locator) + } + } + + const retained = findings + .filter((item) => impactRank(item.impact) >= impactRank(contract.impactThreshold)) + .slice(0, contract.budgets.maxViolationsPerScan) + .map((item) => { + const contextual = { ...item, stateDigest: input.stateDigest, transitionId: input.transitionId, actionId: action?.id } + return { ...contextual, fingerprint: browserAccessibilityFindingFingerprint(contextual) } + }) + if (findings.length > retained.length) truncated = true + + if (state.active.locator !== previousActive && focusHistory.length < contract.budgets.maxFocusTransitions) { + focusHistory.push({ index: focusHistory.length, phase: input.phase, actionId: action?.id, from: previousActive, to: state.active.locator, visible: state.active.visible, insideDialog: state.active.insideDialog }) + } else if (state.active.locator !== previousActive) { + truncated = true + } + previousActive = state.active.locator + + const tree = await accessibilityTree(page, contract) + const requiredUnavailable = tree.status !== "captured" && contract.capabilities.accessibilityTree === "required" + const treeUnavailable = tree.status !== "captured" && contract.capabilities.accessibilityTree !== "disabled" + const status: BrowserAccessibilityScan["status"] = retained.length > 0 ? "findings" : treeUnavailable ? "unsupported" : state.diagnostics.some((item) => item.code === "browser_accessibility_scope_invalid") ? "inconclusive" : "passed" + const scan: BrowserAccessibilityScan = { + index: scanAttempts - 1, + phase: input.phase, + status, + stateDigest: input.stateDigest, + transitionId: input.transitionId, + actionId: action?.id, + findings: retained, + accessibilityTree: tree, + diagnostics: [...state.diagnostics, ...(requiredUnavailable ? [{ code: "browser_accessibility_tree_required_unavailable", message: "The required accessibility-tree capability was unavailable." }] : [])], + } + if (input.record !== false) scans.push(scan) + before = state + currentAction = undefined + return scan + }, + async reset() { + before = undefined + previousActive = undefined + currentAction = undefined + dialogTriggers.clear() + }, + evidence() { + const findingCount = scans.reduce((count, scan) => count + scan.findings.length, 0) + return { + schema: BROWSER_ACCESSIBILITY_SCHEMA, + collector: { + name: "playground-browser-accessibility", + version: "1", + capabilities: { + rules: "supported", + focus: "supported", + accessibilityTree: scans.some((scan) => scan.accessibilityTree?.status === "captured") ? "supported" : "unsupported", + }, + }, + contract, + scans, + focusHistory, + diagnostics, + summary: { + scans: scanAttempts, + findings: findingCount, + passed: scans.filter((scan) => scan.status === "passed").length, + unsupported: scans.filter((scan) => scan.status === "unsupported").length, + inconclusive: scans.filter((scan) => scan.status === "inconclusive").length, + truncated, + }, + } + }, + } +} + +async function inspectPage(page: Page, contract: BrowserAccessibilityContract): Promise { + await page.evaluate("globalThis.__name ||= value => value") + const states: PageAccessibilityState[] = [] + const diagnostics: PageAccessibilityState["diagnostics"] = [] + for (const identity of accessibilityFrameIdentities(page, contract.budgets.maxFrames)) { + try { + await identity.frame.evaluate("globalThis.__name ||= value => value") + states.push(await inspectFrame(identity.frame, identity.id, contract)) + } catch { + diagnostics.push({ code: "browser_accessibility_frame_unsupported", message: `Frame ${identity.id} could not be inspected.` }) + } + } + const main = states.find((state) => state.url === page.url()) ?? states[0] + const focused = [...states].reverse().find((state) => state.focusedDocument) + return { + url: page.url(), + active: focused?.active ?? main?.active ?? { locator: "body", frameId: "document", tag: "body", visible: true, insideDialog: false }, + dialogs: states.flatMap((state) => state.dialogs), + findings: states.flatMap((state) => state.findings).slice(0, contract.budgets.maxViolationsPerScan * contract.budgets.maxTargetsPerViolation), + diagnostics: [...states.flatMap((state) => state.diagnostics), ...diagnostics], + } +} + +async function inspectFrame(frame: Frame, frameId: string, contract: BrowserAccessibilityContract): Promise { + return frame.evaluate(({ includeScopes, excludeScopes, rules, maxTargets, frameId }) => { + const bounded = (value: string) => value.replace(/([_:-](?:[a-z]{0,4})?)[a-f0-9]{8,}$/i, "$1").slice(0, 240) + const path = (element: Element | null): string => { + if (!element || element === document.documentElement || element === document.body) return element === document.body ? "body" : "document" + const parts: string[] = [] + let current: Element | null = element + while (current && current !== document.body && parts.length < 6) { + let part = current.tagName.toLowerCase() + const role = current.hasAttribute("role") + if (role) part += "[role]" + const parent: Element | null = current.parentElement + if (parent) { + const peers = Array.from(parent.children).filter((child) => child.tagName === current?.tagName && child.hasAttribute("role") === role) + if (peers.length > 1) part += `:nth-of-type(${Array.from(parent.children).filter((child) => child.tagName === current?.tagName).indexOf(current) + 1})` + } + parts.unshift(part) + current = parent + } + return bounded(parts.join(" > ") || element.tagName.toLowerCase()) + } + const rendered = (element: Element) => { + const rect = element.getBoundingClientRect() + const style = getComputedStyle(element) + const hiddenAncestor = element.closest("[hidden],[inert],[aria-hidden='true']") + return !hiddenAncestor && rect.width > 0 && rect.height > 0 && style.display !== "none" && style.visibility !== "hidden" && style.opacity !== "0" + } + const visible = (element: Element) => { const rect = element.getBoundingClientRect(); return rendered(element) && rect.bottom > 0 && rect.right > 0 && rect.top < innerHeight && rect.left < innerWidth } + const role = (element: Element) => element.getAttribute("role")?.split(/\s+/).find((candidate) => validRoles.has(candidate)) || ({ BUTTON: "button", SUMMARY: "button", A: (element as HTMLAnchorElement).href ? "link" : undefined, INPUT: (element as HTMLInputElement).type === "checkbox" ? "checkbox" : (element as HTMLInputElement).type === "radio" ? "radio" : "textbox", SELECT: "combobox", TEXTAREA: "textbox" } as Record)[element.tagName] + const evidence = (element: Element | null): ElementEvidence => { + const target = element ?? document.body + const rect = target.getBoundingClientRect() + const states: NonNullable = Object.fromEntries(["expanded", "selected", "checked", "busy", "pressed", "hidden"].flatMap((name) => { + const value = target.getAttribute(`aria-${name}`) + return value === null ? [] : [[name, value.slice(0, 120)]] + })) + if (target.closest("[inert]")) states.inert = "true" + return { locator: path(target), frameId, tag: target.tagName.toLowerCase(), role: role(target), visible: visible(target), insideDialog: Boolean(target.closest("dialog,[role='dialog'],[role='alertdialog']")), states, box: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height), viewportWidth: innerWidth, viewportHeight: innerHeight } } + } + const invalidScopes = [...includeScopes, ...excludeScopes].filter((selector) => { try { document.querySelector(selector); return false } catch { return true } }) + const inScope = (element: Element) => { + const included = includeScopes.length === 0 || includeScopes.some((selector) => { try { return element.matches(selector) || Boolean(element.closest(selector)) } catch { return false } }) + const excluded = excludeScopes.some((selector) => { try { return element.matches(selector) || Boolean(element.closest(selector)) } catch { return false } }) + return included && !excluded + } + const name = (element: Element) => { + const labelledBy = element.getAttribute("aria-labelledby")?.split(/\s+/).map((id) => document.getElementById(id)?.textContent || "").join(" ").trim() + if (labelledBy) return labelledBy + const aria = element.getAttribute("aria-label")?.trim() + if (aria) return aria + const title = element.getAttribute("title")?.trim() + if (title) return title + const input = element as HTMLInputElement + if (input.labels && Array.from(input.labels).some((label) => (label.textContent || "").trim())) return "" + if (element.tagName === "INPUT" && ["submit", "reset", "button"].includes(input.type)) return input.value || (input.type === "submit" ? "Submit" : input.type === "reset" ? "Reset" : "") + if (element.tagName === "INPUT" && input.type === "image") return input.alt + const descendantAlt = element.querySelector("img[alt]")?.getAttribute("alt")?.trim() + if (descendantAlt) return descendantAlt + const svgTitle = element.querySelector("svg title")?.textContent?.trim() + if (svgTitle) return svgTitle + const semanticRole = role(element) + return semanticRole && new Set(["button", "link", "checkbox", "radio", "switch", "tab", "option", "menuitem"]).has(semanticRole) ? ((element as HTMLElement).innerText || "").replace(/\s+/g, " ").trim() : "" + } + const interactiveRoles = new Set(["button", "link", "checkbox", "radio", "switch", "tab", "option", "menuitem", "combobox", "textbox", "slider", "spinbutton"]) + const validRoles = new Set(["alert", "alertdialog", "application", "article", "banner", "blockquote", "button", "caption", "cell", "checkbox", "code", "columnheader", "combobox", "complementary", "contentinfo", "definition", "deletion", "dialog", "directory", "document", "emphasis", "feed", "figure", "form", "generic", "grid", "gridcell", "group", "heading", "img", "insertion", "link", "list", "listbox", "listitem", "log", "main", "marquee", "math", "menu", "menubar", "menuitem", "menuitemcheckbox", "menuitemradio", "meter", "navigation", "none", "note", "option", "paragraph", "presentation", "progressbar", "radio", "radiogroup", "region", "row", "rowgroup", "rowheader", "scrollbar", "search", "searchbox", "separator", "slider", "spinbutton", "status", "strong", "subscript", "suggestion", "superscript", "switch", "tab", "table", "tablist", "tabpanel", "term", "textbox", "time", "timer", "toolbar", "tooltip", "tree", "treegrid", "treeitem"]) + const findings: PageAccessibilityState["findings"] = [] + const add = (oracle: BrowserAccessibilityFinding["oracle"], rule: BrowserAccessibilityFinding["rule"], code: string, impact: BrowserAccessibilityFinding["impact"], classification: string, element: Element, expected?: string, actual?: string) => { + if (findings.length < maxTargets && rules.includes(rule)) findings.push({ oracle, rule, code, impact, classification, target: evidence(element), expected: expected?.slice(0, 120), actual: actual?.slice(0, 120) }) + } + const elements = Array.from(document.querySelectorAll("button,a[href],input:not([type=hidden]),select,textarea,summary,[role],[tabindex],[aria-controls]" )).filter(inScope) + for (const element of elements) { + const semanticRole = role(element) + const explicitRole = element.getAttribute("role")?.trim() + const interactive = Boolean(semanticRole && interactiveRoles.has(semanticRole)) + if (explicitRole && !explicitRole.split(/\s+/).some((candidate) => validRoles.has(candidate))) add("accessibility", "accessible-name", "browser-role-invalid", "serious", "invalid-role-relationship", element, "valid ARIA role", explicitRole) + if (interactive && visible(element) && !name(element)) add("accessibility", "accessible-name", "browser-accessible-name-missing", "serious", "interactive-control-without-accessible-name", element, "non-empty accessible name", "missing") + const native = ["BUTTON", "A", "INPUT", "SELECT", "TEXTAREA", "SUMMARY"].includes(element.tagName) + if (interactive && !native && visible(element) && (element as HTMLElement).tabIndex < 0) add("keyboard", "keyboard-reachable", "browser-keyboard-unreachable", "serious", "actionable-element-unreachable-by-keyboard", element, "tabIndex >= 0", String((element as HTMLElement).tabIndex)) + if (!interactive && (element as HTMLElement).tabIndex >= 0) add("keyboard", "tab-order", "browser-unexpected-tab-stop", "moderate", "non-interactive-element-in-tab-order", element, "interactive semantics", semanticRole || "none") + const controls = element.getAttribute("aria-controls") + const expanded = element.getAttribute("aria-expanded") + if (controls && expanded !== null) { + if (expanded !== "true" && expanded !== "false") add("accessibility", "aria-state", "browser-aria-expanded-invalid", "serious", "invalid-aria-state-token", element, "true or false", expanded) + const controlled = controls.split(/\s+/).map((id) => document.getElementById(id)).filter((item): item is HTMLElement => Boolean(item)) + if (controlled.length === 0) add("accessibility", "aria-state", "browser-aria-controls-missing", "serious", "aria-relationship-target-missing", element, "at least one existing controlled target", "missing") + else { + const actual = controlled.some(rendered) + if ((expanded === "true") !== actual) add("accessibility", "aria-state", "browser-aria-expanded-drift", "serious", "aria-state-divergence", element, String(actual), expanded) + } + } + const ariaChecked = element.getAttribute("aria-checked") + if (ariaChecked !== null && !["true", "false", "mixed"].includes(ariaChecked)) add("accessibility", "aria-state", "browser-aria-checked-invalid", "serious", "invalid-aria-state-token", element, "true, false, or mixed", ariaChecked) + if (ariaChecked !== null && "checked" in element && String((element as HTMLInputElement).checked) !== ariaChecked) add("accessibility", "aria-state", "browser-aria-checked-drift", "serious", "aria-state-divergence", element, String((element as HTMLInputElement).checked), ariaChecked) + const ariaSelected = element.getAttribute("aria-selected") + if (ariaSelected !== null && !["true", "false"].includes(ariaSelected)) add("accessibility", "aria-state", "browser-aria-selected-invalid", "serious", "invalid-aria-state-token", element, "true or false", ariaSelected) + if (ariaSelected !== null && "selected" in element && String((element as HTMLOptionElement).selected) !== ariaSelected) add("accessibility", "aria-state", "browser-aria-selected-drift", "serious", "aria-state-divergence", element, String((element as HTMLOptionElement).selected), ariaSelected) + for (const [attribute, allowed] of [["aria-busy", ["true", "false"]], ["aria-pressed", ["true", "false", "mixed"]]] as const) { + const value = element.getAttribute(attribute) + if (value !== null && !(allowed as readonly string[]).includes(value)) add("accessibility", "aria-state", `browser-${attribute}-invalid`, "serious", "invalid-aria-state-token", element, allowed.join(", "), value) + } + } + const active = evidence(document.activeElement) + if (rules.includes("focus-visible") && document.activeElement && document.activeElement !== document.body && !active.visible) findings.push({ oracle: "focus", rule: "focus-visible", code: "browser-focused-element-hidden", impact: "serious", classification: "focus-in-hidden-inert-or-offscreen-content", target: active, expected: "visible and operable", actual: active.states?.inert === "true" ? "inert" : "hidden, clipped, or offscreen" }) + const dialogs = Array.from(document.querySelectorAll("dialog,[role='dialog'],[role='alertdialog']")).filter((element) => { + let nativeModal = false + try { nativeModal = element.matches(":modal") } catch { nativeModal = element.tagName === "DIALOG" && (element as HTMLDialogElement).open } + return (nativeModal || element.getAttribute("aria-modal") === "true") && visible(element) && inScope(element) + }).map(evidence) + return { url: location.href, focusedDocument: document.hasFocus(), active, dialogs, findings, diagnostics: invalidScopes.length > 0 ? [{ code: "browser_accessibility_scope_invalid", message: "One or more accessibility scope selectors were invalid; the scan is inconclusive." }] : [] } + }, { includeScopes: contract.includeScopes, excludeScopes: contract.excludeScopes, rules: contract.ruleTags, maxTargets: contract.budgets.maxViolationsPerScan * contract.budgets.maxTargetsPerViolation, frameId }) +} + +async function accessibilityTree(page: Page, contract: BrowserAccessibilityContract): Promise> { + if (contract.capabilities.accessibilityTree === "disabled") return { status: "unsupported", reason: "disabled_by_contract" } + try { + const raw = await page.locator("body").ariaSnapshot({ timeout: 2_000 }) + const redacted = raw.replace(/"(?:[^"\\]|\\.)*"/g, '""').replace(/(^|\n)(\s*-?\s*\/?(?:text|url|value|description|placeholder):\s*).*/gi, "$1$2") + const snapshot = redacted.slice(0, contract.budgets.maxTreeChars) + return { status: "captured", snapshot, truncated: redacted.length > snapshot.length } + } catch (error) { + return { status: "unsupported", reason: (error instanceof Error ? error.message : String(error)).slice(0, 240) } + } +} + +function finding(oracle: BrowserAccessibilityFinding["oracle"], rule: BrowserAccessibilityFinding["rule"], code: string, impact: BrowserAccessibilityFinding["impact"], classification: string, target: ElementEvidence, expected?: string, actual?: string): Omit { + return { oracle, rule, code, impact, classification, target, expected, actual } +} + +function impactRank(impact: BrowserAccessibilityFinding["impact"]): number { return { minor: 0, moderate: 1, serious: 2, critical: 3 }[impact] } + +function accessibilityFrameIdentities(page: Page, maximum: number): Array<{ id: string; frame: Frame }> { + const identities: Array<{ id: string; frame: Frame }> = [{ id: "document", frame: page.mainFrame() }] + const walk = (parent: Frame, parentId: string) => parent.childFrames().forEach((frame, index) => { + if (identities.length >= maximum) return + const id = parentId === "document" ? `frame:${index}` : `${parentId}.${index}` + identities.push({ id, frame }) + walk(frame, id) + }) + walk(page.mainFrame(), "document") + return identities +} diff --git a/packages/runtime-playground/src/browser-actions-runner.ts b/packages/runtime-playground/src/browser-actions-runner.ts index c316e4adc..b723de70f 100644 --- a/packages/runtime-playground/src/browser-actions-runner.ts +++ b/packages/runtime-playground/src/browser-actions-runner.ts @@ -1,4 +1,4 @@ -import { readFile } from "node:fs/promises" +import { readFile, writeFile } from "node:fs/promises" import { BROWSER_ACTION_CORPUS_SCHEMA, BROWSER_ADAPTIVE_EXPLORATION_SCHEMA, BROWSER_MULTI_ACTOR_SCENARIO_SCHEMA, BROWSER_TOOL_VERIFIER_RESULT_SCHEMA, HostToolRegistry, assertRuntimeCommandAllowed, browserActionCorpusArtifact, browserActionCorpusContract, browserAdaptiveExplorationContract, browserInteractionScriptUsesEvaluate, browserToolVerifierInputSummary, createHostToolRegistry, executeHostTool, resolveCommandPath, validateBrowserInteractionScript, type BrowserActionCorpusArtifact, type BrowserActionCorpusContract, type BrowserAdaptiveExplorationArtifact, type BrowserAdaptiveExplorationContract, type BrowserInteractionStep, type BrowserMultiActorScenario, type BrowserToolVerifierResult, type ExecutionSpec, type HostToolDefinition, type JsonValue, type RuntimeCreateSpec } from "@automattic/wp-codebox-core" import { now, sha256 } from "@automattic/wp-codebox-core/internals" import { browserInteractionStepsFromArgs, browserStepTimeoutMs, durationStringMs, sanitizeScreenshotName } from "./browser-actions.js" @@ -23,6 +23,7 @@ import type { PlaygroundCliServer } from "./preview-server.js" import type { Page } from "playwright" import { discoverBrowserActionCorpusDescriptors } from "./browser-action-discovery.js" import { exploreAdaptiveBrowserStateMachine } from "./browser-adaptive-explorer.js" +import { createBrowserAccessibilityCollector } from "./browser-accessibility-collector.js" export { discoverBrowserActionCorpusDescriptors } from "./browser-action-discovery.js" @@ -240,8 +241,39 @@ export async function runBrowserActionsCommand({ observations: { consoleMessages, errors, network }, navigationScope: topology.navigationScope, signal: spec.signal, + accessibilityCollector: adaptiveContract.accessibility ? createBrowserAccessibilityCollector(page, adaptiveContract.accessibility) : undefined, + onAccessibilityFindingEvidence: adaptiveContract.accessibility ? async (scan) => { + const basename = `accessibility-${String(scan.index).padStart(3, "0")}` + const screenshotRef = `files/browser/${basename}.png` + const fileBudget = Math.max(256, Math.floor(adaptiveContract.budgets.maxArtifactBytes / 4)) + const screenshot = await page.screenshot({ fullPage: true }) + if (screenshot.byteLength > fileBudget) { + scan.diagnostics.push({ code: "browser_accessibility_screenshot_budget_exhausted", message: "The correlated screenshot exceeded its accessibility evidence byte bound and was omitted." }) + return {} + } + await artifactSession.writeGenerated("screenshot", `${basename}.png`, (path) => writeFile(path, screenshot)) + try { + const snapshot = await captureBrowserActionDomSnapshot({ + artifactSession, + finalUrl: page.url(), + maxElements: Math.min(maxDomSnapshotElements, adaptiveContract.descriptorLimits.maxPerState), + page, + screenshotRef, + snapshotRef: `files/browser/${basename}-dom.json`, + selectors: [...new Set(scan.findings.map((finding) => finding.target.locator).filter((locator) => locator !== "body" && locator !== "document"))], + viewport, + maxArtifactBytes: fileBudget, + }) + domSnapshots.push(snapshot) + return { screenshot: screenshotRef, domSnapshot: snapshot.snapshot } + } catch { + scan.diagnostics.push({ code: "browser_accessibility_dom_budget_exhausted", message: "The correlated DOM snapshot exceeded its accessibility evidence byte bound and was omitted." }) + return { screenshot: screenshotRef } + } + } : undefined, }) adaptiveExplorationArtifact = { schema: "wp-codebox/browser-adaptive-exploration-artifact/v1", contract: adaptiveContract, result, capturedAt: now() } + boundAdaptiveExplorationArtifact(adaptiveExplorationArtifact, Math.max(512, Math.floor(adaptiveContract.budgets.maxArtifactBytes / 2))) await artifactSession.writeJson("adaptiveExploration", "adaptive-exploration.json", adaptiveExplorationArtifact) adaptiveExplorationSummary = { schema: BROWSER_ADAPTIVE_EXPLORATION_SCHEMA, @@ -671,6 +703,8 @@ async function captureBrowserActionDomSnapshot({ screenshotRef, snapshotRef, step, + selectors = [], + maxArtifactBytes, viewport, }: { artifactSession: BrowserArtifactSession @@ -680,12 +714,14 @@ async function captureBrowserActionDomSnapshot({ screenshotRef: string snapshotRef?: string step?: { index: number; name?: string; kind: string } + selectors?: string[] + maxArtifactBytes?: number viewport: BrowserProbeViewport | null }): Promise<{ screenshot: string; snapshot: string; step?: { index: number; name?: string; kind: string }; elementCount: number; capturedElements: number; truncated: boolean }> { const sanitizedName = step?.name ? sanitizeScreenshotName(step.name) : undefined const relativeSnapshotRef = snapshotRef ?? `files/browser/dom-snapshot-${sanitizedName || `step-${step?.index ?? 0}`}.json` const snapshotFileName = relativeSnapshotRef.replace(/^files\/browser\//, "") - const snapshot = await captureBrowserDomSnapshot(page, maxElements) + const snapshot = await captureBrowserDomSnapshot(page, maxElements, selectors) const artifact: BrowserDomSnapshotArtifact = { schema: "wp-codebox/browser-dom-snapshot/v1", command: "wordpress.browser-actions", @@ -702,6 +738,15 @@ async function captureBrowserActionDomSnapshot({ }, snapshot, } + if (maxArtifactBytes) { + while (artifact.snapshot.capturedElements.length > 0 && Buffer.byteLength(JSON.stringify(artifact)) > maxArtifactBytes) { + artifact.snapshot.capturedElements.pop() + artifact.snapshot.truncated = true + artifact.summary.capturedElements = artifact.snapshot.capturedElements.length + artifact.summary.truncated = true + } + if (Buffer.byteLength(JSON.stringify(artifact)) > maxArtifactBytes) throw new Error("Accessibility DOM evidence exceeded its byte bound.") + } await artifactSession.writeJson("domSnapshots", snapshotFileName, artifact) return { screenshot: screenshotRef, @@ -713,6 +758,55 @@ async function captureBrowserActionDomSnapshot({ } } +export function boundAdaptiveExplorationArtifact(artifact: BrowserAdaptiveExplorationArtifact, maxBytes: number): void { + const size = () => Buffer.byteLength(JSON.stringify(artifact, null, 2)) + 1 + if (size() <= maxBytes) return + const evidence = artifact.result.accessibility + if (evidence) { + evidence.summary.truncated = true + for (const scan of evidence.scans) { + if (scan.accessibilityTree?.snapshot) { + delete scan.accessibilityTree.snapshot + scan.accessibilityTree.truncated = true + scan.accessibilityTree.reason = "artifact_byte_budget" + } + } + while (size() > maxBytes && evidence.focusHistory.length > 0) evidence.focusHistory.pop() + while (size() > maxBytes) { + const index = lastMatchingIndex(evidence.scans, (scan) => scan.findings.length === 0) + if (index < 0) break + evidence.scans.splice(index, 1) + } + } + const retainedTransitions = new Set(artifact.result.findings.map((finding) => finding.transitionId)) + while (size() > maxBytes) { + const index = lastMatchingIndex(artifact.result.transitions, (transition) => !retainedTransitions.has(transition.id)) + if (index < 0) break + artifact.result.transitions.splice(index, 1) + pruneUnreferencedAdaptiveStates(artifact) + } + while (size() > maxBytes && artifact.result.diagnostics.length > 0) artifact.result.diagnostics.pop() + artifact.result.summary.states = artifact.result.states.length + artifact.result.summary.transitions = artifact.result.transitions.length + artifact.result.status = "incomplete" + artifact.result.summary.budgetExhausted = "maxArtifactBytes" + if (size() > maxBytes) throw new Error(`Adaptive browser exploration evidence cannot fit maxArtifactBytes=${artifact.contract.budgets.maxArtifactBytes}.`) +} + +function pruneUnreferencedAdaptiveStates(artifact: BrowserAdaptiveExplorationArtifact): void { + const retained = new Set([ + artifact.result.states[0]?.digest ?? "", + ...artifact.result.findings.flatMap((finding) => finding.stateDigest ? [finding.stateDigest] : []), + ...artifact.result.transitions.flatMap((transition) => [transition.sourceDigest, transition.destinationDigest].filter((value): value is string => Boolean(value))), + ]) + artifact.result.states = artifact.result.states.filter((state) => retained.has(state.digest)) +} + +function lastMatchingIndex(items: T[], predicate: (item: T) => boolean): number { + for (let index = items.length - 1; index >= 0; index -= 1) if (predicate(items[index] as T)) return index + return -1 +} + interface BrowserScenarioInput { url?: string profile?: string diff --git a/packages/runtime-playground/src/browser-adaptive-explorer.ts b/packages/runtime-playground/src/browser-adaptive-explorer.ts index dd893f977..cce1a6a4c 100644 --- a/packages/runtime-playground/src/browser-adaptive-explorer.ts +++ b/packages/runtime-playground/src/browser-adaptive-explorer.ts @@ -11,6 +11,8 @@ import { type BrowserAdaptiveFrameIdentity, type BrowserAdaptiveState, type BrowserAdaptiveTransition, + type BrowserAccessibilityCollector, + type BrowserAccessibilityFinding as BrowserA11yFinding, } from "@automattic/wp-codebox-core" import { stableJson } from "@automattic/wp-codebox-core/internals" import type { Frame, Page } from "playwright" @@ -50,6 +52,8 @@ export async function exploreAdaptiveBrowserStateMachine({ signal, now = Date.now, navigationScope, + accessibilityCollector, + onAccessibilityFindingEvidence, }: { page: Page baseUrl: string @@ -58,6 +62,8 @@ export async function exploreAdaptiveBrowserStateMachine({ signal?: AbortSignal now?: () => number navigationScope?: BrowserPreviewNavigationScope + accessibilityCollector?: BrowserAccessibilityCollector + onAccessibilityFindingEvidence?: (scan: Awaited>) => Promise<{ screenshot?: string; domSnapshot?: string }> }): Promise { const started = now() const states = new Map() @@ -68,13 +74,20 @@ export async function exploreAdaptiveBrowserStateMachine({ let actions = 0 let errors = 0 let revisits = 0 + let keyboardActions = 0 let exhausted: BrowserAdaptiveExplorationResult["summary"]["budgetExhausted"] const initial = await captureAdaptiveState(page, contract, 0, navigationScope) appendDiagnostics(diagnostics, initial.diagnostics, contract.descriptorLimits.maxDiagnostics) states.set(initial.state.digest, initial.state) const frontier: FrontierEntry[] = [{ state: initial.state, path: [] }] - if (artifactBytes(states, transitions, diagnostics, findings) > contract.budgets.maxArtifactBytes) exhausted = "maxArtifactBytes" + if (accessibilityCollector && contract.accessibility?.cadence.includes("initial")) { + const scan = await accessibilityCollector.scan({ phase: "initial", stateDigest: initial.state.digest }) + if (scan.findings.length > 0 && onAccessibilityFindingEvidence) scan.artifacts = await onAccessibilityFindingEvidence(scan) + findings.push(...adaptiveAccessibilityFindings(scan.findings, contract, [], "initial")) + if (artifactBytes(states, transitions, diagnostics, findings) + Buffer.byteLength(stableJson(accessibilityCollector.evidence())) > adaptiveJsonArtifactBudget(contract, accessibilityCollector)) exhausted = "maxArtifactBytes" + } + if (artifactBytes(states, transitions, diagnostics, findings) > adaptiveJsonArtifactBudget(contract, accessibilityCollector)) exhausted = "maxArtifactBytes" while (frontier.length > 0 && !exhausted && findings.length === 0) { if (signal?.aborted) { exhausted = "cancelled"; break } @@ -86,6 +99,7 @@ export async function exploreAdaptiveBrowserStateMachine({ if (actions >= contract.budgets.maxActions) { exhausted = "maxActions"; break } if (transitions.length >= contract.budgets.maxTransitions) { exhausted = "maxTransitions"; break } if (now() - started >= contract.budgets.maxDurationMs) { exhausted = "maxDurationMs"; break } + const keyboardBudget = contract.accessibility?.budgets.maxKeyboardActions ?? Number.POSITIVE_INFINITY const actionVisitKey = `${source.state.digest}:${action.id}` const visits = actionVisits.get(actionVisitKey) ?? 0 if (visits >= contract.revisitPolicy.maxActionVisits) continue @@ -93,12 +107,14 @@ export async function exploreAdaptiveBrowserStateMachine({ if (contract.resetPolicy.mode === "start-url" && (transitions.length > 0 || source.path.length > 0)) { if (actions + source.path.length >= contract.budgets.maxActions) { exhausted = "maxActions"; break } - const restored = await restoreAdaptivePath(page, baseUrl, contract, source.path, signal, navigationScope) + if (keyboardActions + countKeyboardActions(source.path) > keyboardBudget) { exhausted = "maxKeyboardActions"; break } + const restored = await restoreAdaptivePath(page, baseUrl, contract, source.path, signal, navigationScope, accessibilityCollector) actions += restored.executed + keyboardActions += restored.keyboardExecuted appendDiagnostics(diagnostics, restored.diagnostics, contract.descriptorLimits.maxDiagnostics) if (!restored.state || restored.state.digest !== source.state.digest) { const rejected = rejectedTransition(source.state, action, page.url(), "browser_adaptive_source_state_not_reproduced", "The declared start URL and replay path did not reproduce the queued source state.") - if (artifactBytes(states, [...transitions, rejected], diagnostics, findings) > contract.budgets.maxArtifactBytes) { exhausted = "maxArtifactBytes"; break } + if (artifactBytes(states, [...transitions, rejected], diagnostics, findings) > adaptiveJsonArtifactBudget(contract, accessibilityCollector)) { exhausted = "maxArtifactBytes"; break } transitions.push(rejected) continue } @@ -109,6 +125,8 @@ export async function exploreAdaptiveBrowserStateMachine({ const beforeNetwork = observations.network.length const transitionStarted = now() await installMutationObservers(page, contract) + if (action.family === "keyboard" && keyboardActions >= keyboardBudget) { exhausted = "maxKeyboardActions"; break } + await accessibilityCollector?.beforeAction(action) let actionError: string | undefined try { await executeAdaptiveAction(page, action, contract.stabilization.maxWaitMs) @@ -116,6 +134,7 @@ export async function exploreAdaptiveBrowserStateMachine({ actionError = error instanceof Error ? error.message : String(error) } actions += 1 + if (action.family === "keyboard") keyboardActions += 1 const stabilized = await stabilizeAdaptiveState(page, contract, source.path.length + 1, now, navigationScope) appendDiagnostics(diagnostics, stabilized.diagnostics, contract.descriptorLimits.maxDiagnostics) const scopeRejection = stabilized.diagnostics.find((diagnostic) => diagnostic.code === "browser_adaptive_redirect_scope_escape_rejected") @@ -124,6 +143,14 @@ export async function exploreAdaptiveBrowserStateMachine({ const fingerprints = [...new Set([...newConsoleErrors, ...newPageErrors].map((message) => browserAdaptiveDigest("oracle", message)))].sort() const existing = states.get(stabilized.state.digest) const newState = !existing + const transitionId = `transition-${transitions.length}` + const accessibilityScan = accessibilityCollector && contract.accessibility?.cadence.includes("novel-state") && (newState || action.family === "keyboard") && !actionError + ? await accessibilityCollector.scan({ phase: "novel-state", stateDigest: stabilized.state.digest, transitionId, action }) + : undefined + const accessibilityFingerprints = accessibilityScan?.findings.map((finding) => finding.fingerprint) ?? [] + if (accessibilityScan && accessibilityScan.findings.length > 0 && onAccessibilityFindingEvidence) accessibilityScan.artifacts = await onAccessibilityFindingEvidence(accessibilityScan) + fingerprints.push(...accessibilityFingerprints) + fingerprints.sort() const replayableDestination = !actionError if (replayableDestination && newState && states.size < contract.budgets.maxStates) { states.set(stabilized.state.digest, stabilized.state) @@ -135,7 +162,7 @@ export async function exploreAdaptiveBrowserStateMachine({ } const destination = existing ?? stabilized.state const transition: BrowserAdaptiveTransition = { - id: `transition-${transitions.length}`, + id: transitionId, sourceDigest: source.state.digest, destinationDigest: destination.digest, action, @@ -156,12 +183,13 @@ export async function exploreAdaptiveBrowserStateMachine({ loadingBefore: source.state.loadingIndicators, loadingAfter: stabilized.loading, oracleFingerprints: fingerprints, + ...(accessibilityFingerprints.length > 0 ? { accessibilityFindingFingerprints: accessibilityFingerprints } : {}), }, status: actionError ? "error" : scopeRejection ? "rejected" : newState ? "ok" : "revisited", ...(actionError ? { diagnostic: { code: "browser_adaptive_action_error", message: actionError } } : scopeRejection ? { diagnostic: { code: scopeRejection.code, message: scopeRejection.message } } : {}), } errors += newConsoleErrors.length + newPageErrors.length + (actionError ? 1 : 0) - if (artifactBytes(states, [...transitions, transition], diagnostics, findings) > contract.budgets.maxArtifactBytes) { + if (artifactBytes(states, [...transitions, transition], diagnostics, findings) + (accessibilityCollector ? Buffer.byteLength(stableJson(accessibilityCollector.evidence())) : 0) > adaptiveJsonArtifactBudget(contract, accessibilityCollector)) { if (replayableDestination && newState) states.delete(stabilized.state.digest) exhausted = "maxArtifactBytes" break @@ -187,7 +215,7 @@ export async function exploreAdaptiveBrowserStateMachine({ resetPolicy: contract.resetPolicy, }, } - if (artifactBytes(states, transitions, diagnostics, [finding]) > contract.budgets.maxArtifactBytes) { + if (artifactBytes(states, transitions, diagnostics, [finding]) + (accessibilityCollector ? Buffer.byteLength(stableJson(accessibilityCollector.evidence())) : 0) > adaptiveJsonArtifactBudget(contract, accessibilityCollector)) { exhausted = "maxArtifactBytes" break } @@ -203,24 +231,33 @@ export async function exploreAdaptiveBrowserStateMachine({ if (findings.length > 0 && !signal?.aborted) { for (const finding of findings) { - const minimized = await minimizeAdaptiveFinding(page, baseUrl, contract, finding, observations, Math.max(0, contract.budgets.maxActions - actions), started + contract.budgets.maxDurationMs, signal, now, navigationScope) + const minimized = await minimizeAdaptiveFinding(page, baseUrl, contract, finding, observations, Math.max(0, contract.budgets.maxActions - actions), Math.max(0, (contract.accessibility?.budgets.maxKeyboardActions ?? Number.POSITIVE_INFINITY) - keyboardActions), started + contract.budgets.maxDurationMs, signal, now, navigationScope, accessibilityCollector) actions += minimized.executed + keyboardActions += minimized.keyboardExecuted finding.minimizedPath = minimized.path finding.replay.actions = finding.minimizedPath if (minimized.exhausted && !exhausted) exhausted = minimized.exhausted } } + if (accessibilityCollector && findings.length === 0 && contract.accessibility?.cadence.includes("final")) { + const finalState = await captureAdaptiveState(page, contract, 0, navigationScope) + const scan = await accessibilityCollector.scan({ phase: "final", stateDigest: finalState.state.digest }) + if (scan.findings.length > 0 && onAccessibilityFindingEvidence) scan.artifacts = await onAccessibilityFindingEvidence(scan) + findings.push(...adaptiveAccessibilityFindings(scan.findings, contract, [], "final")) + if (artifactBytes(states, transitions, diagnostics, findings) + Buffer.byteLength(stableJson(accessibilityCollector.evidence())) > adaptiveJsonArtifactBudget(contract, accessibilityCollector)) exhausted = "maxArtifactBytes" + } if (!exhausted && findings.length === 0 && frontier.length > 0) exhausted = "frontier" if (exhausted) appendTerminalDiagnostic(diagnostics, { code: exhausted === "cancelled" ? "browser_adaptive_cancelled" : "browser_adaptive_budget_exhausted", message: exhausted === "cancelled" ? "Adaptive exploration stopped scheduling actions after cancellation and retained partial evidence." : `Adaptive exploration stopped at the ${exhausted} bound.`, metadata: { budget: exhausted } }, contract.descriptorLimits.maxDiagnostics) return { schema: BROWSER_ADAPTIVE_EXPLORATION_SCHEMA, - status: exhausted ? "incomplete" : findings.length > 0 ? "findings" : "completed", + status: exhausted || accessibilityCollector?.evidence().summary.truncated || accessibilityRequirementsUnavailable(accessibilityCollector?.evidence(), contract) || accessibilityCollector?.evidence().scans.some((scan) => scan.status === "inconclusive") ? "incomplete" : findings.length > 0 ? "findings" : "completed", seed: contract.seed, startUrl: contract.startUrl, states: [...states.values()], transitions, findings, + ...(accessibilityCollector ? { accessibility: accessibilityCollector.evidence() } : {}), diagnostics: diagnostics.slice(0, contract.descriptorLimits.maxDiagnostics), summary: { actions, states: states.size, transitions: transitions.length, revisits, errors, findings: findings.length, ...(exhausted ? { budgetExhausted: exhausted } : {}) }, replay: { schema: BROWSER_ADAPTIVE_EXPLORATION_SCHEMA, seed: contract.seed, startUrl: contract.startUrl, contract }, @@ -339,6 +376,7 @@ async function executeAdaptiveAction(page: Page, action: BrowserAdaptiveAction, } for (const step of action.steps) { const selector = currentSelector ?? step.selector + if (!selector && step.kind === "press") { await page.keyboard.press(String(step.key ?? "")); continue } if (!selector) throw new Error(`Adaptive ${step.kind} action requires a unique selector.`) const locator = frame.locator(selector) const count = await locator.count() @@ -418,30 +456,47 @@ async function readMutationObservers(page: Page, contract: BrowserAdaptiveExplor return { count, truncated } } -async function restoreAdaptivePath(page: Page, baseUrl: string, contract: BrowserAdaptiveExplorationContract, path: BrowserAdaptiveAction[], signal?: AbortSignal, navigationScope?: BrowserPreviewNavigationScope): Promise<{ state?: BrowserAdaptiveState; executed: number; diagnostics: BrowserAdaptiveExplorationResult["diagnostics"] }> { +async function restoreAdaptivePath(page: Page, baseUrl: string, contract: BrowserAdaptiveExplorationContract, path: BrowserAdaptiveAction[], signal?: AbortSignal, navigationScope?: BrowserPreviewNavigationScope, accessibilityCollector?: BrowserAccessibilityCollector): Promise<{ state?: BrowserAdaptiveState; executed: number; keyboardExecuted: number; diagnostics: BrowserAdaptiveExplorationResult["diagnostics"]; oracleFingerprints: string[]; finalAccessibilityFingerprints: string[] }> { const diagnostics: BrowserAdaptiveExplorationResult["diagnostics"] = [] let executed = 0 + let keyboardExecuted = 0 + const oracleFingerprints: string[] = [] + let finalAccessibilityFingerprints: string[] = [] try { + await accessibilityCollector?.reset() await page.goto(resolveUrl(contract.startUrl, baseUrl), { waitUntil: "domcontentloaded", timeout: contract.stabilization.maxWaitMs }) await stabilizeAdaptiveState(page, contract, 0, Date.now, navigationScope) + if (accessibilityCollector) { + const initialScan = await accessibilityCollector.scan({ phase: "replay", record: false }) + finalAccessibilityFingerprints = initialScan.findings.map((finding) => finding.fingerprint) + oracleFingerprints.push(...finalAccessibilityFingerprints) + } for (const action of path) { - if (signal?.aborted) return { executed, diagnostics } + if (signal?.aborted) return { executed, keyboardExecuted, diagnostics, oracleFingerprints, finalAccessibilityFingerprints } executed += 1 + if (action.family === "keyboard") keyboardExecuted += 1 + await accessibilityCollector?.beforeAction(action) await executeAdaptiveAction(page, action, contract.stabilization.maxWaitMs) - await stabilizeAdaptiveState(page, contract, executed, Date.now, navigationScope) + const stabilized = await stabilizeAdaptiveState(page, contract, executed, Date.now, navigationScope) + if (accessibilityCollector) { + const scan = await accessibilityCollector.scan({ phase: "replay", stateDigest: stabilized.state.digest, action, record: false }) + finalAccessibilityFingerprints = scan.findings.map((finding) => finding.fingerprint) + oracleFingerprints.push(...finalAccessibilityFingerprints) + } } - return { state: (await captureAdaptiveState(page, contract, path.length, navigationScope)).state, executed, diagnostics } + return { state: (await captureAdaptiveState(page, contract, path.length, navigationScope)).state, executed, keyboardExecuted, diagnostics, oracleFingerprints, finalAccessibilityFingerprints } } catch (error) { diagnostics.push({ code: "browser_adaptive_reset_replay_failed", message: "The start-URL reset path could not reproduce a queued state.", metadata: { reason: error instanceof Error ? error.message : String(error) } }) - return { executed, diagnostics } + return { executed, keyboardExecuted, diagnostics, oracleFingerprints, finalAccessibilityFingerprints } } } -async function minimizeAdaptiveFinding(page: Page, baseUrl: string, contract: BrowserAdaptiveExplorationContract, finding: BrowserAdaptiveFinding, observations: AdaptiveObservationSources, maximumActions: number, deadline: number, signal?: AbortSignal, now: () => number = Date.now, navigationScope?: BrowserPreviewNavigationScope): Promise<{ path: BrowserAdaptiveAction[]; executed: number; exhausted?: "maxActions" | "maxDurationMs" | "cancelled" }> { +async function minimizeAdaptiveFinding(page: Page, baseUrl: string, contract: BrowserAdaptiveExplorationContract, finding: BrowserAdaptiveFinding, observations: AdaptiveObservationSources, maximumActions: number, maximumKeyboardActions: number, deadline: number, signal?: AbortSignal, now: () => number = Date.now, navigationScope?: BrowserPreviewNavigationScope, accessibilityCollector?: BrowserAccessibilityCollector): Promise<{ path: BrowserAdaptiveAction[]; executed: number; keyboardExecuted: number; exhausted?: "maxActions" | "maxKeyboardActions" | "maxDurationMs" | "cancelled" }> { let current = [...finding.originalPath] let chunk = Math.max(1, Math.floor(current.length / 2)) let executed = 0 - let exhausted: "maxActions" | "maxDurationMs" | "cancelled" | undefined + let keyboardExecuted = 0 + let exhausted: "maxActions" | "maxKeyboardActions" | "maxDurationMs" | "cancelled" | undefined while (current.length > 1 && chunk >= 1 && !signal?.aborted) { let reduced = false for (let start = 0; start < current.length; start += chunk) { @@ -449,11 +504,13 @@ async function minimizeAdaptiveFinding(page: Page, baseUrl: string, contract: Br if (candidate.length === 0) continue if (now() >= deadline) { exhausted = "maxDurationMs"; break } if (executed + candidate.length > maximumActions) { exhausted = "maxActions"; break } + if (keyboardExecuted + countKeyboardActions(candidate) > maximumKeyboardActions) { exhausted = "maxKeyboardActions"; break } const beforeConsole = observations.consoleMessages.length const beforeErrors = observations.errors.length - const replay = await restoreAdaptivePath(page, baseUrl, contract, candidate, signal, navigationScope) + const replay = await restoreAdaptivePath(page, baseUrl, contract, candidate, signal, navigationScope, accessibilityCollector) executed += replay.executed - const fingerprints = [...consoleErrorMessages(observations.consoleMessages.slice(beforeConsole)), ...errorMessages(observations.errors.slice(beforeErrors))].map((message) => browserAdaptiveDigest("oracle", message)) + keyboardExecuted += replay.keyboardExecuted + const fingerprints = [...consoleErrorMessages(observations.consoleMessages.slice(beforeConsole)), ...errorMessages(observations.errors.slice(beforeErrors))].map((message) => browserAdaptiveDigest("oracle", message)).concat(replay.finalAccessibilityFingerprints) if (replay.state && fingerprints.includes(finding.fingerprint) && (!finding.stateDigest || replay.state.digest === finding.stateDigest)) { current = candidate reduced = true @@ -464,7 +521,7 @@ async function minimizeAdaptiveFinding(page: Page, baseUrl: string, contract: Br if (!reduced) chunk = Math.floor(chunk / 2) } if (signal?.aborted) exhausted = "cancelled" - return { path: current, executed, ...(exhausted ? { exhausted } : {}) } + return { path: current, executed, keyboardExecuted, ...(exhausted ? { exhausted } : {}) } } function frameIdentities(page: Page, maximum: number): BrowserAdaptiveFrameIdentity[] { @@ -517,6 +574,32 @@ function artifactBytes(states: Map, transitions: B return Buffer.byteLength(stableJson({ states: [...states.values()], transitions, diagnostics, findings })) } +function adaptiveJsonArtifactBudget(contract: BrowserAdaptiveExplorationContract, collector?: BrowserAccessibilityCollector): number { + return collector ? Math.max(512, Math.floor(contract.budgets.maxArtifactBytes / 2)) : contract.budgets.maxArtifactBytes +} + +function countKeyboardActions(actions: BrowserAdaptiveAction[]): number { + return actions.filter((action) => action.family === "keyboard").length +} + +function accessibilityRequirementsUnavailable(evidence: ReturnType | undefined, contract: BrowserAdaptiveExplorationContract): boolean { + if (!evidence || !contract.accessibility) return false + return (contract.accessibility.capabilities.rules === "required" && evidence.collector.capabilities.rules !== "supported") + || (contract.accessibility.capabilities.focus === "required" && evidence.collector.capabilities.focus !== "supported") + || (contract.accessibility.capabilities.accessibilityTree === "required" && evidence.collector.capabilities.accessibilityTree !== "supported") +} + +function adaptiveAccessibilityFindings(items: BrowserA11yFinding[], contract: BrowserAdaptiveExplorationContract, path: BrowserAdaptiveAction[], transitionId: string): BrowserAdaptiveFinding[] { + return items.map((item) => ({ + fingerprint: item.fingerprint, + stateDigest: item.stateDigest, + transitionId, + originalPath: path, + minimizedPath: path, + replay: { schema: BROWSER_ADAPTIVE_EXPLORATION_SCHEMA, seed: contract.seed, startUrl: contract.startUrl, expectedFingerprint: item.fingerprint, expectedStateDigest: item.stateDigest, actions: path, resetPolicy: contract.resetPolicy }, + })) +} + function appendDiagnostics(target: BrowserAdaptiveExplorationResult["diagnostics"], incoming: BrowserAdaptiveExplorationResult["diagnostics"], maximum: number): void { const remaining = Math.max(0, maximum - target.length) if (remaining > 0) target.push(...incoming.slice(0, remaining)) diff --git a/packages/runtime-playground/src/browser-dom-snapshot.ts b/packages/runtime-playground/src/browser-dom-snapshot.ts index 503517ca1..40a6cc68e 100644 --- a/packages/runtime-playground/src/browser-dom-snapshot.ts +++ b/packages/runtime-playground/src/browser-dom-snapshot.ts @@ -51,7 +51,7 @@ export async function captureBrowserDomSnapshot(page: Page, maxElements: number, .map((element) => elementSnapshot(element, styleProperties, attributeNames)) .filter((element): element is BrowserDomElementSnapshot => Boolean(element)) const capturedByPath = new Map(visibleElements.slice(0, maxElements).map((element) => [element.path, element])) - const selectorSnapshots = selectorInputs.map((selector) => selectorSnapshot(selector, capturedByPath, styleProperties, attributeNames)) + const selectorSnapshots = selectorInputs.map((selector) => selectorSnapshot(selector, capturedByPath, styleProperties, attributeNames, maxElements)) return { url: window.location.href, @@ -62,10 +62,10 @@ export async function captureBrowserDomSnapshot(page: Page, maxElements: number, truncated: visibleElements.length > maxElements, } - function selectorSnapshot(selector: string, captured: Map, styles: string[], attributes: string[]): BrowserDomSelectorSnapshot { + function selectorSnapshot(selector: string, captured: Map, styles: string[], attributes: string[], maximum: number): BrowserDomSelectorSnapshot { try { const matches = Array.from(document.querySelectorAll(selector)) - const snapshots = matches.map((element) => elementSnapshot(element, styles, attributes)).filter((element): element is BrowserDomElementSnapshot => Boolean(element)) + const snapshots = matches.map((element) => elementSnapshot(element, styles, attributes)).filter((element): element is BrowserDomElementSnapshot => Boolean(element)).slice(0, Math.max(0, maximum - captured.size)) for (const snapshot of snapshots) { captured.set(snapshot.path, snapshot) } diff --git a/tests/adversarial-browser.test.ts b/tests/adversarial-browser.test.ts index 69ad9153e..01beb9a83 100644 --- a/tests/adversarial-browser.test.ts +++ b/tests/adversarial-browser.test.ts @@ -2,6 +2,7 @@ import assert from "node:assert/strict" import test from "node:test" import { evaluateAdversarialBrowserOracles, minimizeAdversarialBrowserJourney, planAdversarialBrowserJourney } from "../packages/runtime-core/src/adversarial-browser.js" +import { browserAccessibilityFindingFingerprint, type BrowserAccessibilityFinding } from "../packages/runtime-core/src/browser-accessibility.js" const descriptors = [ { id: "input:name", kind: "input" as const, selector: "#name", type: "text" }, @@ -17,16 +18,26 @@ test("browser adversary discovers hostile inputs and repeated interactions deter }) test("generic browser oracles report crashes, dead controls, layout, accessibility, duplicate effects, and stuck state", () => { + const normalized = { + oracle: "keyboard", + rule: "keyboard-reachable", + code: "browser-keyboard-unreachable", + impact: "serious", + classification: "actionable-element-unreachable-by-keyboard", + target: { locator: "div[role=button]", frameId: "document", tag: "div", role: "button" }, + } satisfies Omit const result = evaluateAdversarialBrowserOracles({ pageErrors: ["uncaught fixture error"], controls: [{ id: "save", expectedAction: true, actionObserved: false }], loadingIndicators: [{ id: "spinner", visibleForMs: 20_000 }], boxes: [{ id: "dialog", x: 900, y: 0, width: 300, height: 100, viewportWidth: 1024, viewportHeight: 768 }], accessibilityViolations: [{ rule: "label", target: "#name" }], + accessibilityFindings: [{ ...normalized, fingerprint: browserAccessibilityFindingFingerprint(normalized) }], effects: [{ id: "save", count: 2 }], }) assert.equal(result.failed, true) assert.deepEqual(new Set(result.failures.map((failure) => failure.oracle)), new Set(["crash", "dead-control", "stuck-interaction", "layout", "accessibility", "duplicate-effect"])) + assert(result.failures.some((failure) => failure.code === "browser-keyboard-unreachable")) }) test("browser journeys minimize automatically to the shortest reproduced failure", async () => { diff --git a/tests/browser-accessibility-oracles.test.ts b/tests/browser-accessibility-oracles.test.ts new file mode 100644 index 000000000..297471443 --- /dev/null +++ b/tests/browser-accessibility-oracles.test.ts @@ -0,0 +1,287 @@ +import assert from "node:assert/strict" +import test from "node:test" +import { chromium, type Page } from "playwright" + +import { browserAccessibilityContract } from "../packages/runtime-core/src/browser-accessibility.js" +import { browserAdaptiveExplorationContract, type BrowserAdaptiveAction } from "../packages/runtime-core/src/browser-adaptive-exploration.js" +import { createBrowserAccessibilityCollector } from "../packages/runtime-playground/src/browser-accessibility-collector.js" +import { boundAdaptiveExplorationArtifact } from "../packages/runtime-playground/src/browser-actions-runner.js" +import { exploreAdaptiveBrowserStateMachine } from "../packages/runtime-playground/src/browser-adaptive-explorer.js" + +const brokenControls = ` + + + + +
Custom action
+
Decorative
+
Panel
+Private` + +const dialogFixture = ` + + +` + +test("collector classifies bounded accessibility, keyboard, focus, and ARIA failures", async () => { + await withPage(brokenControls, async (page) => { + const contract = requiredContract() + const collector = createBrowserAccessibilityCollector(page, contract) + await page.locator("#offscreen").focus() + const scan = await collector.scan({ phase: "initial", stateDigest: "state-a" }) + const codes = new Set(scan.findings.map((finding) => finding.code)) + assert(codes.has("browser-accessible-name-missing")) + assert(codes.has("browser-keyboard-unreachable")) + assert(codes.has("browser-unexpected-tab-stop")) + assert(codes.has("browser-focused-element-hidden")) + assert(codes.has("browser-aria-expanded-drift")) + assert(scan.findings.every((finding) => finding.target.locator.length <= 240)) + assert(!JSON.stringify(scan).includes("Custom action"), "findings do not retain accessible text") + assert(scan.accessibilityTree?.snapshot?.includes(""), "tree names are redacted") + assert(!JSON.stringify(scan).includes("token=secret"), "tree URLs are redacted") + + const unnamed = scan.findings.filter((finding) => finding.code === "browser-accessible-name-missing") + assert.equal(unnamed.length, 2, "valid value and descendant-alt names pass") + assert.equal(new Set(unnamed.map((finding) => finding.fingerprint)).size, unnamed.length, "distinct targets cannot collide") + + await collector.reset() + await page.reload({ waitUntil: "load" }) + await page.locator("#offscreen").focus() + const replay = await collector.scan({ phase: "replay", stateDigest: "state-b", record: false }) + assert.deepEqual(replay.findings.map((finding) => finding.fingerprint), scan.findings.map((finding) => finding.fingerprint), "volatile transition context does not affect fingerprints") + }) +}) + +test("collector detects dialog entry, containment, and restoration failures", async () => { + await withPage(dialogFixture, async (page) => { + const collector = createBrowserAccessibilityCollector(page, requiredContract()) + await page.locator("#open").focus() + const openAction = action("open", "#open") + await collector.beforeAction(openAction) + await page.locator("#open").click() + const opened = await collector.scan({ phase: "novel-state", transitionId: "open", action: openAction }) + assert(opened.findings.some((finding) => finding.code === "browser-dialog-focus-entry")) + + await page.locator("#close").focus() + const closeAction = action("close", "#close") + await collector.beforeAction(closeAction) + await page.locator("#close").click() + const closed = await collector.scan({ phase: "novel-state", transitionId: "close", action: closeAction }) + assert(closed.findings.some((finding) => finding.code === "browser-dialog-focus-restoration")) + assert(collector.evidence().focusHistory.length <= requiredContract().budgets.maxFocusTransitions) + }) +}) + +test("non-modal dialogs do not imply focus containment", async () => { + await withPage("
Persistent panel
", async (page) => { + const scan = await createBrowserAccessibilityCollector(page, requiredContract()).scan({ phase: "initial" }) + assert(!scan.findings.some((finding) => finding.code.startsWith("browser-dialog-focus"))) + }) +}) + +test("navigation focus reset and below-fold expanded content do not false-positive", async () => { + await withPage("Next
Below fold
", async (page) => { + const collector = createBrowserAccessibilityCollector(page, requiredContract()) + const navigate = action("navigate", "a") + await page.locator("a").focus() + await collector.beforeAction(navigate) + await page.goto("data:text/html,Next", { waitUntil: "load" }) + const navigated = await collector.scan({ phase: "novel-state", action: navigate }) + assert(!navigated.findings.some((finding) => finding.code === "browser-focus-lost")) + }) + await withPage("
Below fold
", async (page) => { + const scan = await createBrowserAccessibilityCollector(page, requiredContract()).scan({ phase: "initial" }) + assert(!scan.findings.some((finding) => finding.code === "browser-aria-expanded-drift")) + }) +}) + +test("controls without nameable content and broken relationships are classified", async () => { + await withPage("
Typo
", async (page) => { + const scan = await createBrowserAccessibilityCollector(page, requiredContract()).scan({ phase: "initial" }) + assert(scan.findings.some((finding) => finding.code === "browser-accessible-name-missing" && finding.target.tag === "select")) + assert(scan.findings.some((finding) => finding.code === "browser-aria-controls-missing")) + assert(scan.findings.some((finding) => finding.code === "browser-aria-expanded-invalid")) + assert(scan.findings.some((finding) => finding.code === "browser-role-invalid")) + }) +}) + +test("same-origin frame findings retain frame identity", async () => { + await withPage("", async (page) => { + await page.locator("iframe").contentFrame().locator("button").waitFor() + const scan = await createBrowserAccessibilityCollector(page, requiredContract()).scan({ phase: "initial" }) + assert(scan.findings.some((finding) => finding.code === "browser-accessible-name-missing" && finding.target.frameId === "frame:0")) + }) +}) + +test("ARIA state drift is observed after both opening and closing a controlled panel", async () => { + await withPage(``, async (page) => { + const collector = createBrowserAccessibilityCollector(page, requiredContract()) + const toggle = action("toggle", "#toggle") + await collector.beforeAction(toggle) + await page.locator("#toggle").click() + const opened = await collector.scan({ phase: "novel-state", transitionId: "open", action: toggle }) + assert(opened.findings.some((finding) => finding.code === "browser-aria-expanded-drift" && finding.actual === "false")) + + await page.locator("#toggle").evaluate((element) => element.setAttribute("aria-expanded", "true")) + await collector.beforeAction(toggle) + await page.locator("#toggle").click() + const closed = await collector.scan({ phase: "novel-state", transitionId: "close", action: toggle }) + assert(closed.findings.some((finding) => finding.code === "browser-aria-expanded-drift" && finding.actual === "true")) + }) +}) + +test("adaptive exploration emits stable first-class findings and replay evidence", async () => { + const run = async () => withPage("", async (page) => { + const startUrl = page.url() + const contract = browserAdaptiveExplorationContract({ + seed: "accessibility-replay", + startUrl, + actionFamilies: ["keyboard"], + budgets: { maxActions: 4, maxStates: 4, maxTransitions: 4, maxDurationMs: 10_000, maxArtifactBytes: 500_000 }, + accessibility: { cadence: ["initial", "novel-state", "final"], budgets: { maxScans: 6, maxKeyboardActions: 2, maxTreeChars: 2_000 } }, + }) + const collector = createBrowserAccessibilityCollector(page, contract.accessibility!) + return exploreAdaptiveBrowserStateMachine({ page, baseUrl: startUrl, contract, accessibilityCollector: collector, observations: { consoleMessages: [], errors: [], network: [] } }) + }) + const first = await run() + const second = await run() + assert.equal(first.status, "findings") + assert.equal(first.findings.length, 1) + assert.deepEqual(first.findings[0]?.originalPath, []) + assert.deepEqual(first.findings[0]?.minimizedPath, []) + assert.equal(first.findings[0]?.replay.expectedFingerprint, first.findings[0]?.fingerprint) + assert.equal(second.findings[0]?.fingerprint, first.findings[0]?.fingerprint) + assert.equal(first.accessibility?.scans[0]?.findings[0]?.transitionId, undefined) + assert.equal(first.accessibility?.summary.findings, 1) + + const artifact = { schema: "wp-codebox/browser-adaptive-exploration-artifact/v1" as const, contract: first.replay.contract, result: structuredClone(first), capturedAt: "2026-01-01T00:00:00.000Z" } + artifact.result.accessibility!.scans[0]!.accessibilityTree = { status: "captured", snapshot: "sensitive".repeat(20_000) } + boundAdaptiveExplorationArtifact(artifact, 12_000) + assert(Buffer.byteLength(JSON.stringify(artifact, null, 2)) + 1 <= 12_000) + assert.equal(artifact.result.findings.length, 1, "artifact bounding retains replayable findings") +}) + +test("adaptive transition findings retain deterministic action replay context", async () => { + const run = async () => withPage(``, async (page) => { + const startUrl = page.url() + const contract = browserAdaptiveExplorationContract({ + seed: "aria-transition-replay", + startUrl, + actionFamilies: ["click"], + budgets: { maxActions: 6, maxStates: 4, maxTransitions: 4, maxDurationMs: 10_000, maxArtifactBytes: 500_000 }, + stabilization: { pollIntervalMs: 20, quietWindowMs: 40, maxWaitMs: 500 }, + accessibility: { ruleTags: ["aria-state"], cadence: ["initial", "novel-state"], capabilities: { accessibilityTree: "disabled" } }, + }) + return exploreAdaptiveBrowserStateMachine({ page, baseUrl: startUrl, contract, accessibilityCollector: createBrowserAccessibilityCollector(page, contract.accessibility!), observations: { consoleMessages: [], errors: [], network: [] } }) + }) + const first = await run() + const second = await run() + const finding = first.findings[0] + assert(finding, JSON.stringify({ status: first.status, summary: first.summary, transitions: first.transitions, accessibility: first.accessibility })) + assert.equal(finding.originalPath.length, 1) + assert.deepEqual(finding.minimizedPath, finding.originalPath) + assert.deepEqual(finding.replay.actions, finding.minimizedPath) + assert.equal(first.transitions[0]?.observations.accessibilityFindingFingerprints?.[0], finding.fingerprint) + assert.equal(first.accessibility?.scans.find((scan) => scan.transitionId === finding.transitionId)?.findings[0]?.code, "browser-aria-expanded-drift") + assert.equal(second.findings[0]?.fingerprint, finding.fingerprint) +}) + +test("keyboard-only focus transitions are scanned within the declared scan budget", async () => { + await withPage("0", async (page) => { + const startUrl = page.url() + const contract = browserAdaptiveExplorationContract({ + seed: "keyboard-focus-history", + startUrl, + actionFamilies: ["keyboard"], + failOnFinding: false, + budgets: { maxActions: 8, maxStates: 8, maxTransitions: 8, maxDurationMs: 10_000, maxArtifactBytes: 500_000 }, + stabilization: { pollIntervalMs: 20, quietWindowMs: 40, maxWaitMs: 300 }, + accessibility: { ruleTags: ["focus-visible"], cadence: ["initial", "novel-state"], capabilities: { accessibilityTree: "disabled" }, budgets: { maxScans: 20, maxKeyboardActions: 4 } }, + }) + const result = await exploreAdaptiveBrowserStateMachine({ page, baseUrl: startUrl, contract, accessibilityCollector: createBrowserAccessibilityCollector(page, contract.accessibility!), observations: { consoleMessages: [], errors: [], network: [] } }) + assert.equal(result.status, "incomplete") + assert.equal(result.summary.budgetExhausted, "maxKeyboardActions") + assert(result.transitions.filter((transition) => transition.action.family === "keyboard").length <= 4) + assert(result.accessibility?.focusHistory.some((entry) => entry.to.includes("button"))) + }) +}) + +test("focus-only keyboard transitions are scanned even when DOM state is revisited", async () => { + await withPage("", async (page) => { + const startUrl = page.url() + const contract = browserAdaptiveExplorationContract({ + seed: "focus-only-revisit", + startUrl, + actionFamilies: ["keyboard"], + failOnFinding: false, + budgets: { maxActions: 3, maxStates: 2, maxTransitions: 3, maxDurationMs: 10_000, maxArtifactBytes: 500_000 }, + stabilization: { pollIntervalMs: 20, quietWindowMs: 40, maxWaitMs: 300 }, + accessibility: { ruleTags: ["focus-visible"], cadence: ["initial", "novel-state"], capabilities: { accessibilityTree: "disabled" }, budgets: { maxScans: 10, maxKeyboardActions: 2 } }, + }) + const result = await exploreAdaptiveBrowserStateMachine({ page, baseUrl: startUrl, contract, accessibilityCollector: createBrowserAccessibilityCollector(page, contract.accessibility!), observations: { consoleMessages: [], errors: [], network: [] } }) + assert(result.transitions.some((transition) => transition.action.family === "keyboard" && transition.status === "revisited")) + assert(result.accessibility?.scans.some((scan) => scan.phase === "novel-state" && scan.actionId?.startsWith("keyboard:"))) + assert(result.accessibility?.focusHistory.some((entry) => entry.to.includes("button"))) + }) +}) + +test("collector stops scanning at maxScans", async () => { + await withPage("", async (page) => { + const contract = browserAccessibilityContract({ capabilities: { accessibilityTree: "disabled" }, budgets: { maxScans: 2 } })! + const collector = createBrowserAccessibilityCollector(page, contract) + await collector.scan({ phase: "initial" }) + await collector.scan({ phase: "novel-state" }) + const exhausted = await collector.scan({ phase: "final" }) + assert.equal(exhausted.status, "inconclusive") + assert.equal(exhausted.diagnostics[0]?.code, "browser_accessibility_scan_budget_exhausted") + assert.equal(collector.evidence().summary.scans, 2) + assert.equal(collector.evidence().summary.truncated, true) + }) +}) + +test("optional and required accessibility-tree capability results are distinct", async () => { + await withPage("", async (page) => { + const disabled = browserAccessibilityContract({ capabilities: { accessibilityTree: "disabled" } })! + const optional = await createBrowserAccessibilityCollector(page, disabled).scan({ phase: "initial" }) + assert.equal(optional.status, "passed") + assert.equal(optional.accessibilityTree?.status, "unsupported") + + const required = browserAccessibilityContract({ capabilities: { accessibilityTree: "required" } })! + const supported = await createBrowserAccessibilityCollector(page, required).scan({ phase: "initial" }) + assert.equal(supported.status, "passed") + assert.equal(supported.accessibilityTree?.status, "captured") + }) +}) + +test("invalid scopes are inconclusive rather than false passes", async () => { + await withPage("", async (page) => { + const contract = browserAccessibilityContract({ includeScopes: ["["] })! + const scan = await createBrowserAccessibilityCollector(page, contract).scan({ phase: "initial" }) + assert.equal(scan.status, "inconclusive") + assert.equal(scan.diagnostics[0]?.code, "browser_accessibility_scope_invalid") + }) +}) + +test("accessibility-enabled contracts normalize impossible artifact budgets", () => { + const contract = browserAdaptiveExplorationContract({ accessibility: {}, budgets: { maxArtifactBytes: 1_024 } }) + assert.equal(contract.budgets.maxArtifactBytes, 1_048_576) +}) + +function requiredContract() { + return browserAccessibilityContract({ impactThreshold: "moderate", capabilities: { accessibilityTree: "required" }, budgets: { maxScans: 10, maxViolationsPerScan: 20, maxFocusTransitions: 10, maxTreeChars: 4_000 } })! +} + +function action(id: string, selector: string): BrowserAdaptiveAction { + return { id, family: "click", frameId: "document", steps: [{ kind: "click", selector }] } +} + +async function withPage(html: string, callback: (page: Page) => Promise): Promise { + const browser = await chromium.launch({ headless: true }) + const page = await browser.newPage() + await page.addInitScript("globalThis.__name = value => value") + await page.goto(`data:text/html;charset=utf-8,${encodeURIComponent(html)}`, { waitUntil: "load" }) + try { return await callback(page) } finally { await browser.close() } +}