Skip to content

Commit 882afc1

Browse files
authored
feat: add browser accessibility fuzzing oracles (#2065)
1 parent cb87a50 commit 882afc1

17 files changed

Lines changed: 1048 additions & 28 deletions

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -725,7 +725,7 @@ Supported runtime commands today:
725725
- `wordpress.server-page-load`: load `surface=admin|frontend` plus `path=<path>` or `url=<path-or-url>` through the live preview HTTP server without a browser. Its performance observation is marked `source=server-http` and `kind=server-page-load`.
726726
- `wordpress.browser-page-load`: load `surface=admin|frontend` plus `path=<path>` or `url=<path-or-url>` in Playwright by wrapping `wordpress.browser-probe`.
727727
- `wordpress.browser-probe`: boot the live preview, visit `url=<path-or-url>` with Playwright, and capture generic browser replay/audit evidence under `files/browser/`.
728-
- `wordpress.browser-actions`: boot the live preview, drive it with an ordered interaction script (`steps-json`), assert browser behavior, and capture replay/audit evidence under `files/browser/`.
728+
- `wordpress.browser-actions`: boot the live preview, drive it with an ordered interaction script (`steps-json`) or bounded adaptive exploration, optionally classify accessibility/keyboard/focus failures, and capture replay/audit evidence under `files/browser/`.
729729

730730
`wordpress.run-php` loads `/wordpress/wp-load.php` by default. Use `--arg bootstrap=none` for raw PHP.
731731

docs/adversarial-runtime.md

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@ and replay surfaces; they do not replace those surfaces.
1616
does not assume a browser, HTTP library, application, or transport.
1717
- `adversarial-browser.ts` owns DOM-derived journey planning, hostile generic
1818
input generation, journey minimization, and user-facing oracle contracts.
19+
- `browser-accessibility.ts` owns provider-neutral accessibility configuration,
20+
normalized findings, capability status, and stable privacy-safe fingerprints.
21+
The Playground adapter collects bounded DOM, focus, keyboard, geometry, and
22+
redacted ARIA-tree evidence through the existing adaptive browser runner.
1923
- `adversarial-artifacts.ts` writes bounded manifested finding and replay bundles,
2024
removes declared secrets and machine-specific paths, and seals corpus/finding
2125
identity with a deterministic content digest.
@@ -166,6 +170,40 @@ controls are repeated to expose duplicate side effects. Generic oracles cover:
166170
Journey minimization uses deterministic subset replay to retain the shortest
167171
sequence that preserves the oracle failure.
168172

173+
### Accessibility and keyboard exploration
174+
175+
Add `accessibility` to `adaptive-exploration-json` to scan the initial state,
176+
each novel state, and the final state. The contract accepts `ruleTags`,
177+
`includeScopes`, `excludeScopes`, `impactThreshold`, `cadence`, required or
178+
optional `rules`, `focus`, and `accessibilityTree` capabilities, plus explicit
179+
scan, violation, focus-history, tree-size, and keyboard-action budgets.
180+
181+
```json
182+
{
183+
"schema": "wp-codebox/browser-adaptive-exploration/v1",
184+
"seed": "accessible-settings",
185+
"startUrl": "/settings/",
186+
"accessibility": {
187+
"ruleTags": ["accessible-name", "keyboard-reachable", "focus-visible", "dialog-focus", "aria-state"],
188+
"impactThreshold": "serious",
189+
"cadence": ["initial", "novel-state", "final"],
190+
"capabilities": { "rules": "required", "focus": "required", "accessibilityTree": "optional" },
191+
"budgets": { "maxScans": 24, "maxViolationsPerScan": 25, "maxFocusTransitions": 100, "maxTreeChars": 20000, "maxKeyboardActions": 8 }
192+
}
193+
}
194+
```
195+
196+
The collector classifies unnamed controls, keyboard-unreachable custom
197+
controls, unexpected tab stops, hidden/inert/clipped/offscreen focus, focus loss,
198+
dialog entry/containment/restoration failures, and observable ARIA state drift.
199+
Every finding records its oracle, rule, impact, structural target, state and
200+
transition context, screenshot, correlated bounded DOM snapshot, and stable
201+
fingerprint. ARIA-tree names are redacted and target evidence excludes text.
202+
Unavailable tree capabilities produce an explicit `unsupported` scan. Required
203+
capabilities make the exploration incomplete; optional capabilities allow the
204+
campaign to continue without turning the unsupported check into a pass.
205+
Existing caller-supplied `accessibilityViolations` remain supported.
206+
169207
## Evidence and safety
170208

171209
`writeAdversarialEvidenceBundle()` writes a manifest, campaign result, one

examples/recipes/adversarial-adaptive-browser.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
{
3535
"command": "wordpress.browser-actions",
3636
"args": [
37-
"adaptive-exploration-json={\"schema\":\"wp-codebox/browser-adaptive-exploration/v1\",\"seed\":\"adaptive-browser-v1\",\"startUrl\":\"/\",\"budgets\":{\"maxActions\":24,\"maxStates\":16,\"maxTransitions\":32,\"maxDurationMs\":60000,\"maxArtifactBytes\":2097152,\"maxErrors\":10},\"actionFamilies\":[\"click\",\"fill\",\"select\",\"submit\",\"keyboard\",\"back\",\"reload\",\"repeat\",\"double-submit\"],\"resetPolicy\":{\"mode\":\"start-url\"}}",
37+
"adaptive-exploration-json={\"schema\":\"wp-codebox/browser-adaptive-exploration/v1\",\"seed\":\"adaptive-browser-v1\",\"startUrl\":\"/\",\"budgets\":{\"maxActions\":24,\"maxStates\":16,\"maxTransitions\":32,\"maxDurationMs\":60000,\"maxArtifactBytes\":2097152,\"maxErrors\":10},\"actionFamilies\":[\"click\",\"fill\",\"select\",\"submit\",\"keyboard\",\"back\",\"reload\",\"repeat\",\"double-submit\"],\"resetPolicy\":{\"mode\":\"start-url\"},\"accessibility\":{\"ruleTags\":[\"accessible-name\",\"keyboard-reachable\",\"focus-visible\",\"dialog-focus\",\"aria-state\"],\"impactThreshold\":\"serious\",\"cadence\":[\"initial\",\"novel-state\",\"final\"],\"capabilities\":{\"rules\":\"required\",\"focus\":\"required\",\"accessibilityTree\":\"optional\"},\"budgets\":{\"maxScans\":24,\"maxViolationsPerScan\":25,\"maxFocusTransitions\":100,\"maxTreeChars\":20000,\"maxKeyboardActions\":8}}}",
3838
"capture=steps,console,errors,network"
3939
]
4040
}

package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,7 @@
233233
"test:fuzz-run-recipe": "tsx tests/fuzz-run-recipe.test.ts",
234234
"test:fuzz-suite-runner": "tsx tests/fuzz-suite-runner.test.ts",
235235
"test:adversarial-runtime": "tsx --test tests/transport-faults.test.ts tests/adversarial-campaign.test.ts tests/adversarial-browser.test.ts tests/browser-clock-control.test.ts tests/adversarial-recipe-orchestration.test.ts tests/wordpress-adversarial-adapter.test.ts",
236+
"test:browser-accessibility-oracles": "tsx --test tests/browser-accessibility-oracles.test.ts tests/browser-adaptive-exploration.test.ts tests/adversarial-browser.test.ts",
236237
"test:wordpress-adversarial-adapter": "tsx tests/wordpress-adversarial-adapter.test.ts",
237238
"test:playground-fuzz-suite-public": "tsx tests/playground-fuzz-suite-public.test.ts",
238239
"test:nested-fuzz-suite-recipe-command": "tsx tests/nested-fuzz-suite-recipe-command.test.ts",

packages/runtime-core/src/adversarial-browser.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { createHash } from "node:crypto"
22

33
import type { BrowserActionCorpusDescriptor, BrowserInteractionStep } from "./browser-interaction.js"
4+
import type { BrowserAccessibilityFinding } from "./browser-accessibility.js"
45

56
export const ADVERSARIAL_BROWSER_PLAN_SCHEMA = "wp-codebox/adversarial-browser-plan/v1" as const
67
export const ADVERSARIAL_BROWSER_ORACLE_RESULT_SCHEMA = "wp-codebox/adversarial-browser-oracle-result/v1" as const
@@ -26,6 +27,7 @@ export interface AdversarialBrowserObservation {
2627
loadingIndicators?: Array<{ id: string; visibleForMs: number }>
2728
boxes?: Array<{ id: string; x: number; y: number; width: number; height: number; viewportWidth: number; viewportHeight: number; focused?: boolean; visible?: boolean; clipped?: boolean }>
2829
accessibilityViolations?: Array<{ rule: string; target: string; impact?: string }>
30+
accessibilityFindings?: BrowserAccessibilityFinding[]
2931
effects?: Array<{ id: string; count: number; expectedMaximum?: number }>
3032
}
3133

@@ -91,6 +93,7 @@ export function evaluateAdversarialBrowserOracles(observation: AdversarialBrowse
9193
if (box.focused && box.visible === false) failures.push({ oracle: "layout", code: "browser-invisible-focus", message: `Focused element ${box.id} is not visible.`, target: box.id })
9294
}
9395
for (const violation of observation.accessibilityViolations ?? []) failures.push({ oracle: "accessibility", code: `browser-a11y-${violation.rule}`, message: `Accessibility rule ${violation.rule} failed.`, target: violation.target })
96+
for (const finding of observation.accessibilityFindings ?? []) failures.push({ oracle: finding.oracle === "accessibility" ? "accessibility" : finding.oracle === "focus" ? "layout" : "dead-control", code: finding.code, message: `${finding.classification} (${finding.impact}).`, target: finding.target.locator })
9497
for (const effect of observation.effects ?? []) {
9598
if (effect.count > (effect.expectedMaximum ?? 1)) failures.push({ oracle: "duplicate-effect", code: "browser-duplicate-effect", message: `Effect ${effect.id} occurred ${effect.count} times.`, target: effect.id })
9699
}
Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
import { createHash } from "node:crypto"
2+
3+
import type { BrowserAdaptiveAction } from "./browser-adaptive-exploration.js"
4+
import { isPlainObject, stableJson, stripUndefined } from "./object-utils.js"
5+
6+
export const BROWSER_ACCESSIBILITY_SCHEMA = "wp-codebox/browser-accessibility/v1" as const
7+
export const BROWSER_ACCESSIBILITY_RULES = [
8+
"accessible-name",
9+
"keyboard-reachable",
10+
"tab-order",
11+
"focus-visible",
12+
"focus-loss",
13+
"dialog-focus",
14+
"aria-state",
15+
] as const
16+
17+
export type BrowserAccessibilityRule = typeof BROWSER_ACCESSIBILITY_RULES[number]
18+
export type BrowserAccessibilityImpact = "minor" | "moderate" | "serious" | "critical"
19+
export type BrowserAccessibilityScanPhase = "initial" | "novel-state" | "final" | "replay"
20+
21+
export interface BrowserAccessibilityContract {
22+
schema: typeof BROWSER_ACCESSIBILITY_SCHEMA
23+
ruleTags: BrowserAccessibilityRule[]
24+
includeScopes: string[]
25+
excludeScopes: string[]
26+
impactThreshold: BrowserAccessibilityImpact
27+
cadence: Array<"initial" | "novel-state" | "final">
28+
capabilities: {
29+
rules: "required" | "optional"
30+
focus: "required" | "optional"
31+
accessibilityTree: "required" | "optional" | "disabled"
32+
}
33+
budgets: {
34+
maxScans: number
35+
maxViolationsPerScan: number
36+
maxTargetsPerViolation: number
37+
maxFocusTransitions: number
38+
maxTreeChars: number
39+
maxKeyboardActions: number
40+
maxFrames: number
41+
}
42+
}
43+
44+
export interface BrowserAccessibilityTargetEvidence {
45+
locator: string
46+
frameId: string
47+
tag: string
48+
role?: string
49+
states?: Partial<Record<"expanded" | "selected" | "checked" | "busy" | "pressed" | "hidden" | "inert", string>>
50+
box?: { x: number; y: number; width: number; height: number; viewportWidth: number; viewportHeight: number }
51+
}
52+
53+
export interface BrowserAccessibilityFinding {
54+
oracle: "accessibility" | "keyboard" | "focus"
55+
rule: BrowserAccessibilityRule
56+
code: string
57+
impact: BrowserAccessibilityImpact
58+
classification: string
59+
fingerprint: string
60+
target: BrowserAccessibilityTargetEvidence
61+
stateDigest?: string
62+
transitionId?: string
63+
actionId?: string
64+
expected?: string
65+
actual?: string
66+
}
67+
68+
export interface BrowserAccessibilityFocusTransition {
69+
index: number
70+
phase: BrowserAccessibilityScanPhase
71+
actionId?: string
72+
from?: string
73+
to: string
74+
visible: boolean
75+
insideDialog: boolean
76+
}
77+
78+
export interface BrowserAccessibilityScan {
79+
index: number
80+
phase: BrowserAccessibilityScanPhase
81+
status: "passed" | "findings" | "unsupported" | "inconclusive"
82+
stateDigest?: string
83+
transitionId?: string
84+
actionId?: string
85+
findings: BrowserAccessibilityFinding[]
86+
accessibilityTree?: { status: "captured" | "unsupported" | "inconclusive"; snapshot?: string; reason?: string; truncated?: boolean }
87+
diagnostics: Array<{ code: string; message: string }>
88+
artifacts?: { screenshot?: string; domSnapshot?: string }
89+
}
90+
91+
export interface BrowserAccessibilityEvidence {
92+
schema: typeof BROWSER_ACCESSIBILITY_SCHEMA
93+
collector: { name: string; version: string; capabilities: { rules: "supported" | "unsupported"; focus: "supported" | "unsupported"; accessibilityTree: "supported" | "unsupported" } }
94+
contract: BrowserAccessibilityContract
95+
scans: BrowserAccessibilityScan[]
96+
focusHistory: BrowserAccessibilityFocusTransition[]
97+
diagnostics: Array<{ code: string; message: string }>
98+
summary: { scans: number; findings: number; passed: number; unsupported: number; inconclusive: number; truncated: boolean }
99+
}
100+
101+
export interface BrowserAccessibilityCollector {
102+
beforeAction(action: BrowserAdaptiveAction): Promise<void>
103+
scan(input: { phase: BrowserAccessibilityScanPhase; stateDigest?: string; transitionId?: string; action?: BrowserAdaptiveAction; record?: boolean }): Promise<BrowserAccessibilityScan>
104+
reset(): Promise<void>
105+
evidence(): BrowserAccessibilityEvidence
106+
}
107+
108+
export function browserAccessibilityContract(input: unknown): BrowserAccessibilityContract | undefined {
109+
if (!isPlainObject(input) || input.enabled === false) return undefined
110+
const budgets = object(input.budgets)
111+
const capabilities = object(input.capabilities)
112+
const requestedRules = array(input.ruleTags ?? input.rule_tags).filter((value): value is BrowserAccessibilityRule => (BROWSER_ACCESSIBILITY_RULES as readonly unknown[]).includes(value))
113+
const requestedCadence = array(input.cadence).filter((value): value is "initial" | "novel-state" | "final" => value === "initial" || value === "novel-state" || value === "final")
114+
return {
115+
schema: BROWSER_ACCESSIBILITY_SCHEMA,
116+
ruleTags: requestedRules.length > 0 ? [...new Set(requestedRules)] : [...BROWSER_ACCESSIBILITY_RULES],
117+
includeScopes: boundedStrings(input.includeScopes ?? input.include_scopes, 20),
118+
excludeScopes: boundedStrings(input.excludeScopes ?? input.exclude_scopes, 20),
119+
impactThreshold: impact(input.impactThreshold ?? input.impact_threshold),
120+
cadence: requestedCadence.length > 0 ? [...new Set(requestedCadence)] : ["initial", "novel-state", "final"],
121+
capabilities: {
122+
rules: requirement(capabilities.rules),
123+
focus: requirement(capabilities.focus),
124+
accessibilityTree: capabilities.accessibilityTree === "disabled" || capabilities.accessibility_tree === "disabled" ? "disabled" : requirement(capabilities.accessibilityTree ?? capabilities.accessibility_tree),
125+
},
126+
budgets: {
127+
maxScans: integer(budgets.maxScans ?? budgets.max_scans, 32, 1, 500),
128+
maxViolationsPerScan: integer(budgets.maxViolationsPerScan ?? budgets.max_violations_per_scan, 25, 1, 200),
129+
maxTargetsPerViolation: integer(budgets.maxTargetsPerViolation ?? budgets.max_targets_per_violation, 3, 1, 20),
130+
maxFocusTransitions: integer(budgets.maxFocusTransitions ?? budgets.max_focus_transitions, 100, 1, 1_000),
131+
maxTreeChars: integer(budgets.maxTreeChars ?? budgets.max_tree_chars, 20_000, 256, 200_000),
132+
maxKeyboardActions: integer(budgets.maxKeyboardActions ?? budgets.max_keyboard_actions, 12, 0, 50),
133+
maxFrames: integer(budgets.maxFrames ?? budgets.max_frames, 16, 1, 100),
134+
},
135+
}
136+
}
137+
138+
export function browserAccessibilityFindingFingerprint(finding: Omit<BrowserAccessibilityFinding, "fingerprint">): string {
139+
const stable = stripUndefined({
140+
oracle: finding.oracle,
141+
rule: finding.rule,
142+
code: finding.code,
143+
impact: finding.impact,
144+
classification: finding.classification,
145+
target: {
146+
locator: normalizeLocator(finding.target.locator),
147+
frameId: finding.target.frameId,
148+
tag: finding.target.tag,
149+
role: finding.target.role,
150+
states: finding.target.states,
151+
},
152+
expected: finding.expected,
153+
actual: finding.actual,
154+
})
155+
return createHash("sha256").update("wp-codebox/browser-accessibility-finding/v1\n").update(stableJson(stable)).digest("hex")
156+
}
157+
158+
function normalizeLocator(value: string): string {
159+
return value.replace(/([_:-](?:[a-z]{0,4})?)[a-f0-9]{8,}/gi, "$1<generated>").slice(0, 240)
160+
}
161+
162+
function object(value: unknown): Record<string, unknown> { return isPlainObject(value) ? value : {} }
163+
function array(value: unknown): unknown[] { return Array.isArray(value) ? value : [] }
164+
function boundedStrings(value: unknown, maximum: number): string[] { return array(value).filter((item): item is string => typeof item === "string" && item.trim().length > 0).slice(0, maximum).map((item) => item.trim().slice(0, 240)) }
165+
function requirement(value: unknown): "required" | "optional" { return value === "required" ? "required" : "optional" }
166+
function impact(value: unknown): BrowserAccessibilityImpact { return value === "minor" || value === "moderate" || value === "critical" ? value : "serious" }
167+
function integer(value: unknown, fallback: number, minimum: number, maximum: number): number { const numeric = Number(value); return Number.isFinite(numeric) ? Math.max(minimum, Math.min(maximum, Math.floor(numeric))) : fallback }

0 commit comments

Comments
 (0)