Skip to content

Commit f356e33

Browse files
authored
fix: classify adaptive input actions by control type (#2070)
1 parent 882afc1 commit f356e33

2 files changed

Lines changed: 99 additions & 8 deletions

File tree

packages/runtime-core/src/browser-adaptive-exploration.ts

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@ export const BROWSER_ADAPTIVE_EXPLORATION_ARTIFACT_SCHEMA = "wp-codebox/browser-
1010
export const BROWSER_ADAPTIVE_ACTION_FAMILIES = ["click", "fill", "select", "submit", "keyboard", "back", "reload", "repeat", "double-submit"] as const
1111
export type BrowserAdaptiveActionFamily = typeof BROWSER_ADAPTIVE_ACTION_FAMILIES[number]
1212

13+
const TEXT_FILLABLE_INPUT_TYPES = new Set(["text", "search", "tel", "url", "email", "password", "number"])
14+
const CLICKABLE_INPUT_TYPES = new Set(["checkbox", "radio", "button", "reset", "submit", "image"])
15+
const SUBMIT_INPUT_TYPES = new Set(["submit", "image"])
16+
1317
export interface BrowserAdaptiveExplorationContract {
1418
schema: typeof BROWSER_ADAPTIVE_EXPLORATION_SCHEMA
1519
context: BrowserRandomWalkContext
@@ -197,28 +201,35 @@ export function planBrowserAdaptiveStateActions(state: BrowserAdaptiveState, con
197201
const id = `${family}:${frameId}:${descriptor?.id ?? state.digest}`
198202
actions.push({ id, family, frameId, ...(descriptor ? { descriptorId: descriptor.id } : {}), steps, ...(input !== undefined ? { input } : {}) })
199203
}
204+
const addClicks = (descriptor: BrowserActionCorpusDescriptor, submit: boolean) => {
205+
const selector = descriptor.selector
206+
add("click", descriptor, [{ kind: "click", selector }])
207+
add("repeat", descriptor, [{ kind: "click", selector }, { kind: "click", selector }])
208+
if (submit && descriptor.formId) {
209+
add("submit", descriptor, [{ kind: "click", selector }])
210+
add("double-submit", descriptor, [{ kind: "click", selector }, { kind: "click", selector }])
211+
}
212+
}
200213
for (const descriptor of state.descriptors) {
201214
if (descriptor.disabled || descriptor.readonly) continue
202215
const selector = descriptor.selector
203-
if (descriptor.kind === "input" || descriptor.kind === "textarea") {
216+
const inputType = (descriptor.type ?? "text").toLowerCase()
217+
if (descriptor.kind === "textarea" || (descriptor.kind === "input" && TEXT_FILLABLE_INPUT_TYPES.has(inputType))) {
204218
const value = generatedValue(contract.seed, descriptor)
205219
add("fill", descriptor, [{ kind: "fill", selector, value }], value)
206220
add("keyboard", descriptor, [{ kind: "press", selector, key: "Enter" }])
207221
if (descriptor.formId) add("submit", descriptor, [{ kind: "fill", selector, value }, { kind: "press", selector, key: "Enter" }], value)
208222
add("repeat", descriptor, [{ kind: "fill", selector, value }, { kind: "fill", selector, value }], value)
223+
} else if (descriptor.kind === "input" && CLICKABLE_INPUT_TYPES.has(inputType)) {
224+
addClicks(descriptor, SUBMIT_INPUT_TYPES.has(inputType))
209225
} else if (descriptor.kind === "select") {
210226
const values = descriptor.optionValues?.filter(Boolean) ?? []
211227
if (values.length > 0) {
212228
const value = values[parseInt(browserAdaptiveDigest("action", `${contract.seed}:${descriptor.id}`).slice(0, 8), 16) % values.length] as string
213229
add("select", descriptor, [{ kind: "select", selector, value }], value)
214230
}
215-
} else {
216-
add("click", descriptor, [{ kind: "click", selector }])
217-
add("repeat", descriptor, [{ kind: "click", selector }, { kind: "click", selector }])
218-
if (descriptor.type === "submit" && descriptor.formId) {
219-
add("submit", descriptor, [{ kind: "click", selector }])
220-
add("double-submit", descriptor, [{ kind: "click", selector }, { kind: "click", selector }])
221-
}
231+
} else if (descriptor.kind !== "input") {
232+
addClicks(descriptor, inputType === "submit")
222233
}
223234
}
224235
if (contract.accessibility && contract.actionFamilies.includes("keyboard")) {

tests/browser-adaptive-exploration.test.ts

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@ import {
99
planBrowserAdaptiveStateActions,
1010
} from "../packages/runtime-core/src/browser-adaptive-exploration.js"
1111
import { getCommandDefinition } from "../packages/runtime-core/src/command-registry.js"
12+
import { discoverBrowserActionCorpusDescriptors } from "../packages/runtime-playground/src/browser-action-discovery.js"
1213
import { exploreAdaptiveBrowserStateMachine } from "../packages/runtime-playground/src/browser-adaptive-explorer.js"
14+
import { executeBrowserInteractionStep } from "../packages/runtime-playground/src/browser-interactions.js"
1315
import { browserPreviewTopology, routeBrowserPreviewContextNetwork } from "../packages/runtime-playground/src/browser-preview-routing.js"
1416
import { closeHttpServer, listenLocalHttpServer } from "../packages/runtime-playground/src/preview-server.js"
1517

@@ -111,6 +113,84 @@ test("adaptive contract normalizes every safety and exploration bound", () => {
111113
assert.deepEqual(new Set(planned.map((action) => action.family)), new Set(["click", "fill", "select", "submit", "keyboard", "back", "reload", "repeat", "double-submit"]))
112114
})
113115

116+
test("adaptive planner selects actions by HTML input type capability", () => {
117+
const fillableTypes = ["text", "search", "tel", "url", "email", "password", "number"]
118+
const toggleTypes = ["checkbox", "radio"]
119+
const buttonTypes = ["button", "reset"]
120+
const submitTypes = ["submit", "image"]
121+
const unsupportedTypes = ["color", "date", "datetime-local", "file", "hidden", "month", "range", "time", "week"]
122+
const descriptors = [
123+
{ id: "default", kind: "input" as const, selector: "#default", formId: "form", frameId: "document" },
124+
...[...fillableTypes, ...toggleTypes, ...buttonTypes, ...submitTypes, ...unsupportedTypes].map((type) => ({ id: type, kind: "input" as const, selector: `#${type}`, type, formId: "form", frameId: "document" })),
125+
{ id: "textarea", kind: "textarea" as const, selector: "#textarea", formId: "form", frameId: "document" },
126+
{ id: "disabled", kind: "input" as const, selector: "#disabled", type: "text", disabled: true, frameId: "document" },
127+
{ id: "readonly", kind: "input" as const, selector: "#readonly", type: "text", readonly: true, frameId: "document" },
128+
]
129+
const contract = browserAdaptiveExplorationContract({ seed: "input-capabilities", startUrl: "/fixture" })
130+
const planned = planBrowserAdaptiveStateActions({
131+
digest: "state",
132+
url: "/fixture",
133+
historyLength: 1,
134+
historyStateDigest: "history",
135+
descriptorDigest: "descriptors",
136+
descriptors,
137+
frames: [{ id: "document", url: "/fixture", scope: "document" }],
138+
visits: 1,
139+
depth: 0,
140+
loadingIndicators: 0,
141+
}, contract)
142+
const familiesFor = (id: string) => planned.filter((action) => action.descriptorId === id).map((action) => action.family)
143+
144+
for (const type of ["default", ...fillableTypes, "textarea"]) assert.deepEqual(familiesFor(type), ["fill", "keyboard", "submit", "repeat"], type)
145+
for (const type of toggleTypes) assert.deepEqual(familiesFor(type), ["click", "repeat"], type)
146+
for (const type of buttonTypes) assert.deepEqual(familiesFor(type), ["click", "repeat"], type)
147+
for (const type of submitTypes) assert.deepEqual(familiesFor(type), ["click", "repeat", "submit", "double-submit"], type)
148+
for (const type of [...unsupportedTypes, "disabled", "readonly"]) assert.deepEqual(familiesFor(type), [], type)
149+
assert(planned.filter((action) => action.steps.some((step) => step.kind === "fill")).every((action) => ["default", ...fillableTypes, "textarea"].includes(action.descriptorId ?? "")))
150+
})
151+
152+
test("planned input actions execute safely in a real browser", async () => {
153+
const browser = await chromium.launch({ headless: true })
154+
const page = await browser.newPage()
155+
const fixture = `<!doctype html>
156+
<style>input,textarea { display:block; width:180px; height:30px; margin:4px; }</style>
157+
<form id="form">
158+
<input id="checkbox" type="checkbox"><input id="radio" type="radio" name="choice">
159+
<input id="text" type="text"><input id="password" type="password"><input id="email" type="email"><input id="search" type="search">
160+
<textarea id="textarea"></textarea><input id="submit" type="submit" value="Submit"><input id="range" type="range"><input id="file" type="file">
161+
</form>
162+
<script>document.querySelector('#form').addEventListener('submit', (event) => event.preventDefault())</script>`
163+
try {
164+
await page.addInitScript("globalThis.__name = value => value")
165+
await page.setContent(fixture)
166+
await page.evaluate("globalThis.__name = value => value")
167+
const discovery = await discoverBrowserActionCorpusDescriptors(page)
168+
const contract = browserAdaptiveExplorationContract({ seed: "browser-input-capabilities", startUrl: page.url() })
169+
const actions = planBrowserAdaptiveStateActions({
170+
digest: "state",
171+
url: page.url(),
172+
historyLength: 1,
173+
historyStateDigest: "history",
174+
descriptorDigest: "descriptors",
175+
descriptors: discovery.descriptors,
176+
frames: [{ id: "document", url: page.url(), scope: "document" }],
177+
visits: 1,
178+
depth: 0,
179+
loadingIndicators: 0,
180+
}, contract).filter((action) => action.descriptorId)
181+
182+
assert(actions.some((action) => action.descriptorId?.includes("\"type\":\"checkbox\"") && action.steps.every((step) => step.kind === "click")))
183+
assert(actions.some((action) => action.descriptorId?.includes("\"type\":\"radio\"") && action.steps.every((step) => step.kind === "click")))
184+
assert(!actions.some((action) => action.descriptorId?.includes("\"type\":\"range\"") || action.descriptorId?.includes("\"type\":\"file\"")))
185+
for (const action of actions) {
186+
await page.setContent(fixture)
187+
for (const step of action.steps) await executeBrowserInteractionStep(page, step, page.url(), 2_000, async () => ({ path: "unused", isDefault: false }))
188+
}
189+
} finally {
190+
await browser.close()
191+
}
192+
})
193+
114194
test("rediscovery finds, minimizes, and replays a defect revealed by a dynamic modal", async () => {
115195
const run = await runFixture(modalFixture, {
116196
seed: "modal-seed",

0 commit comments

Comments
 (0)