diff --git a/.changeset/swift-geckos-arrive.md b/.changeset/swift-geckos-arrive.md new file mode 100644 index 000000000..f1a8f86b8 --- /dev/null +++ b/.changeset/swift-geckos-arrive.md @@ -0,0 +1,5 @@ +--- +"@browserbasehq/stagehand": patch +--- + +include-native-selector-state-into-snapshot diff --git a/packages/core/lib/v3/types/private/snapshot.ts b/packages/core/lib/v3/types/private/snapshot.ts index 0d3a6f2a1..a1aca0e56 100644 --- a/packages/core/lib/v3/types/private/snapshot.ts +++ b/packages/core/lib/v3/types/private/snapshot.ts @@ -101,6 +101,7 @@ export type A11yNode = { name?: string; description?: string; value?: string | number | boolean; + state?: "selected" | "checked"; nodeId: string; backendDOMNodeId?: number; parentId?: string; diff --git a/packages/core/lib/v3/understudy/a11y/snapshot/a11yTree.ts b/packages/core/lib/v3/understudy/a11y/snapshot/a11yTree.ts index a8d70b056..72cf12dbd 100644 --- a/packages/core/lib/v3/understudy/a11y/snapshot/a11yTree.ts +++ b/packages/core/lib/v3/understudy/a11y/snapshot/a11yTree.ts @@ -9,6 +9,14 @@ import { resolveObjectIdForCss, resolveObjectIdForXPath, } from "./focusSelectors.js"; +import { + A11Y_STATE_CHECKED, + A11Y_STATE_SELECTED, + AX_BOOLEAN_FALSE_NUMBER, + AX_BOOLEAN_FALSE_STRING, + AX_BOOLEAN_TRUE_NUMBER, + AX_BOOLEAN_TRUE_STRING, +} from "./constants.js"; import { formatTreeLine, normaliseSpaces } from "./treeFormatUtils.js"; /** @@ -141,11 +149,14 @@ export function decorateRoles( role = tag; } + const state = resolveA11yState(n); + return { role, name: n.name?.value, description: n.description?.value, value: n.value?.value, + state, nodeId: n.nodeId, backendDOMNodeId: n.backendDOMNodeId, parentId: n.parentId, @@ -235,6 +246,50 @@ export function extractUrlFromAXNode( return typeof value === "string" && value.trim() ? value.trim() : undefined; } + +function toBoolean( + value: Protocol.Accessibility.AXValue | undefined, +): boolean | undefined { + const raw = value?.value; + if (typeof raw === "boolean") return raw; + if (typeof raw === "number") { + if (raw === AX_BOOLEAN_TRUE_NUMBER) return true; + if (raw === AX_BOOLEAN_FALSE_NUMBER) return false; + } + if (typeof raw === "string") { + const normalized = raw.toLowerCase(); + if (normalized === AX_BOOLEAN_TRUE_STRING) return true; + if (normalized === AX_BOOLEAN_FALSE_STRING) return false; + } + return undefined; +} + +function extractBooleanState( + node: Protocol.Accessibility.AXNode, + stateName: string, +): boolean | undefined { + const directValue = ( + node as unknown as Record + )[stateName]; + const direct = toBoolean(directValue); + if (direct !== undefined) return direct; + + const fromProperties = node.properties?.find((prop) => prop.name === stateName); + return toBoolean(fromProperties?.value); +} + +function resolveA11yState( + node: Protocol.Accessibility.AXNode, +): A11yNode["state"] { + const checked = extractBooleanState(node, A11Y_STATE_CHECKED); + if (checked === true) return A11Y_STATE_CHECKED; + + const selected = extractBooleanState(node, A11Y_STATE_SELECTED); + if (selected === true) return A11Y_STATE_SELECTED; + + return undefined; +} + export function removeRedundantStaticTextChildren( parent: A11yNode, children: A11yNode[], diff --git a/packages/core/lib/v3/understudy/a11y/snapshot/constants.ts b/packages/core/lib/v3/understudy/a11y/snapshot/constants.ts new file mode 100644 index 000000000..16ec387ca --- /dev/null +++ b/packages/core/lib/v3/understudy/a11y/snapshot/constants.ts @@ -0,0 +1,14 @@ +export const A11Y_STATE_SELECTED = "selected"; +export const A11Y_STATE_CHECKED = "checked"; + +export const A11Y_SELECTED_VISIBLE_ROLES = new Set([ + "option", + "menuitemradio", + "tab", + "treeitem", +]); + +export const AX_BOOLEAN_TRUE_NUMBER = 1; +export const AX_BOOLEAN_FALSE_NUMBER = 0; +export const AX_BOOLEAN_TRUE_STRING = "true"; +export const AX_BOOLEAN_FALSE_STRING = "false"; diff --git a/packages/core/lib/v3/understudy/a11y/snapshot/treeFormatUtils.ts b/packages/core/lib/v3/understudy/a11y/snapshot/treeFormatUtils.ts index fd90e1529..25fd61857 100644 --- a/packages/core/lib/v3/understudy/a11y/snapshot/treeFormatUtils.ts +++ b/packages/core/lib/v3/understudy/a11y/snapshot/treeFormatUtils.ts @@ -1,4 +1,9 @@ import type { A11yNode } from "../../../types/private/snapshot.js"; +import { + A11Y_SELECTED_VISIBLE_ROLES, + A11Y_STATE_CHECKED, + A11Y_STATE_SELECTED, +} from "./constants.js"; /** * Render a formatted outline (with encoded ids) for the accessibility tree. @@ -8,12 +13,30 @@ import type { A11yNode } from "../../../types/private/snapshot.js"; export function formatTreeLine(node: A11yNode, level = 0): string { const indent = " ".repeat(level); const labelId = node.encodedId ?? node.nodeId; - const label = `[${labelId}] ${node.role}${node.name ? `: ${cleanText(node.name)}` : ""}`; + const stateFlags = formatStateFlags(node); + const label = `[${labelId}] ${node.role}${node.name ? `: ${cleanText(node.name)}` : ""}${stateFlags}`; const kids = node.children?.map((c) => formatTreeLine(c, level + 1)).join("\n") ?? ""; return kids ? `${indent}${label}\n${kids}` : `${indent}${label}`; } +function formatStateFlags(node: A11yNode): string { + const flags: string[] = []; + + if ( + node.state === A11Y_STATE_CHECKED || + (node.state === A11Y_STATE_SELECTED && shouldRenderSelected(node.role)) + ) { + flags.push(node.state); + } + return flags.length ? ` [${flags.join(", ")}]` : ""; +} + +function shouldRenderSelected(role: string): boolean { + const normalizedRole = role.toLowerCase(); + return A11Y_SELECTED_VISIBLE_ROLES.has(normalizedRole); +} + /** * Inject each child frame outline under the parent's iframe node line. * Keys in `idToTree` are the parent's iframe encoded ids. diff --git a/packages/core/tests/unit/snapshot-a11y-tree-utils.test.ts b/packages/core/tests/unit/snapshot-a11y-tree-utils.test.ts index 7b03212d3..6a442d429 100644 --- a/packages/core/tests/unit/snapshot-a11y-tree-utils.test.ts +++ b/packages/core/tests/unit/snapshot-a11y-tree-utils.test.ts @@ -17,6 +17,11 @@ const axString = (value: string): Protocol.Accessibility.AXValue => ({ value, }); +const axBool = (value: boolean): Protocol.Accessibility.AXValue => ({ + type: "boolean", + value, +}); + const defaultOpts: A11yOptions = { focusSelector: undefined, experimental: false, @@ -114,6 +119,39 @@ describe("decorateRoles", () => { const decorated = decorateRoles(nodes, opts); expect(decorated[0]?.encodedId).toBeUndefined(); }); + + it("maps selected/checked AX states into a single state field", () => { + const nodes = [ + makeAxNode({ + backendDOMNodeId: 12, + role: axString("option"), + name: axString("Option B"), + properties: [ + { name: "selected", value: axBool(true) }, + ], + }), + makeAxNode({ + backendDOMNodeId: 13, + role: axString("radio"), + name: axString("Option C"), + properties: [{ name: "checked", value: axBool(true) }], + }), + makeAxNode({ + backendDOMNodeId: 14, + role: axString("radio"), + name: axString("Option D"), + properties: [ + { name: "selected", value: axBool(true) }, + { name: "checked", value: axBool(true) }, + ], + }), + ]; + + const decorated = decorateRoles(nodes, defaultOpts); + expect(decorated[0]?.state).toBe("selected"); + expect(decorated[1]?.state).toBe("checked"); + expect(decorated[2]?.state).toBe("checked"); + }); }); describe("buildHierarchicalTree", () => { diff --git a/packages/core/tests/unit/snapshot-tree-format-utils.test.ts b/packages/core/tests/unit/snapshot-tree-format-utils.test.ts index 3245cabbd..da5c1c434 100644 --- a/packages/core/tests/unit/snapshot-tree-format-utils.test.ts +++ b/packages/core/tests/unit/snapshot-tree-format-utils.test.ts @@ -28,6 +28,65 @@ describe("formatTreeLine", () => { "[frame-1] section: Container\n [ax-2] button: Submit", ); }); + + it("renders a select with child options and only one selected option", () => { + const outline = formatTreeLine({ + role: "select", + name: "Select field", + nodeId: "ax-4", + children: [ + { role: "option", name: "Option A", nodeId: "ax-5" }, + { + role: "option", + name: "Option B", + state: "selected", + nodeId: "ax-6", + }, + { role: "option", name: "Option C", nodeId: "ax-7" }, + ], + }); + + expect(outline).toBe( + "[ax-4] select: Select field\n [ax-5] option: Option A\n [ax-6] option: Option B [selected]\n [ax-7] option: Option C", + ); + expect(outline.match(/\[selected]/g)?.length ?? 0).toBe(1); + }); + + it("renders a radio group with children and only one checked radio", () => { + const outline = formatTreeLine({ + role: "group", + name: "Select field", + nodeId: "ax-8", + children: [ + { role: "radio", name: "Option A", nodeId: "ax-9" }, + { role: "radio", name: "Option B", state: "checked", nodeId: "ax-10" }, + { role: "radio", name: "Option C", nodeId: "ax-11" }, + ], + }); + + expect(outline).toBe( + "[ax-8] group: Select field\n [ax-9] radio: Option A\n [ax-10] radio: Option B [checked]\n [ax-11] radio: Option C", + ); + expect(outline.match(/\[checked]/g)?.length ?? 0).toBe(1); + }); + + it("does not render [selected] on combobox/select nodes", () => { + const combo = formatTreeLine({ + role: "combobox", + name: "Select field", + state: "selected", + nodeId: "ax-12", + }); + const select = formatTreeLine({ + role: "select", + name: "Another select", + state: "selected", + nodeId: "ax-13", + }); + + expect(combo).toBe("[ax-12] combobox: Select field"); + expect(select).toBe("[ax-13] select: Another select"); + }); }); describe("injectSubtrees", () => {