Skip to content

Commit cb586a1

Browse files
[fix]: include native selector state into snapshot (#2116)
thanks to @imorhun for this contribution! original PR can be found here #2105 # why Stagehand accessibility snapshot output was missing native selection state markers (`selected` / `checked`) that are visible in Playwright `ariaSnapshot({ mode: "ai" })`. That mismatch made it impossible to extract the selected/checked item for dropdowns and radio groups. Example: ```html <select aria-label="Select field"> <option>Option A</option> <option selected>Option B</option> <option>Option C</option> </select> ``` Before this fix, Stagehand-style snapshot output did not indicate the selected option: ``` [1-4] select: Select field [1-5] option: Option A [1-6] option: Option B [1-7] option: Option C ``` Expected (and now aligned more closely with Playwright-style output): selected state is explicit: ``` [1-4] select: Select field [1-5] option: Option A [1-6] option: Option B [selected] [1-7] option: Option C ``` # what changed - added explicit boolean fields on `A11yNode`: - `selected?: boolean` - `checked?: boolean` - updated AX mapping in `a11yTree` to: - read native AX `selected` / `checked` from AX properties - normalize boolean-like AX values (`true/false`, `1/0`, `"true"/"false"`) - store the result directly on `selected` / `checked` - updated tree formatting in `treeFormatUtils` to render: - `[selected]` when `selected === true` - `[checked]` when `checked === true` # test plan - AX selected/checked extraction into boolean fields - select/radio parent-child rendering with exactly one selected/checked child - rendering both flags when a node carries both states --------- Co-authored-by: Ihor <vreysu@gmail.com>
1 parent 9806b2d commit cb586a1

6 files changed

Lines changed: 131 additions & 1 deletion

File tree

.changeset/swift-geckos-arrive.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@browserbasehq/stagehand": patch
3+
---
4+
5+
include "[selected]" or "[checked]" state in snapshot

packages/core/lib/v3/types/private/snapshot.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,8 @@ export type A11yNode = {
101101
name?: string;
102102
description?: string;
103103
value?: string | number | boolean;
104+
selected?: boolean;
105+
checked?: boolean;
104106
nodeId: string;
105107
backendDOMNodeId?: number;
106108
parentId?: string;

packages/core/lib/v3/understudy/a11y/snapshot/a11yTree.ts

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,8 @@ export function decorateRoles(
146146
name: n.name?.value,
147147
description: n.description?.value,
148148
value: n.value?.value,
149+
selected: extractBooleanProperty(n, "selected"),
150+
checked: extractBooleanProperty(n, "checked"),
149151
nodeId: n.nodeId,
150152
backendDOMNodeId: n.backendDOMNodeId,
151153
parentId: n.parentId,
@@ -234,6 +236,28 @@ export function extractUrlFromAXNode(
234236
const value = urlProp?.value?.value;
235237
return typeof value === "string" && value.trim() ? value.trim() : undefined;
236238
}
239+
function toBooleanValue(value: unknown): boolean | undefined {
240+
if (typeof value === "boolean") return value;
241+
if (typeof value === "number") {
242+
if (value === 1) return true;
243+
if (value === 0) return false;
244+
}
245+
if (typeof value === "string") {
246+
const normalized = value.toLowerCase();
247+
if (normalized === "true") return true;
248+
if (normalized === "false") return false;
249+
}
250+
return undefined;
251+
}
252+
253+
function extractBooleanProperty(
254+
node: Protocol.Accessibility.AXNode,
255+
propertyName: string,
256+
): boolean | undefined {
257+
const value = node.properties?.find((p) => p.name === propertyName)?.value
258+
?.value;
259+
return toBooleanValue(value);
260+
}
237261

238262
export function removeRedundantStaticTextChildren(
239263
parent: A11yNode,

packages/core/lib/v3/understudy/a11y/snapshot/treeFormatUtils.ts

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,20 @@ import type { A11yNode } from "../../../types/private/snapshot.js";
88
export function formatTreeLine(node: A11yNode, level = 0): string {
99
const indent = " ".repeat(level);
1010
const labelId = node.encodedId ?? node.nodeId;
11-
const label = `[${labelId}] ${node.role}${node.name ? `: ${cleanText(node.name)}` : ""}`;
11+
const stateFlags = formatStateFlags(node);
12+
const label = `[${labelId}] ${node.role}${node.name ? `: ${cleanText(node.name)}` : ""}${stateFlags}`;
1213
const kids =
1314
node.children?.map((c) => formatTreeLine(c, level + 1)).join("\n") ?? "";
1415
return kids ? `${indent}${label}\n${kids}` : `${indent}${label}`;
1516
}
1617

18+
function formatStateFlags(node: A11yNode): string {
19+
let flags = "";
20+
if (node.selected) flags += " [selected]";
21+
if (node.checked) flags += " [checked]";
22+
return flags;
23+
}
24+
1725
/**
1826
* Inject each child frame outline under the parent's iframe node line.
1927
* Keys in `idToTree` are the parent's iframe encoded ids.

packages/core/tests/unit/snapshot-a11y-tree-utils.test.ts

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,11 @@ const axString = (value: string): Protocol.Accessibility.AXValue => ({
1717
value,
1818
});
1919

20+
const axBool = (value: boolean): Protocol.Accessibility.AXValue => ({
21+
type: "boolean",
22+
value,
23+
});
24+
2025
const defaultOpts: A11yOptions = {
2126
focusSelector: undefined,
2227
experimental: false,
@@ -114,6 +119,37 @@ describe("decorateRoles", () => {
114119
const decorated = decorateRoles(nodes, opts);
115120
expect(decorated[0]?.encodedId).toBeUndefined();
116121
});
122+
123+
it("maps selected/checked AX properties into boolean fields", () => {
124+
const nodes = [
125+
makeAxNode({
126+
backendDOMNodeId: 12,
127+
role: axString("option"),
128+
name: axString("Option B"),
129+
properties: [{ name: "selected", value: axBool(true) }],
130+
}),
131+
makeAxNode({
132+
backendDOMNodeId: 13,
133+
role: axString("radio"),
134+
name: axString("Option C"),
135+
properties: [{ name: "checked", value: axBool(true) }],
136+
}),
137+
makeAxNode({
138+
backendDOMNodeId: 14,
139+
role: axString("radio"),
140+
name: axString("Option D"),
141+
properties: [
142+
{ name: "selected", value: axBool(true) },
143+
{ name: "checked", value: axBool(true) },
144+
],
145+
}),
146+
];
147+
148+
const decorated = decorateRoles(nodes, defaultOpts);
149+
expect(decorated[0]).toMatchObject({ selected: true, checked: undefined });
150+
expect(decorated[1]).toMatchObject({ selected: undefined, checked: true });
151+
expect(decorated[2]).toMatchObject({ selected: true, checked: true });
152+
});
117153
});
118154

119155
describe("buildHierarchicalTree", () => {

packages/core/tests/unit/snapshot-tree-format-utils.test.ts

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,61 @@ describe("formatTreeLine", () => {
2828
"[frame-1] section: Container\n [ax-2] button: Submit",
2929
);
3030
});
31+
32+
it("renders a select with child options and only one selected option", () => {
33+
const outline = formatTreeLine({
34+
role: "select",
35+
name: "Select field",
36+
nodeId: "ax-4",
37+
children: [
38+
{ role: "option", name: "Option A", nodeId: "ax-5" },
39+
{
40+
role: "option",
41+
name: "Option B",
42+
selected: true,
43+
nodeId: "ax-6",
44+
},
45+
{ role: "option", name: "Option C", nodeId: "ax-7" },
46+
],
47+
});
48+
49+
expect(outline).toBe(
50+
"[ax-4] select: Select field\n [ax-5] option: Option A\n [ax-6] option: Option B [selected]\n [ax-7] option: Option C",
51+
);
52+
expect(outline.match(/\[selected]/g)?.length ?? 0).toBe(1);
53+
});
54+
55+
it("renders a radio group with children and only one checked radio", () => {
56+
const outline = formatTreeLine({
57+
role: "group",
58+
name: "Select field",
59+
nodeId: "ax-8",
60+
children: [
61+
{ role: "radio", name: "Option A", nodeId: "ax-9" },
62+
{ role: "radio", name: "Option B", checked: true, nodeId: "ax-10" },
63+
{ role: "radio", name: "Option C", nodeId: "ax-11" },
64+
],
65+
});
66+
67+
expect(outline).toBe(
68+
"[ax-8] group: Select field\n [ax-9] radio: Option A\n [ax-10] radio: Option B [checked]\n [ax-11] radio: Option C",
69+
);
70+
expect(outline.match(/\[checked]/g)?.length ?? 0).toBe(1);
71+
});
72+
73+
it("renders both flags when a node carries both states", () => {
74+
const outline = formatTreeLine({
75+
role: "menuitemcheckbox",
76+
name: "Hybrid state",
77+
selected: true,
78+
checked: true,
79+
nodeId: "ax-12",
80+
});
81+
82+
expect(outline).toBe(
83+
"[ax-12] menuitemcheckbox: Hybrid state [selected] [checked]",
84+
);
85+
});
3186
});
3287

3388
describe("injectSubtrees", () => {

0 commit comments

Comments
 (0)