Fix/include native selector state into snapshot

.changeset/swift-geckos-arrive.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": patch
+---
+
+include-native-selector-state-into-snapshot

packages/core/lib/v3/types/private/snapshot.ts

@@ -101,6 +101,7 @@ export type A11yNode = {
   name?: string;
   description?: string;
   value?: string | number | boolean;
+  state?: "selected" | "checked";
   nodeId: string;
   backendDOMNodeId?: number;
   parentId?: string;

packages/core/lib/v3/understudy/a11y/snapshot/a11yTree.ts

@@ -9,6 +9,14 @@ import {
   resolveObjectIdForCss,
   resolveObjectIdForXPath,
 } from "./focusSelectors.js";
+import {
+  A11Y_STATE_CHECKED,
+  A11Y_STATE_SELECTED,
+  AX_BOOLEAN_FALSE_NUMBER,
+  AX_BOOLEAN_FALSE_STRING,
+  AX_BOOLEAN_TRUE_NUMBER,
+  AX_BOOLEAN_TRUE_STRING,
+} from "./constants.js";
 import { formatTreeLine, normaliseSpaces } from "./treeFormatUtils.js";
 
 /**
@@ -141,11 +149,14 @@ export function decorateRoles(
       role = tag;
     }
 
+    const state = resolveA11yState(n);
+
     return {
       role,
       name: n.name?.value,
       description: n.description?.value,
       value: n.value?.value,
+      state,
       nodeId: n.nodeId,
       backendDOMNodeId: n.backendDOMNodeId,
       parentId: n.parentId,
@@ -235,6 +246,50 @@ export function extractUrlFromAXNode(
   return typeof value === "string" && value.trim() ? value.trim() : undefined;
 }
 
+
+function toBoolean(
+  value: Protocol.Accessibility.AXValue | undefined,
+): boolean | undefined {
+  const raw = value?.value;
+  if (typeof raw === "boolean") return raw;
+  if (typeof raw === "number") {
+    if (raw === AX_BOOLEAN_TRUE_NUMBER) return true;
+    if (raw === AX_BOOLEAN_FALSE_NUMBER) return false;
+  }
+  if (typeof raw === "string") {
+    const normalized = raw.toLowerCase();
+    if (normalized === AX_BOOLEAN_TRUE_STRING) return true;
+    if (normalized === AX_BOOLEAN_FALSE_STRING) return false;
+  }
+  return undefined;
+}
+
+function extractBooleanState(
+  node: Protocol.Accessibility.AXNode,
+  stateName: string,
+): boolean | undefined {
+  const directValue = (
+    node as unknown as Record<string, Protocol.Accessibility.AXValue | undefined>
+  )[stateName];
+  const direct = toBoolean(directValue);
+  if (direct !== undefined) return direct;
+
+  const fromProperties = node.properties?.find((prop) => prop.name === stateName);
+  return toBoolean(fromProperties?.value);
+}
+
+function resolveA11yState(
+  node: Protocol.Accessibility.AXNode,
+): A11yNode["state"] {
+  const checked = extractBooleanState(node, A11Y_STATE_CHECKED);
+  if (checked === true) return A11Y_STATE_CHECKED;
+
+  const selected = extractBooleanState(node, A11Y_STATE_SELECTED);
+  if (selected === true) return A11Y_STATE_SELECTED;
+
+  return undefined;
+}
+
 export function removeRedundantStaticTextChildren(
   parent: A11yNode,
   children: A11yNode[],

packages/core/lib/v3/understudy/a11y/snapshot/constants.ts

@@ -0,0 +1,14 @@
+export const A11Y_STATE_SELECTED = "selected";
+export const A11Y_STATE_CHECKED = "checked";
+
+export const A11Y_SELECTED_VISIBLE_ROLES = new Set<string>([
+  "option",
+  "menuitemradio",
+  "tab",
+  "treeitem",
+]);
+
+export const AX_BOOLEAN_TRUE_NUMBER = 1;
+export const AX_BOOLEAN_FALSE_NUMBER = 0;
+export const AX_BOOLEAN_TRUE_STRING = "true";
+export const AX_BOOLEAN_FALSE_STRING = "false";

packages/core/lib/v3/understudy/a11y/snapshot/treeFormatUtils.ts

@@ -1,4 +1,9 @@
 import type { A11yNode } from "../../../types/private/snapshot.js";
+import {
+  A11Y_SELECTED_VISIBLE_ROLES,
+  A11Y_STATE_CHECKED,
+  A11Y_STATE_SELECTED,
+} from "./constants.js";
 
 /**
  * Render a formatted outline (with encoded ids) for the accessibility tree.
@@ -8,12 +13,30 @@ import type { A11yNode } from "../../../types/private/snapshot.js";
 export function formatTreeLine(node: A11yNode, level = 0): string {
   const indent = "  ".repeat(level);
   const labelId = node.encodedId ?? node.nodeId;
-  const label = `[${labelId}] ${node.role}${node.name ? `: ${cleanText(node.name)}` : ""}`;
+  const stateFlags = formatStateFlags(node);
+  const label = `[${labelId}] ${node.role}${node.name ? `: ${cleanText(node.name)}` : ""}${stateFlags}`;
   const kids =
     node.children?.map((c) => formatTreeLine(c, level + 1)).join("\n") ?? "";
   return kids ? `${indent}${label}\n${kids}` : `${indent}${label}`;
 }
 
+function formatStateFlags(node: A11yNode): string {
+  const flags: string[] = [];
+
+  if (
+    node.state === A11Y_STATE_CHECKED ||
+    (node.state === A11Y_STATE_SELECTED && shouldRenderSelected(node.role))
+  ) {
+    flags.push(node.state);
+  }
+  return flags.length ? ` [${flags.join(", ")}]` : "";
+}
+
+function shouldRenderSelected(role: string): boolean {
+  const normalizedRole = role.toLowerCase();
+  return A11Y_SELECTED_VISIBLE_ROLES.has(normalizedRole);
+}
+
 /**
  * Inject each child frame outline under the parent's iframe node line.
  * Keys in `idToTree` are the parent's iframe encoded ids.

packages/core/tests/unit/snapshot-a11y-tree-utils.test.ts

@@ -17,6 +17,11 @@ const axString = (value: string): Protocol.Accessibility.AXValue => ({
   value,
 });
 
+const axBool = (value: boolean): Protocol.Accessibility.AXValue => ({
+  type: "boolean",
+  value,
+});
+
 const defaultOpts: A11yOptions = {
   focusSelector: undefined,
   experimental: false,
@@ -114,6 +119,39 @@ describe("decorateRoles", () => {
     const decorated = decorateRoles(nodes, opts);
     expect(decorated[0]?.encodedId).toBeUndefined();
   });
+
+  it("maps selected/checked AX states into a single state field", () => {
+    const nodes = [
+      makeAxNode({
+        backendDOMNodeId: 12,
+        role: axString("option"),
+        name: axString("Option B"),
+        properties: [
+          { name: "selected", value: axBool(true) },
+        ],
+      }),
+      makeAxNode({
+        backendDOMNodeId: 13,
+        role: axString("radio"),
+        name: axString("Option C"),
+        properties: [{ name: "checked", value: axBool(true) }],
+      }),
+      makeAxNode({
+        backendDOMNodeId: 14,
+        role: axString("radio"),
+        name: axString("Option D"),
+        properties: [
+          { name: "selected", value: axBool(true) },
+          { name: "checked", value: axBool(true) },
+        ],
+      }),
+    ];
+
+    const decorated = decorateRoles(nodes, defaultOpts);
+    expect(decorated[0]?.state).toBe("selected");
+    expect(decorated[1]?.state).toBe("checked");
+    expect(decorated[2]?.state).toBe("checked");
+  });
 });
 
 describe("buildHierarchicalTree", () => {

packages/core/tests/unit/snapshot-tree-format-utils.test.ts

@@ -28,6 +28,65 @@ describe("formatTreeLine", () => {
       "[frame-1] section: Container\n  [ax-2] button: Submit",
     );
   });
+
+  it("renders a select with child options and only one selected option", () => {
+    const outline = formatTreeLine({
+      role: "select",
+      name: "Select field",
+      nodeId: "ax-4",
+      children: [
+        { role: "option", name: "Option A", nodeId: "ax-5" },
+        {
+          role: "option",
+          name: "Option B",
+          state: "selected",
+          nodeId: "ax-6",
+        },
+        { role: "option", name: "Option C", nodeId: "ax-7" },
+      ],
+    });
+
+    expect(outline).toBe(
+      "[ax-4] select: Select field\n  [ax-5] option: Option A\n  [ax-6] option: Option B [selected]\n  [ax-7] option: Option C",
+    );
+    expect(outline.match(/\[selected]/g)?.length ?? 0).toBe(1);
+  });
+
+  it("renders a radio group with children and only one checked radio", () => {
+    const outline = formatTreeLine({
+      role: "group",
+      name: "Select field",
+      nodeId: "ax-8",
+      children: [
+        { role: "radio", name: "Option A", nodeId: "ax-9" },
+        { role: "radio", name: "Option B", state: "checked", nodeId: "ax-10" },
+        { role: "radio", name: "Option C", nodeId: "ax-11" },
+      ],
+    });
+
+    expect(outline).toBe(
+      "[ax-8] group: Select field\n  [ax-9] radio: Option A\n  [ax-10] radio: Option B [checked]\n  [ax-11] radio: Option C",
+    );
+    expect(outline.match(/\[checked]/g)?.length ?? 0).toBe(1);
+  });
+
+  it("does not render [selected] on combobox/select nodes", () => {
+    const combo = formatTreeLine({
+      role: "combobox",
+      name: "Select field",
+      state: "selected",
+      nodeId: "ax-12",
+    });
+    const select = formatTreeLine({
+      role: "select",
+      name: "Another select",
+      state: "selected",
+      nodeId: "ax-13",
+    });
+
+    expect(combo).toBe("[ax-12] combobox: Select field");
+    expect(select).toBe("[ax-13] select: Another select");
+  });
 });
 
 describe("injectSubtrees", () => {