[fix]: make file upload elements more explicit in page snapshot (#1975)

# why
- file upload elements sometimes have a 'button' AX role 
- this causes issues when users prompt `.observe()` to find "file upload
elements", since the model sees them as a 'button'
- addresses #972 
# what changed
- this PR preserves the semantics of file upload elements by using the
actual DOM tag name and 'type'
- eg, instead of `[0-12] button: Choose File`, we now use `[0-12] input,
file: Choose File`
# test plan
- added a unit test for the replacement logic
- added an `observe()` eval which checks that hidden file upload
elements are correctly found by the LLM


<!-- This is an auto-generated description by cubic. -->
---
## Summary by cubic
Make file upload inputs explicit in the page snapshot so `.observe()`
can reliably find them. Addresses Linear STG-934 by preventing file
inputs from being misclassified as buttons.

- **Bug Fixes**
- Enrich DOM tag names to include input type (e.g., `input, file`) and
use them in DOM maps.
- Force file inputs to appear as `input, file` in the a11y outline
instead of AX `button`.
- Add unit test and fix/register the `observe_file_uploads` eval in
`evals.config.json`.
  - Publish patch for `@browserbasehq/stagehand`.

<sup>Written for commit 2fad86f.
Summary will update on new commits. <a
href="https://cubic.dev/pr/browserbase/stagehand/pull/1975">Review in
cubic</a></sup>

<!-- End of auto-generated description by cubic. -->

Author: seanmcguire12

.changeset/violet-moons-attend.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": patch
+---
+
+make file upload elements more explicit in page snapshot

packages/core/lib/v3/understudy/a11y/snapshot/a11yTree.ts

@@ -130,6 +130,12 @@ export function decorateRoles(
         : `scrollable${role ? `, ${role}` : ""}`;
     }
 
+    // File inputs typically get role "button" from Chrome's AX tree;
+    // override so they appear as "input, file" in the outline.
+    if (tag === "input, file") {
+      role = tag;
+    }
+
     return {
       role,
       name: n.name?.value,

packages/core/lib/v3/understudy/a11y/snapshot/domTree.ts

@@ -219,7 +219,7 @@ export async function domMapsForSession(
 
     if (node.backendNodeId) {
       const encId = encode(frameId, node.backendNodeId);
-      tagNameMap[encId] = String(node.nodeName).toLowerCase();
+      tagNameMap[encId] = enrichedTagName(node);
       xpathMap[encId] = xpath || "/";
       const isScrollable = node?.isScrollable === true;
       if (isScrollable) scrollableMap[encId] = true;
@@ -275,7 +275,7 @@ export async function buildSessionDomIndex(
     const { node, xp, docRootBe } = stack.pop()!;
     if (node.backendNodeId) {
       absByBe.set(node.backendNodeId, xp || "/");
-      tagByBe.set(node.backendNodeId, String(node.nodeName).toLowerCase());
+      tagByBe.set(node.backendNodeId, enrichedTagName(node));
       if (node?.isScrollable === true) scrollByBe.set(node.backendNodeId, true);
       docRootOf.set(node.backendNodeId, docRootBe);
     }
@@ -328,6 +328,31 @@ export function relativizeXPath(baseAbs: string, nodeAbs: string): string {
   return abs;
 }
 
+/**
+ * Extract an attribute value from a CDP DOM node's flat attributes array.
+ * Attributes are stored as [name1, value1, name2, value2, ...].
+ */
+function getAttr(
+  attrs: string[] | undefined,
+  name: string,
+): string | undefined {
+  if (!attrs) return undefined;
+  for (let i = 0; i < attrs.length; i += 2) {
+    if (attrs[i] === name) return attrs[i + 1];
+  }
+  return undefined;
+}
+
+/** Build an enriched tag name that includes the type attribute for inputs. */
+function enrichedTagName(node: Protocol.DOM.Node): string {
+  const tag = String(node.nodeName).toLowerCase();
+  if (tag === "input") {
+    const type = getAttr(node.attributes, "type");
+    if (type) return `input, ${type}`;
+  }
+  return tag;
+}
+
 /** Find a node by backendNodeId inside a DOM.getDocument tree. */
 export function findNodeByBackendId(
   root: Protocol.DOM.Node,

packages/core/tests/unit/snapshot-a11y-tree-utils.test.ts

@@ -81,6 +81,28 @@ describe("decorateRoles", () => {
     ]);
   });
 
+  it("overrides role to 'input, file' for file inputs", () => {
+    const opts: A11yOptions = {
+      ...defaultOpts,
+      tagNameMap: { "enc-10": "input, file" },
+      scrollableMap: {},
+    };
+    const nodes = [
+      makeAxNode({
+        backendDOMNodeId: 10,
+        role: axString("button"),
+        name: axString("Choose File"),
+      }),
+    ];
+
+    const decorated = decorateRoles(nodes, opts);
+    expect(decorated[0]).toMatchObject({
+      encodedId: "enc-10",
+      role: "input, file",
+      name: "Choose File",
+    });
+  });
+
   it("falls back when encoding fails", () => {
     const opts: A11yOptions = {
       ...defaultOpts,

packages/evals/evals.config.json

@@ -256,6 +256,10 @@
       "name": "observe_iframes2",
       "categories": ["regression", "observe"]
     },
+    {
+      "name": "observe_file_uploads",
+      "categories": ["observe"]
+    },
     {
       "name": "extract_hamilton_weather",
       "categories": ["targeted_extract", "regression"]

packages/evals/tasks/observe_file_uploads.ts

@@ -0,0 +1,58 @@
+import { EvalFunction } from "../types/evals.js";
+
+export const observe_file_uploads: EvalFunction = async ({
+  debugUrl,
+  sessionUrl,
+  v3,
+  logger,
+}) => {
+  try {
+    const page = v3.context.pages()[0];
+    await page.goto(
+      "https://browserbase.github.io/stagehand-eval-sites/sites/file-uploads-3/",
+    );
+
+    const observations = await v3.observe("find the file upload element");
+
+    if (observations.length === 0) {
+      return {
+        _success: false,
+        message: "observe returned no results",
+        observations,
+        debugUrl,
+        sessionUrl,
+        logs: logger.getLogs(),
+      };
+    }
+
+    const expectedLocator = `xpath=/html/body/input`;
+
+    const expectedBackendNodeId = await page
+      .locator(expectedLocator)
+      .backendNodeId();
+
+    const actualBackendNodeId = await page
+      .locator(observations[0].selector)
+      .backendNodeId();
+    const foundMatch = expectedBackendNodeId === actualBackendNodeId;
+
+    return {
+      _success: foundMatch,
+      observations,
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } catch (error) {
+    return {
+      _success: false,
+      error: error,
+      message: "returned selector does not resolve to same node as expected",
+      debugUrl,
+      sessionUrl,
+      logs: logger.getLogs(),
+    };
+  } finally {
+    await v3.close();
+  }
+};