refactor: share android hierarchy metadata (#497)

* refactor: share android hierarchy metadata

* refactor: drop unused android bounds helper

Author: thymikee

src/platforms/android/__tests__/index.test.ts

@@ -28,7 +28,7 @@ import {
 import { withAndroidAdbProvider } from '../adb-executor.ts';
 import type { DeviceInfo } from '../../../utils/device.ts';
 import { AppError } from '../../../utils/errors.ts';
-import { findBounds, parseUiHierarchy } from '../ui-hierarchy.ts';
+import { androidUiNodes, parseUiHierarchy } from '../ui-hierarchy.ts';
 
 async function withMockedAdb(
   tempPrefix: string,
@@ -116,17 +116,26 @@ test('parseUiHierarchy decodes XML entities in Android node attributes', () => {
   assert.equal(result.nodes[0].label, 'Line 1\nLine 2\t&<>"\'');
 });
 
-test('findBounds supports single and double quoted attributes', () => {
-  const xml = [
-    '<hierarchy>',
-    '<node text="Nothing" content-desc="Irrelevant" bounds="[0,0][10,10]"/>',
-    "<node text='Target from single quote' content-desc='Alt single' bounds='[100,200][300,500]'/>",
-    '<node text="Target from double quote" content-desc="Alt double" bounds="[50,50][150,250]"/>',
-    '</hierarchy>',
-  ].join('');
-
-  assert.deepEqual(findBounds(xml, 'single quote'), { x: 200, y: 350 });
-  assert.deepEqual(findBounds(xml, 'alt double'), { x: 100, y: 150 });
+test('androidUiNodes exposes decoded Android hierarchy metadata', () => {
+  const xml =
+    '<hierarchy><node package="com.example.app" class="android.widget.EditText" text="Fish &amp; Chips" content-desc="Search&#10;field" resource-id="com.example.app:id/search" bounds="[10,20][110,70]" clickable="false" enabled="true" focusable="true" focused="true" password="true"/></hierarchy>';
+
+  assert.deepEqual(Array.from(androidUiNodes(xml)), [
+    {
+      text: 'Fish & Chips',
+      desc: 'Search\nfield',
+      resourceId: 'com.example.app:id/search',
+      packageName: 'com.example.app',
+      className: 'android.widget.EditText',
+      bounds: '[10,20][110,70]',
+      rect: { x: 10, y: 20, width: 100, height: 50 },
+      clickable: false,
+      enabled: true,
+      focusable: true,
+      focused: true,
+      password: true,
+    },
+  ]);
 });
 
 test('parseUiHierarchy ignores attribute-name prefix spoofing', () => {
@@ -138,16 +147,6 @@ test('parseUiHierarchy ignores attribute-name prefix spoofing', () => {
   assert.equal(result.nodes[0].value, 'Actual');
 });
 
-test('findBounds ignores bounds-like fragments inside other attribute values', () => {
-  const xml = [
-    '<hierarchy>',
-    "<node text='Target' content-desc=\"metadata bounds='[900,900][1000,1000]'\" bounds='[100,200][300,500]'/>",
-    '</hierarchy>',
-  ].join('');
-
-  assert.deepEqual(findBounds(xml, 'target'), { x: 200, y: 350 });
-});
-
 test('scrollAndroid supports explicit pixel travel distance', async () => {
   await withMockedAdb(
     'agent-device-android-scroll-pixels-',

src/platforms/android/fill-verification.ts

@@ -9,7 +9,7 @@ import {
 } from '../fill-diagnostics.ts';
 import { sleep } from './adb.ts';
 import { dumpUiHierarchy } from './snapshot.ts';
-import { parseBounds, readNodeAttributes } from './ui-hierarchy.ts';
+import { androidUiNodes, type AndroidUiNodeMetadata } from './ui-hierarchy.ts';
 
 export type AndroidFillVerificationNode = FillDiagnosticNode & {
   className: string | null;
@@ -145,16 +145,14 @@ function inspectAndroidTextAtPointInHierarchy(
   x: number,
   y: number,
 ): AndroidTextAtPointInspection {
-  const nodeRegex = /<node\b[^>]*>/g;
-  let match: RegExpExecArray | null;
   const scan: AndroidTextAtPointScan = {
     focusedEdit: null,
     editAtPoint: null,
     anyAtPoint: null,
   };
 
-  while ((match = nodeRegex.exec(xml)) !== null) {
-    const candidate = androidFillCandidateFromNode(match[0]);
+  for (const node of androidUiNodes(xml)) {
+    const candidate = androidFillCandidateFromNode(node);
     if (candidate) updateAndroidTextAtPointScan(scan, candidate, x, y);
   }
 
@@ -228,23 +226,23 @@ function normalizeFillVerificationText(value: string | null): string {
   return (value ?? '').replace(/\s+/g, ' ').trim();
 }
 
-function androidFillCandidateFromNode(node: string): AndroidFillVerificationCandidate | null {
-  const attrs = readNodeAttributes(node);
-  const rect = parseBounds(attrs.bounds);
-  if (!rect) return null;
-  const text = attrs.text ?? '';
-  const area = Math.max(1, rect.width * rect.height);
+function androidFillCandidateFromNode(
+  node: AndroidUiNodeMetadata,
+): AndroidFillVerificationCandidate | null {
+  if (!node.rect) return null;
+  const text = node.text ?? '';
+  const area = Math.max(1, node.rect.width * node.rect.height);
   return {
     text: text || null,
-    className: attrs.className,
-    resourceId: attrs.resourceId,
-    packageName: attrs.packageName,
-    rect,
-    focused: attrs.focused ?? false,
-    password: attrs.password === true,
-    inputMethodOwned: isInputMethodOwnedAndroidNode(attrs.packageName, attrs.resourceId),
+    className: node.className,
+    resourceId: node.resourceId,
+    packageName: node.packageName,
+    rect: node.rect,
+    focused: node.focused ?? false,
+    password: node.password === true,
+    inputMethodOwned: isInputMethodOwnedAndroidNode(node.packageName, node.resourceId),
     area,
-    editText: isEditTextClass(attrs.className ?? ''),
+    editText: isEditTextClass(node.className ?? ''),
   };
 }
 

src/platforms/android/ui-hierarchy.ts

@@ -6,28 +6,37 @@ export type AndroidSnapshotAnalysis = {
   maxDepth: number;
 };
 
-export function findBounds(xml: string, query: string): { x: number; y: number } | null {
-  const q = query.toLowerCase();
-  const nodeRegex = /<node[^>]+>/g;
+export type AndroidUiNodeMetadata = {
+  text: string | null;
+  desc: string | null;
+  resourceId: string | null;
+  packageName: string | null;
+  className: string | null;
+  bounds: string | null;
+  rect?: Rect;
+  clickable?: boolean;
+  enabled?: boolean;
+  focusable?: boolean;
+  focused?: boolean;
+  password?: boolean;
+};
+
+export function* androidUiNodes(xml: string): IterableIterator<AndroidUiNodeMetadata> {
+  const nodeRegex = /<node\b[^>]*>/g;
   let match = nodeRegex.exec(xml);
   while (match) {
-    const node = match[0];
-    const attrs = parseXmlNodeAttributes(node);
-    const textVal = (readXmlAttr(attrs, 'text') ?? '').toLowerCase();
-    const descVal = (readXmlAttr(attrs, 'content-desc') ?? '').toLowerCase();
-    if (textVal.includes(q) || descVal.includes(q)) {
-      const rect = parseBounds(readXmlAttr(attrs, 'bounds'));
-      if (rect) {
-        return {
-          x: Math.floor(rect.x + rect.width / 2),
-          y: Math.floor(rect.y + rect.height / 2),
-        };
-      }
-      return { x: 0, y: 0 };
-    }
+    yield readAndroidUiNodeMetadata(match[0]);
     match = nodeRegex.exec(xml);
   }
-  return null;
+}
+
+function readAndroidUiNodeMetadata(node: string): AndroidUiNodeMetadata {
+  const attrs = readNodeAttributes(node);
+  const rect = parseBounds(attrs.bounds);
+  return {
+    ...attrs,
+    ...(rect ? { rect } : {}),
+  };
 }
 
 export function parseUiHierarchy(
@@ -172,19 +181,7 @@ function hasInteractiveDescendant(state: AndroidSnapshotBuildState, node: Androi
   return false;
 }
 
-export function readNodeAttributes(node: string): {
-  text: string | null;
-  desc: string | null;
-  resourceId: string | null;
-  packageName: string | null;
-  className: string | null;
-  bounds: string | null;
-  clickable?: boolean;
-  enabled?: boolean;
-  focusable?: boolean;
-  focused?: boolean;
-  password?: boolean;
-} {
+function readNodeAttributes(node: string): Omit<AndroidUiNodeMetadata, 'rect'> {
   const attrs = parseXmlNodeAttributes(node);
   const getAttr = (name: string): string | null => readXmlAttr(attrs, name);
   const boolAttr = (name: string): boolean | undefined => {
@@ -332,7 +329,7 @@ function readXmlAttr(attrs: Map<string, string>, name: string): string | null {
   return attrs.get(name) ?? null;
 }
 
-export function parseBounds(bounds: string | null): Rect | undefined {
+function parseBounds(bounds: string | null): Rect | undefined {
   if (!bounds) return undefined;
   const match = /\[(\d+),(\d+)\]\[(\d+),(\d+)\]/.exec(bounds);
   if (!match) return undefined;
@@ -387,15 +384,14 @@ export function parseUiHierarchyTree(xml: string): AndroidUiHierarchy {
       match = tokenRegex.exec(xml);
       continue;
     }
-    const attrs = readNodeAttributes(token);
-    const rect = parseBounds(attrs.bounds);
+    const attrs = readAndroidUiNodeMetadata(token);
     const parent = stack[stack.length - 1];
     const node: AndroidUiHierarchy = {
       type: attrs.className,
       label: attrs.text || attrs.desc,
       value: attrs.text,
       identifier: attrs.resourceId,
-      rect,
+      rect: attrs.rect,
       enabled: attrs.enabled,
       hittable: attrs.clickable ?? attrs.focusable,
       depth: parent.depth + 1,