fix: harden press @ref bounds recovery (#141)

Author: thymikee

src/daemon/handlers/__tests__/interaction.test.ts

@@ -155,6 +155,161 @@ test('press @ref resolves snapshot node and records press action', async () => {
   assert.ok(Array.isArray(result.selectorChain));
 });
 
+test('press @ref refreshes snapshot when stored ref bounds are invalid', async () => {
+  const sessionStore = makeSessionStore();
+  const sessionName = 'default';
+  const session = makeSession(sessionName);
+  session.device = {
+    platform: 'android',
+    id: 'emulator-5554',
+    name: 'Pixel 8 Pro',
+    kind: 'emulator',
+    booted: true,
+  };
+  session.snapshot = {
+    nodes: attachRefs([
+      {
+        index: 0,
+        type: 'android.widget.TextView',
+        label: 'My App',
+        // Simulate malformed persisted bounds from older/stale snapshot state.
+        rect: { x: 20, y: 40, width: Number.NaN, height: 40 },
+        enabled: true,
+        hittable: true,
+      },
+    ]),
+    createdAt: Date.now(),
+    backend: 'android',
+  };
+  sessionStore.set(sessionName, session);
+
+  const pressCalls: Array<{ command: string; positionals: string[] }> = [];
+  let snapshotCalls = 0;
+  const response = await handleInteractionCommands({
+    req: {
+      token: 't',
+      session: sessionName,
+      command: 'press',
+      positionals: ['@e1'],
+      flags: {},
+    },
+    sessionName,
+    sessionStore,
+    contextFromFlags,
+    dispatch: async (_device, command, positionals) => {
+      if (command === 'snapshot') {
+        snapshotCalls += 1;
+        return {
+          nodes: [
+            {
+              index: 0,
+              type: 'android.widget.TextView',
+              label: 'My App',
+              rect: { x: 20, y: 40, width: 100, height: 40 },
+              enabled: true,
+              hittable: true,
+            },
+          ],
+          backend: 'android',
+        };
+      }
+      pressCalls.push({ command, positionals });
+      return { pressed: true };
+    },
+  });
+
+  assert.ok(response);
+  assert.equal(response.ok, true);
+  assert.equal(snapshotCalls, 1);
+  assert.equal(pressCalls.length, 1);
+  assert.equal(pressCalls[0]?.command, 'press');
+  assert.deepEqual(pressCalls[0]?.positionals, ['70', '60']);
+  if (response.ok) {
+    assert.equal(response.data?.x, 70);
+    assert.equal(response.data?.y, 60);
+    assert.equal(response.data?.ref, 'e1');
+  }
+});
+
+test('press @ref fallback label is used after refresh when ref bounds remain invalid', async () => {
+  const sessionStore = makeSessionStore();
+  const sessionName = 'default';
+  const session = makeSession(sessionName);
+  session.device = {
+    platform: 'android',
+    id: 'emulator-5554',
+    name: 'Pixel 8 Pro',
+    kind: 'emulator',
+    booted: true,
+  };
+  session.snapshot = {
+    nodes: attachRefs([
+      {
+        index: 0,
+        type: 'android.widget.TextView',
+        label: 'My App',
+        rect: { x: 20, y: 40, width: Number.NaN, height: 40 },
+        enabled: true,
+        hittable: true,
+      },
+    ]),
+    createdAt: Date.now(),
+    backend: 'android',
+  };
+  sessionStore.set(sessionName, session);
+
+  const pressCalls: Array<{ command: string; positionals: string[] }> = [];
+  const response = await handleInteractionCommands({
+    req: {
+      token: 't',
+      session: sessionName,
+      command: 'press',
+      positionals: ['@e1', 'My App'],
+      flags: {},
+    },
+    sessionName,
+    sessionStore,
+    contextFromFlags,
+    dispatch: async (_device, command, positionals) => {
+      if (command === 'snapshot') {
+        return {
+          nodes: [
+            {
+              index: 0,
+              type: 'android.widget.TextView',
+              label: 'Different',
+              rect: { x: 20, y: 40, width: Number.NaN, height: 40 },
+              enabled: true,
+              hittable: true,
+            },
+            {
+              index: 1,
+              type: 'android.widget.TextView',
+              label: 'My App',
+              rect: { x: 100, y: 200, width: 80, height: 40 },
+              enabled: true,
+              hittable: true,
+            },
+          ],
+          backend: 'android',
+        };
+      }
+      pressCalls.push({ command, positionals });
+      return { pressed: true };
+    },
+  });
+
+  assert.ok(response);
+  assert.equal(response.ok, true);
+  assert.equal(pressCalls.length, 1);
+  assert.equal(pressCalls[0]?.command, 'press');
+  assert.deepEqual(pressCalls[0]?.positionals, ['140', '220']);
+  if (response.ok) {
+    assert.equal(response.data?.x, 140);
+    assert.equal(response.data?.y, 220);
+  }
+});
+
 test('press coordinates does not treat extra trailing args as selector', async () => {
   const sessionStore = makeSessionStore();
   const sessionName = 'default';

src/daemon/handlers/interaction.ts

@@ -6,6 +6,7 @@ import {
   findNodeByRef,
   normalizeRef,
   type RawSnapshotNode,
+  type Rect,
   type SnapshotNode,
 } from '../../utils/snapshot.ts';
 import type { DaemonCommandContext } from '../context.ts';
@@ -91,16 +92,40 @@ export async function handleInteractionCommands(params: {
         notFoundMessage: `Ref ${refInput} not found or has no bounds`,
       });
       if (!resolvedRefTarget.ok) return resolvedRefTarget.response;
-      const { ref, node, snapshotNodes } = resolvedRefTarget.target;
-      if (!node.rect) {
+      const { ref } = resolvedRefTarget.target;
+      let node = resolvedRefTarget.target.node;
+      let snapshotNodes = resolvedRefTarget.target.snapshotNodes;
+      let pressPoint = resolveRectCenter(node.rect);
+      if (!pressPoint) {
+        const refreshed = await captureSnapshotForSession(
+          session,
+          req.flags,
+          sessionStore,
+          contextFromFlags,
+          { interactiveOnly: true },
+          dispatch,
+        );
+        const refNode = findNodeByRef(refreshed.nodes, ref);
+        const fallbackNode = fallbackLabel.length > 0 ? findNodeByLabel(refreshed.nodes, fallbackLabel) : null;
+        const fallbackNodePoint = resolveRectCenter(fallbackNode?.rect);
+        const refNodePoint = resolveRectCenter(refNode?.rect);
+        const refreshedNode = refNodePoint ? refNode : fallbackNodePoint ? fallbackNode : refNode ?? fallbackNode;
+        const refreshedPoint = resolveRectCenter(refreshedNode?.rect);
+        if (refreshedNode && refreshedPoint) {
+          node = refreshedNode;
+          snapshotNodes = refreshed.nodes;
+          pressPoint = refreshedPoint;
+        }
+      }
+      if (!pressPoint) {
         return {
           ok: false,
-          error: { code: 'COMMAND_FAILED', message: `Ref ${refInput} not found or has no bounds` },
+          error: { code: 'COMMAND_FAILED', message: `Ref ${refInput} not found or has invalid bounds` },
         };
       }
       const refLabel = resolveRefLabel(node, snapshotNodes);
       const selectorChain = buildSelectorChainForNode(node, session.device.platform, { action: selectorAction });
-      const { x, y } = centerOfRect(node.rect);
+      const { x, y } = pressPoint;
       const data = await dispatch(session.device, 'press', [String(x), String(y)], req.flags?.out, {
         ...contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath),
       });
@@ -149,7 +174,17 @@ export async function handleInteractionCommands(params: {
         },
       };
     }
-    const { x, y } = centerOfRect(resolved.node.rect);
+    const pressPoint = resolveRectCenter(resolved.node.rect);
+    if (!pressPoint) {
+      return {
+        ok: false,
+        error: {
+          code: 'COMMAND_FAILED',
+          message: `Selector ${resolved.selector.raw} resolved to invalid bounds`,
+        },
+      };
+    }
+    const { x, y } = pressPoint;
     const data = await dispatch(session.device, 'press', [String(x), String(y)], req.flags?.out, {
       ...contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath),
     });
@@ -772,3 +807,24 @@ function resolveRefTarget(params: {
   }
   return { ok: true, target: { ref, node, snapshotNodes: session.snapshot.nodes } };
 }
+
+function resolveRectCenter(rect: Rect | undefined): { x: number; y: number } | null {
+  const normalized = normalizeRect(rect);
+  if (!normalized) return null;
+  const center = centerOfRect(normalized);
+  if (!Number.isFinite(center.x) || !Number.isFinite(center.y)) return null;
+  return center;
+}
+
+function normalizeRect(rect: Rect | undefined): Rect | null {
+  if (!rect) return null;
+  const x = Number(rect.x);
+  const y = Number(rect.y);
+  const width = Number(rect.width);
+  const height = Number(rect.height);
+  if (!Number.isFinite(x) || !Number.isFinite(y) || !Number.isFinite(width) || !Number.isFinite(height)) {
+    return null;
+  }
+  if (width < 0 || height < 0) return null;
+  return { x, y, width, height };
+}