fix: simplify iOS interactive snapshots (#578)

Author: thymikee

src/commands/react-native/__tests__/overlay.test.ts

@@ -26,6 +26,46 @@ describe('React Native overlay helpers', () => {
     });
   });
 
+  test('targets visible close affordance when collapsed banner keeps outer bounds', () => {
+    const nodes = [
+      node({
+        ref: 'e3',
+        label: '!, Open debugger to view warnings.',
+        rect: { x: 0, y: 0, width: 402, height: 874 },
+        hittable: false,
+      }),
+      node({
+        ref: 'e125',
+        label: '!, Open debugger to view warnings.',
+        rect: { x: 10, y: 786.666, width: 382, height: 67.333 },
+        hittable: false,
+      }),
+    ];
+
+    const target = resolveReactNativeOverlayDismissTarget(nodes);
+
+    expect(detectReactNativeOverlay(nodes).detected).toBe(true);
+    expect(target).toMatchObject({
+      action: 'close-collapsed-banner',
+      ref: 'e125',
+      point: { x: 369, y: 813 },
+    });
+  });
+
+  test('detects full-screen open-debugger wrappers but does not use them as targets', () => {
+    const nodes = [
+      node({
+        ref: 'e3',
+        label: '!, Open debugger to view warnings.',
+        rect: { x: 0, y: 0, width: 402, height: 874 },
+        hittable: false,
+      }),
+    ];
+
+    expect(detectReactNativeOverlay(nodes).detected).toBe(true);
+    expect(resolveReactNativeOverlayDismissTarget(nodes)).toBeNull();
+  });
+
   test('prefers Minimize for RedBox overlays', () => {
     const nodes = [
       node({ ref: 'e1', label: 'Runtime Error', rect: { x: 0, y: 0, width: 390, height: 100 } }),

src/commands/react-native/overlay.ts

@@ -1,4 +1,10 @@
 import { centerOfRect, type Point, type SnapshotNode } from '../../utils/snapshot.ts';
+import {
+  hasKnownReactNativeOverlayText,
+  isReactNativeCollapsedWarningLabel,
+  isReactNativeOpenDebuggerWarningLabel,
+  isReactNativeStackFrame,
+} from '../../utils/react-native-overlay-signals.ts';
 
 export type ReactNativeOverlayState = {
   detected: boolean;
@@ -42,9 +48,13 @@ export function detectReactNativeOverlay(nodes: SnapshotNode[]): ReactNativeOver
   const minimizeNodes = collectOverlayNodes(nodes, isMinimizeLabel);
   const collapsedNodes = collectOverlayNodes(
     nodes,
-    isCollapsedReactNativeWarningLabel,
+    isReactNativeCollapsedWarningLabel,
     isLikelyCollapsedWarningControl,
   );
+  const openDebuggerWarningNodes = collectOverlayNodes(
+    nodes,
+    isReactNativeOpenDebuggerWarningLabel,
+  );
   const dismissRefs = refsOf(dismissNodes);
   const minimizeRefs = refsOf(minimizeNodes);
   const collapsedRefs = refsOf(collapsedNodes);
@@ -56,6 +66,7 @@ export function detectReactNativeOverlay(nodes: SnapshotNode[]): ReactNativeOver
     (hasReactNativeStackFrame && hasOverlayControl);
   const detected =
     collapsedRefs.length > 0 ||
+    openDebuggerWarningNodes.length > 0 ||
     (hasOverlayControl && (hasKnownReactNativeOverlayText(text) || hasReactNativeStackFrame));
   return {
     detected,
@@ -100,19 +111,6 @@ export function resolveReactNativeOverlayDismissTarget(
   };
 }
 
-function hasKnownReactNativeOverlayText(text: string): boolean {
-  return /\b(logbox|redbox|reload js|copy stack|component stack|call stack|runtime error|open debugger to view warnings)\b/.test(
-    text,
-  );
-}
-
-function isReactNativeStackFrame(text: string): boolean {
-  return (
-    /\b[\w.$<>/-]+\.(?:tsx?|jsx?):\d+(?::\d+)?\b/.test(text) ||
-    /\b[\w.$<>/-]+\.(?:tsx?|jsx?)\s+\(\d+:\d+\)/.test(text)
-  );
-}
-
 function isDismissControlLabel(label: string): boolean {
   return label === 'dismiss' || label === 'close' || isCloseIconLabel(label);
 }
@@ -125,20 +123,6 @@ function isMinimizeLabel(label: string): boolean {
   return /^minimi[sz]e$/.test(label);
 }
 
-function isCollapsedReactNativeWarningLabel(label: string): boolean {
-  return (
-    label.includes('open debugger to view warnings') ||
-    /^!,\s+/.test(label) ||
-    /^(warn|warning|error):\s+/.test(label) ||
-    /\b(?:possible\s+)?unhandled (?:promise )?rejection\b/.test(label) ||
-    label.includes('getsnapshot should be cached to avoid an infinite loop') ||
-    label.includes('unique "key" prop') ||
-    label.includes("unique 'key' prop") ||
-    label.includes('virtualizedlists should never be nested') ||
-    label.includes('failed prop type')
-  );
-}
-
 function isLikelyCollapsedWarningControl(node: SnapshotNode): boolean {
   return !node.rect || node.rect.height <= 180;
 }
@@ -155,7 +139,7 @@ function collectOverlayNodes(
     const labels = [node.label, node.value, node.identifier]
       .map((value) => value?.trim().toLowerCase())
       .filter((value): value is string => Boolean(value));
-    if (!labels.some(matches)) continue;
+    if (!labels.some((label) => matches(label))) continue;
     matchedNodes.push(node);
   }
   return matchedNodes;
@@ -206,7 +190,8 @@ function chooseCollapsedWarningNode(nodes: SnapshotNode[]): SnapshotNode | null
 
 function collapsedBannerClosePoint(node: SnapshotNode): Point {
   if (!node.rect) throw new Error('Collapsed React Native warning node must have rect');
-  const inset = Math.min(36, Math.max(18, node.rect.height * 0.45));
+  const closeTargetHeight = Math.min(node.rect.height, 52);
+  const inset = Math.min(36, Math.max(18, closeTargetHeight * 0.45));
   return {
     x: Math.round(
       clamp(
@@ -215,7 +200,7 @@ function collapsedBannerClosePoint(node: SnapshotNode): Point {
         node.rect.x + node.rect.width - 1,
       ),
     ),
-    y: Math.round(node.rect.y + node.rect.height / 2),
+    y: Math.round(node.rect.y + closeTargetHeight / 2),
   };
 }
 

src/daemon/__tests__/request-router-screenshot.test.ts

@@ -15,7 +15,7 @@ import type { SessionState } from '../types.ts';
 import { LeaseRegistry } from '../lease-registry.ts';
 import { attachRefs } from '../../utils/snapshot.ts';
 import { PNG } from 'pngjs';
-import { ANDROID_EMULATOR } from '../../__tests__/test-utils/device-fixtures.ts';
+import { ANDROID_EMULATOR, IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts';
 import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts';
 import { makeSession as makeBaseSession } from '../../__tests__/test-utils/session-factories.ts';
 
@@ -25,6 +25,10 @@ function makeSession(name: string): SessionState {
   return makeBaseSession(name, { device: ANDROID_EMULATOR });
 }
 
+function makeIosSession(name: string): SessionState {
+  return makeBaseSession(name, { device: IOS_SIMULATOR });
+}
+
 function makeMacOsMenubarSession(name: string): SessionState {
   return {
     name,
@@ -415,6 +419,102 @@ test('screenshot --overlay-refs captures a fresh snapshot when the session has n
   expect(mockDispatch.mock.calls.map((call) => call[1])).toEqual(['screenshot', 'snapshot']);
 });
 
+test('screenshot --overlay-refs uses interactive iOS presentation for row-like other nodes', async () => {
+  const sessionStore = makeSessionStore('agent-device-router-screenshot-');
+  sessionStore.set('default', makeIosSession('default'));
+  const screenshotPath = path.join(os.tmpdir(), `agent-device-overlay-ios-${Date.now()}.png`);
+
+  mockDispatch.mockImplementation(async (_device, command) => {
+    if (command === 'screenshot') {
+      writeSolidPng(screenshotPath, 402, 874);
+      return { path: screenshotPath };
+    }
+    if (command === 'snapshot') {
+      return {
+        backend: 'xctest',
+        nodes: [
+          {
+            index: 0,
+            depth: 0,
+            type: 'Application',
+            label: 'New Expensify Dev',
+            rect: { x: 0, y: 0, width: 402, height: 874 },
+          },
+          {
+            index: 1,
+            depth: 1,
+            parentIndex: 0,
+            type: 'Other',
+            label: '!, Open debugger to view warnings.',
+            rect: { x: 0, y: 0, width: 402, height: 874 },
+          },
+          {
+            index: 2,
+            depth: 1,
+            parentIndex: 0,
+            type: 'ScrollView',
+            label: 'Recent chats',
+            rect: { x: 8, y: 212, width: 386, height: 600 },
+          },
+          {
+            index: 3,
+            depth: 2,
+            parentIndex: 2,
+            type: 'Other',
+            label: 'Recent chats',
+            rect: { x: 0, y: 220, width: 402, height: 16 },
+          },
+          {
+            index: 4,
+            depth: 2,
+            parentIndex: 2,
+            type: 'Other',
+            label: 'Receipt missing details, Receipt scanning failed. Enter details manually.',
+            rect: { x: 8, y: 367, width: 386, height: 64 },
+          },
+        ],
+      };
+    }
+    return {};
+  });
+
+  const handler = createRequestHandler({
+    logPath: path.join(os.tmpdir(), 'daemon.log'),
+    token: 'test-token',
+    sessionStore,
+    leaseRegistry: new LeaseRegistry(),
+    trackDownloadableArtifact: () => 'artifact-id',
+  });
+
+  const response = await handler({
+    token: 'test-token',
+    session: 'default',
+    command: 'screenshot',
+    positionals: [screenshotPath],
+    flags: { overlayRefs: true },
+    meta: { requestId: 'req-overlay-ios-rows' },
+  });
+
+  expect(response.ok).toBe(true);
+  if (response.ok) {
+    expect(response.data?.overlayRefs).toEqual([
+      {
+        ref: 'e5',
+        label: 'Receipt missing details, Receipt scanning failed. Enter details manually.',
+        rect: { x: 8, y: 367, width: 386, height: 64 },
+        overlayRect: { x: 8, y: 367, width: 386, height: 64 },
+        center: { x: 201, y: 399 },
+      },
+    ]);
+  }
+  expect(mockDispatch.mock.calls.map((call) => call[1])).toEqual(['screenshot', 'snapshot']);
+  expect(mockDispatch.mock.calls[1]?.[4]).toMatchObject({
+    snapshotInteractiveOnly: true,
+    snapshotCompact: true,
+  });
+  expect(sessionStore.get('default')?.snapshot?.nodes[4]?.type).toBe('Cell');
+});
+
 test('screenshot --overlay-refs uses a fresh snapshot instead of stale session snapshot', async () => {
   const sessionStore = makeSessionStore('agent-device-router-screenshot-');
   const session = makeSession('default');

src/daemon/__tests__/screenshot-overlay.test.ts

@@ -80,6 +80,45 @@ test('buildScreenshotOverlayRefs promotes labeled children to actionable ancesto
   ]);
 });
 
+test('buildScreenshotOverlayRefs includes non-hittable iOS cell rows', () => {
+  const snapshot = makeSnapshotState([
+    {
+      index: 0,
+      type: 'XCUIElementTypeApplication',
+      label: 'New Expensify Dev',
+      hittable: true,
+      rect: { x: 0, y: 0, width: 402, height: 874 },
+    },
+    {
+      index: 1,
+      parentIndex: 0,
+      type: 'XCUIElementTypeScrollView',
+      label: 'Recent chats',
+      rect: { x: 8, y: 212, width: 386, height: 600 },
+    },
+    {
+      index: 2,
+      parentIndex: 1,
+      type: 'Cell',
+      label: 'Receipt missing details, Receipt scanning failed. Enter details manually.',
+      hittable: false,
+      rect: { x: 8, y: 367, width: 386, height: 64 },
+    },
+  ]);
+
+  const overlayRefs = buildScreenshotOverlayRefs(snapshot, 804, 1748);
+
+  assert.deepEqual(overlayRefs, [
+    {
+      ref: 'e3',
+      label: 'Receipt missing details, Receipt scanning failed. Enter details manually.',
+      rect: { x: 8, y: 367, width: 386, height: 64 },
+      overlayRect: { x: 16, y: 734, width: 772, height: 128 },
+      center: { x: 402, y: 798 },
+    },
+  ]);
+});
+
 test('buildScreenshotOverlayRefs suppresses contained duplicates with the same label, keeping the smaller rect', () => {
   const snapshot = makeSnapshotState([
     {

src/daemon/handlers/__tests__/replay-heal.test.ts

@@ -183,6 +183,56 @@ test('replay heal rewrites longpress selector and preserves duration', async ()
   expect(healed?.positionals[1]).toBe('800');
 });
 
+test('replay heal uses canonical iOS snapshot presentation', async () => {
+  const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-heal-ios-row-'));
+  const sessionsDir = path.join(tempRoot, 'sessions');
+  const sessionStore = new SessionStore(sessionsDir);
+  const sessionName = 'heal-ios-row-session';
+  sessionStore.set(sessionName, makeSession(sessionName));
+  const rowRect = { x: 16, y: 293, width: 370, height: 52 };
+
+  mockDispatchCommand.mockResolvedValue({
+    nodes: [
+      { index: 0, depth: 0, type: 'Application', label: 'Settings' },
+      { index: 1, depth: 1, parentIndex: 0, type: 'CollectionView' },
+      { index: 2, depth: 2, parentIndex: 1, type: 'Cell', label: 'General', rect: rowRect },
+      { index: 3, depth: 3, parentIndex: 2, type: 'Other', label: 'General', rect: rowRect },
+      {
+        index: 4,
+        depth: 4,
+        parentIndex: 3,
+        type: 'Button',
+        label: 'General',
+        identifier: 'com.apple.settings.general',
+        rect: rowRect,
+      },
+      { index: 5, depth: 5, parentIndex: 4, type: 'StaticText', label: 'General', rect: rowRect },
+    ],
+    truncated: false,
+    backend: 'xctest',
+  });
+
+  const healed = await healReplayAction({
+    action: {
+      ts: Date.now(),
+      command: 'click',
+      positionals: ['label="General"'],
+      flags: {},
+      result: { selectorChain: ['label="General"'] },
+    },
+    sessionName,
+    logPath: '/tmp/replay.log',
+    sessionStore,
+  });
+
+  expect(healed?.positionals[0]).toContain('com.apple.settings.general');
+  expect(sessionStore.get(sessionName)?.snapshot?.nodes.map((node) => node.type)).toEqual([
+    'Application',
+    'CollectionView',
+    'Cell',
+  ]);
+});
+
 test('replay --update heals selector and rewrites replay file', async () => {
   const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-heal-'));
   const sessionsDir = path.join(tempRoot, 'sessions');