fix: harden react native overlay dismissal

Author: thymikee

src/__tests__/runtime-snapshot.test.ts

@@ -446,5 +446,5 @@ function assertReactNativeOverlayWarning(warnings: string[] | undefined) {
   assert.equal(warnings?.length, 1);
   assert.match(warnings[0] ?? '', /Hint: React Native warning\/error overlay detected/);
   assert.match(warnings[0] ?? '', /agent-device react-native dismiss-overlay/);
-  assert.match(warnings[0] ?? '', /agent-device snapshot -i -c/);
+  assert.match(warnings[0] ?? '', /verifies the overlay is gone/);
 }

src/commands/react-native/overlay.ts

@@ -1,10 +1,15 @@
-import { centerOfRect, type Point, type SnapshotNode } from '../../utils/snapshot.ts';
 import {
-  hasKnownReactNativeOverlayText,
-  isReactNativeCollapsedWarningLabel,
-  isReactNativeOpenDebuggerWarningLabel,
-  isReactNativeStackFrame,
-} from '../../utils/react-native-overlay-signals.ts';
+  centerOfRect,
+  type Point,
+  type RawSnapshotNode,
+  type Rect,
+  type SnapshotNode,
+} from '../../utils/snapshot.ts';
+
+type ReactNativeOverlayNode = Pick<
+  RawSnapshotNode,
+  'index' | 'type' | 'role' | 'subrole' | 'label' | 'value' | 'identifier' | 'rect' | 'hittable'
+>;
 
 export type ReactNativeOverlayState = {
   detected: boolean;
@@ -15,35 +20,84 @@ export type ReactNativeOverlayState = {
   dismissNodes: SnapshotNode[];
   minimizeNodes: SnapshotNode[];
   collapsedNodes: SnapshotNode[];
+  safeActions: ReactNativeOverlayDismissTarget[];
+  primaryAction: ReactNativeOverlayDismissTarget | null;
 };
 
 export type ReactNativeOverlayDismissTarget = {
   action: 'close' | 'dismiss' | 'minimize' | 'close-collapsed-banner';
   point: Point;
+  rect?: Rect;
   ref?: string;
   label?: string;
   warning?: string;
 };
 
+type ReactNativeOverlayFacts = {
+  dismissNodes: SnapshotNode[];
+  minimizeNodes: SnapshotNode[];
+  collapsedNodes: SnapshotNode[];
+  redBox: boolean;
+  detected: boolean;
+};
+
 export function formatReactNativeOverlayWarning(nodes: SnapshotNode[]): string | undefined {
-  const overlay = detectReactNativeOverlay(nodes);
+  const overlay = analyzeReactNativeOverlay(nodes);
   if (!overlay.detected) return undefined;
   return [
     'Hint: React Native warning/error overlay detected. It overlays part of the app and should be handled before interacting.',
     'Run: agent-device react-native dismiss-overlay',
-    'Then run: agent-device snapshot -i -c',
-    'Use refs from the new snapshot.',
+    'The command verifies the overlay is gone. Run agent-device snapshot -i -c afterward only when you need fresh refs for the next action.',
   ].join('\n');
 }
 
 export function detectReactNativeOverlay(nodes: SnapshotNode[]): ReactNativeOverlayState {
-  const text = nodes
-    .map((node) =>
-      [node.label, node.value, node.identifier, node.type, node.role].filter(Boolean).join(' '),
-    )
-    .join('\n')
-    .toLowerCase();
+  return analyzeReactNativeOverlay(nodes);
+}
 
+export function analyzeReactNativeOverlay(nodes: SnapshotNode[]): ReactNativeOverlayState {
+  const facts = collectReactNativeOverlayFacts(nodes);
+  const safeActions = facts.detected ? buildSafeDismissActions(facts) : [];
+  const dismissRefs = refsOf(facts.dismissNodes);
+  const minimizeRefs = refsOf(facts.minimizeNodes);
+  const collapsedRefs = refsOf(facts.collapsedNodes);
+
+  return {
+    detected: facts.detected,
+    redBox: facts.redBox,
+    dismissRefs,
+    minimizeRefs,
+    collapsedRefs,
+    dismissNodes: facts.dismissNodes,
+    minimizeNodes: facts.minimizeNodes,
+    collapsedNodes: facts.collapsedNodes,
+    safeActions,
+    primaryAction: safeActions[0] ?? null,
+  };
+}
+
+export function resolveReactNativeOverlayDismissTarget(
+  nodes: SnapshotNode[],
+): ReactNativeOverlayDismissTarget | null {
+  return analyzeReactNativeOverlay(nodes).primaryAction;
+}
+
+export function isReactNativeCollapsedWarningWrapperWithVisibleBanner(
+  node: ReactNativeOverlayNode,
+  descendants: ReactNativeOverlayNode[],
+): boolean {
+  const nodeLabel = node.label?.trim();
+  if (!isReactNativeCollapsedWarningLabel(nodeLabel) || !isFullScreenOverlayRect(node.rect)) {
+    return false;
+  }
+  return descendants.some(
+    (descendant) =>
+      descendant.label?.trim() === nodeLabel && isReactNativeCollapsedWarningBanner(descendant),
+  );
+}
+
+function collectReactNativeOverlayFacts(nodes: SnapshotNode[]): ReactNativeOverlayFacts {
+  const text = nodes.map(formatNodeSearchText).join('\n').toLowerCase();
   const dismissNodes = collectOverlayNodes(nodes, isDismissControlLabel);
   const minimizeNodes = collectOverlayNodes(nodes, isMinimizeLabel);
   const collapsedNodes = collectOverlayNodes(
@@ -55,82 +109,143 @@ export function detectReactNativeOverlay(nodes: SnapshotNode[]): ReactNativeOver
     nodes,
     isReactNativeOpenDebuggerWarningLabel,
   );
-  const dismissRefs = refsOf(dismissNodes);
-  const minimizeRefs = refsOf(minimizeNodes);
-  const collapsedRefs = refsOf(collapsedNodes);
   const hasReactNativeStackFrame = isReactNativeStackFrame(text);
   const hasControllessRedBoxText =
     /\buncaught\b/.test(text) && /unable to download asset/.test(text);
   const hasOverlayControl =
-    dismissRefs.length > 0 || minimizeRefs.length > 0 || /\b(reload js|copy stack)\b/.test(text);
+    dismissNodes.length > 0 || minimizeNodes.length > 0 || /\b(reload js|copy stack)\b/.test(text);
   const redBox =
     /\b(redbox|runtime error|reload js|copy stack|component stack|call stack)\b/.test(text) ||
     hasControllessRedBoxText ||
     (hasReactNativeStackFrame && hasOverlayControl);
   const detected =
-    collapsedRefs.length > 0 ||
+    collapsedNodes.length > 0 ||
     openDebuggerWarningNodes.length > 0 ||
     hasControllessRedBoxText ||
     (hasOverlayControl && (hasKnownReactNativeOverlayText(text) || hasReactNativeStackFrame));
   return {
-    detected,
-    redBox,
-    dismissRefs,
-    minimizeRefs,
-    collapsedRefs,
     dismissNodes,
     minimizeNodes,
     collapsedNodes,
+    redBox,
+    detected,
   };
 }
 
-export function resolveReactNativeOverlayDismissTarget(
-  nodes: SnapshotNode[],
-): ReactNativeOverlayDismissTarget | null {
-  const overlay = detectReactNativeOverlay(nodes);
-  if (!overlay.detected) return null;
-
-  if (overlay.redBox) {
-    const minimize = firstNodeWithRect(overlay.minimizeNodes);
-    if (minimize) return targetFromNode(minimize, 'minimize');
-    const dismiss = firstNodeWithRect(overlay.dismissNodes);
+function buildSafeDismissActions(
+  facts: ReactNativeOverlayFacts,
+): ReactNativeOverlayDismissTarget[] {
+  if (facts.redBox) {
+    const minimize = firstControlNodeWithRect(facts.minimizeNodes);
+    if (minimize) return [targetFromNode(minimize, 'minimize')];
+    const dismiss = firstControlNodeWithRect(facts.dismissNodes);
     return dismiss
-      ? {
-          ...targetFromNode(dismiss, actionFromDismissNode(dismiss)),
-          warning: 'RedBox Minimize control was not exposed; used Dismiss fallback',
-        }
-      : null;
+      ? [
+          {
+            ...targetFromNode(dismiss, actionFromDismissNode(dismiss)),
+            warning: 'RedBox Minimize control was not exposed; used Dismiss fallback',
+          },
+        ]
+      : [];
   }
 
-  const dismiss = firstNodeWithRect(overlay.dismissNodes);
-  if (dismiss) return targetFromNode(dismiss, actionFromDismissNode(dismiss));
+  const dismiss = firstControlNodeWithRect(facts.dismissNodes);
+  if (dismiss) return [targetFromNode(dismiss, actionFromDismissNode(dismiss))];
 
-  const collapsed = chooseCollapsedWarningNode(overlay.collapsedNodes);
-  if (!collapsed?.rect) return null;
-  return {
-    action: 'close-collapsed-banner',
-    point: collapsedBannerClosePoint(collapsed),
-    ref: collapsed.ref,
-    label: readNodeLabel(collapsed),
-  };
+  const collapsed = chooseCollapsedWarningNode(
+    facts.collapsedNodes.filter(isSafeCollapsedWarningCoordinateFallback),
+  );
+  if (!collapsed?.rect) return [];
+  return [
+    {
+      action: 'close-collapsed-banner',
+      point: collapsedBannerClosePoint(collapsed),
+      rect: collapsed.rect,
+      ref: collapsed.ref,
+      label: readNodeLabel(collapsed),
+    },
+  ];
+}
+
+function formatNodeSearchText(node: SnapshotNode): string {
+  return [node.label, node.value, node.identifier, node.type, node.role].filter(Boolean).join(' ');
+}
+
+function hasKnownReactNativeOverlayText(text: string): boolean {
+  return /\b(logbox|redbox|reload js|copy stack|component stack|call stack|runtime error|open debugger to view warnings)\b/.test(
+    text,
+  );
+}
+
+function isReactNativeStackFrame(text: string): boolean {
+  return (
+    /\b[\w.$<>/-]+\.(?:tsx?|jsx?):\d+(?::\d+)?\b/.test(text) ||
+    /\b[\w.$<>/-]+\.(?:tsx?|jsx?)\s+\(\d+:\d+\)/.test(text)
+  );
+}
+
+function isReactNativeCollapsedWarningLabel(rawLabel: string | undefined): boolean {
+  const label = rawLabel?.trim().toLowerCase();
+  if (!label) return false;
+  return (
+    label.includes('open debugger to view warnings') ||
+    /^!,\s+/.test(label) ||
+    /^(warn|warning|error):\s+/.test(label) ||
+    /\b(?:possible\s+)?unhandled (?:promise )?rejection\b/.test(label) ||
+    label.includes('getsnapshot should be cached to avoid an infinite loop') ||
+    label.includes('unique "key" prop') ||
+    label.includes("unique 'key' prop") ||
+    label.includes('virtualizedlists should never be nested') ||
+    label.includes('failed prop type')
+  );
+}
+
+function isReactNativeOpenDebuggerWarningLabel(label: string): boolean {
+  return label.includes('open debugger to view warnings') || /^!,\s+open debugger\b/.test(label);
 }
 
 function isDismissControlLabel(label: string): boolean {
-  return label === 'dismiss' || label === 'close' || isCloseIconLabel(label);
+  return isDismissLabel(label) || isCloseLabel(label) || isCloseIconLabel(label);
+}
+
+function isDismissLabel(label: string): boolean {
+  return /^dismiss(?:\b|\s|\()/i.test(label);
+}
+
+function isCloseLabel(label: string): boolean {
+  return /^close(?:\b|\s|\()/i.test(label);
 }
 
 function isCloseIconLabel(label: string): boolean {
   return ['x', '×', '✕', '✖', '⨯'].includes(label);
 }
 
 function isMinimizeLabel(label: string): boolean {
-  return /^minimi[sz]e$/.test(label);
+  return /^minimi[sz]e(?:\b|\s|\()/i.test(label);
 }
 
 function isLikelyCollapsedWarningControl(node: SnapshotNode): boolean {
   return !node.rect || node.rect.height <= 180;
 }
 
+function isSafeCollapsedWarningCoordinateFallback(node: SnapshotNode): boolean {
+  const label = readNodeLabel(node);
+  return (
+    isReactNativeOpenDebuggerWarningLabel(label?.trim().toLowerCase() ?? '') &&
+    isReactNativeCollapsedWarningBanner(node)
+  );
+}
+
+function isFullScreenOverlayRect(rect: RawSnapshotNode['rect']): boolean {
+  if (!rect) return false;
+  return rect.x <= 1 && rect.y <= 1 && rect.width >= 300 && rect.height >= 600;
+}
+
+function isReactNativeCollapsedWarningBanner(node: ReactNativeOverlayNode): boolean {
+  if (!node.rect) return false;
+  return node.rect.width >= 120 && node.rect.height >= 36 && node.rect.height <= 180;
+}
+
 function collectOverlayNodes(
   nodes: SnapshotNode[],
   matches: (label: string) => boolean,
@@ -150,11 +265,32 @@ function collectOverlayNodes(
 }
 
 function refsOf(nodes: SnapshotNode[]): string[] {
-  return nodes.map((node) => node.ref);
+  return Array.from(new Set(nodes.map((node) => node.ref)));
+}
+
+function firstControlNodeWithRect(nodes: SnapshotNode[]): SnapshotNode | null {
+  const withRect = nodes.filter((node) => node.rect);
+  if (withRect.length === 0) return null;
+  return (
+    withRect.sort((a, b) => {
+      const aSemanticControl = isSemanticControlNode(a) ? 1 : 0;
+      const bSemanticControl = isSemanticControlNode(b) ? 1 : 0;
+      if (aSemanticControl !== bSemanticControl) return bSemanticControl - aSemanticControl;
+      const aHittable = a.hittable === true ? 1 : 0;
+      const bHittable = b.hittable === true ? 1 : 0;
+      if (aHittable !== bHittable) return bHittable - aHittable;
+      return rectArea(a.rect) - rectArea(b.rect);
+    })[0] ?? null
+  );
+}
+
+function isSemanticControlNode(node: ReactNativeOverlayNode): boolean {
+  const roleText = [node.type, node.role, node.subrole].join(' ').toLowerCase();
+  return /\b(button|menuitem|link)\b/.test(roleText);
 }
 
-function firstNodeWithRect(nodes: SnapshotNode[]): SnapshotNode | null {
-  return nodes.find((node) => node.rect) ?? null;
+function rectArea(rect: Rect | undefined): number {
+  return rect ? rect.width * rect.height : Number.POSITIVE_INFINITY;
 }
 
 function targetFromNode(
@@ -167,14 +303,15 @@ function targetFromNode(
   return {
     action,
     point: centerOfRect(node.rect),
+    rect: node.rect,
     ref: node.ref,
     label: readNodeLabel(node),
   };
 }
 
 function actionFromDismissNode(node: SnapshotNode): ReactNativeOverlayDismissTarget['action'] {
   const label = readNodeLabel(node)?.trim().toLowerCase();
-  if (label === 'dismiss') return 'dismiss';
+  if (label && isDismissLabel(label)) return 'dismiss';
   return 'close';
 }
 
@@ -214,6 +351,6 @@ function clamp(value: number, min: number, max: number): number {
   return Math.min(max, Math.max(min, value));
 }
 
-function readNodeLabel(node: SnapshotNode): string | undefined {
+function readNodeLabel(node: ReactNativeOverlayNode): string | undefined {
   return node.label ?? node.value ?? node.identifier;
 }