fix: improve ios selector reads and maestro reliability

Author: thymikee

examples/test-app/replays/checkout-form-android.ad

@@ -2,6 +2,7 @@ context platform=android timeout=60000
 env APP_TARGET="Agent Device Tester"
 env APP_URL=""
 open "${APP_TARGET}" --relaunch --launch-url "${APP_URL}"
+react-native dismiss-overlay
 wait "label=\"Form\"" 30000
 click "label=\"Form\""
 wait "Checkout form" 5000
@@ -12,10 +13,11 @@ wait "Checkout form" 5000
 scroll down 0.6
 click id="shipping-pickup"
 click id="payment-cash"
-wait "Delivery choices" 5000
+wait "Delivery" 5000
 scroll down 0.7
 click id="checkbox-agree"
 click id="submit-order"
+scroll up 0.6
 wait "Order summary" 5000
 wait "Ada Lovelace chose pickup with cash payment." 5000
 close

examples/test-app/replays/checkout-form.ad

@@ -2,6 +2,7 @@ context platform=ios timeout=60000
 env APP_TARGET="Agent Device Tester"
 env APP_URL=""
 open "${APP_TARGET}" --relaunch --launch-url "${APP_URL}"
+react-native dismiss-overlay
 wait "label=\"Form\"" 30000
 click "label=\"Form\""
 wait "Checkout form" 5000

examples/test-app/replays/gesture-lab.ad

@@ -4,6 +4,7 @@ env APP_TARGET="Agent Device Tester"
 env APP_URL=""
 
 open "${APP_TARGET}" --relaunch --launch-url "${APP_URL}"
+react-native dismiss-overlay
 wait "Gesture lab" 30000
 
 gesture fling left 195 443 180

ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift

@@ -283,6 +283,13 @@ extension RunnerTests {
 
   func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? {
     let point = CGPoint(x: x, y: y)
+    let textInputCandidates = textInputCandidatesAt(app: app, point: point)
+    for element in textInputCandidates where prefersExpandedTextRead(element) {
+      if let text = readableText(for: element) {
+        return text
+      }
+    }
+
     let candidates = app.descendants(matching: .any).allElementsBoundByIndex
       .filter { element in
         element.exists && !element.frame.isEmpty && element.frame.contains(point)
@@ -337,15 +344,18 @@ extension RunnerTests {
   }
 
   func textInputAt(app: XCUIApplication, x: Double, y: Double) -> XCUIElement? {
-    let point = CGPoint(x: x, y: y)
-    var matched: XCUIElement?
+    return textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y)).first
+  }
+
+  private func textInputCandidatesAt(app: XCUIApplication, point: CGPoint) -> [XCUIElement] {
+    var candidates: [XCUIElement] = []
     let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
       // Query the text-input element types directly instead of enumerating the entire tree
       // (app.descendants(.any).allElementsBoundByIndex snapshots every element and is ~10x
       // slower — it dominated fill latency because resolveTextEntryElement re-runs this on
       // each verify/repair poll once the focused field reference goes stale).
       // Prefer the smallest matching field so nested editable controls win over large containers.
-      let candidates = [
+      candidates = [
         app.textFields,
         app.secureTextFields,
         app.searchFields,
@@ -371,16 +381,15 @@ extension RunnerTests {
           }
           return left.elementType.rawValue < right.elementType.rawValue
         }
-      matched = candidates.first
     })
     if let exceptionMessage {
       NSLog(
         "AGENT_DEVICE_RUNNER_TEXT_INPUT_AT_POINT_IGNORED_EXCEPTION=%@",
         exceptionMessage
       )
-      return nil
+      return []
     }
-    return matched
+    return candidates
   }
 
   private func frameContainsPoint(_ frame: CGRect, _ point: CGPoint, tolerance: CGFloat) -> Bool {
@@ -1019,6 +1028,14 @@ extension RunnerTests {
       return (wasVisible: true, dismissed: !visible, visible: visible)
     }
 
+    if tapKeyboardReturnControl(app: app, allowCoordinateFallback: true) {
+      sleepFor(0.2)
+      let visible = isKeyboardVisible(app: app)
+      if !visible {
+        return (wasVisible: true, dismissed: true, visible: false)
+      }
+    }
+
     return (wasVisible: true, dismissed: false, visible: isKeyboardVisible(app: app))
 #endif
   }
@@ -1139,7 +1156,10 @@ extension RunnerTests {
 #endif
   }
 
-  private func tapKeyboardReturnControl(app: XCUIApplication) -> Bool {
+  private func tapKeyboardReturnControl(
+    app: XCUIApplication,
+    allowCoordinateFallback: Bool = false
+  ) -> Bool {
 #if os(iOS)
     for label in ["return", "Return", "Enter", "Go", "Search", "Next", "Done", "Send", "Join"] {
       let candidates = [
@@ -1150,6 +1170,21 @@ extension RunnerTests {
         hittable.tap()
         return true
       }
+      if allowCoordinateFallback,
+         let keyboardFrame = visibleKeyboardFrame(app: app),
+         let framed = candidates.first(where: {
+           guard $0.exists else { return false }
+           let frame = $0.frame
+           return !frame.isEmpty && keyboardFrame.contains(CGPoint(x: frame.midX, y: frame.midY))
+         }) {
+        let frame = framed.frame
+        switch tapAt(app: app, x: frame.midX, y: frame.midY) {
+        case .performed:
+          return true
+        case .unsupported:
+          return false
+        }
+      }
     }
 #endif
     return false

ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift

@@ -110,11 +110,25 @@ extension RunnerTests {
 #if os(tvOS)
     return .unsupported("element tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element")
 #else
-    element.tap()
+    let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
+      element.tap()
+    })
+    if let exceptionMessage {
+      NSLog("AGENT_DEVICE_RUNNER_ELEMENT_TAP_IGNORED_EXCEPTION=%@", exceptionMessage)
+      if isPostTapElementDisappearance(exceptionMessage) {
+        return .performed
+      }
+      return .unsupported("element tap failed: \(exceptionMessage)")
+    }
     return .performed
 #endif
   }
 
+  private func isPostTapElementDisappearance(_ message: String) -> Bool {
+    message.contains("No matches found")
+      || message.contains("Failed to get matching snapshot")
+  }
+
   private func selectFocusedTvElement(app: XCUIApplication, element: XCUIElement, action: String) -> RunnerInteractionOutcome? {
 #if os(tvOS)
     guard tvFocusedElementMatches(app: app, target: element) else {

scripts/perf/scenario.ts

@@ -55,12 +55,14 @@ export function buildSettingsTour(p: ResolvedProfile, ctx: StepContext): Scenari
         // iOS: editable search field exists at root; fill it directly (freshRoot resets scroll).
         bat('fill search', 'fill', { command: 'fill', positionals: [s.searchFieldEditable, 'general'] }, { freshRoot: true }),
         bat('type', 'type', { command: 'type', positionals: ['wifi'] }),
+        bat('get editable text', 'get', { command: 'get', positionals: ['text', s.searchFieldEditable] }),
       ]
     : [
         // Android: tap the search entry first to reveal the editable, then type/fill it.
         bat('press search field', 'press', { command: 'press', positionals: [s.searchField] }, { freshRoot: true }),
         bat('type', 'type', { command: 'type', positionals: ['wifi'] }),
         bat('fill search', 'fill', { command: 'fill', positionals: [s.searchFieldEditable, 'general'] }),
+        bat('get editable text', 'get', { command: 'get', positionals: ['text', s.searchFieldEditable] }),
       ];
 
   return [

src/commands/client-command-metadata.ts

@@ -196,6 +196,7 @@ export const clientCommandMetadata = [
   defineClientCommandMetadata('settings', {
     setting: requiredField(stringField()),
     state: requiredField(stringField()),
+    app: stringField(),
     latitude: numberField(),
     longitude: numberField(),
     permission: stringField(),

src/commands/react-native/overlay.ts

@@ -59,14 +59,18 @@ export function detectReactNativeOverlay(nodes: SnapshotNode[]): ReactNativeOver
   const minimizeRefs = refsOf(minimizeNodes);
   const collapsedRefs = refsOf(collapsedNodes);
   const hasReactNativeStackFrame = isReactNativeStackFrame(text);
+  const hasControllessRedBoxText =
+    /\buncaught\b/.test(text) && /unable to download asset/.test(text);
   const hasOverlayControl =
     dismissRefs.length > 0 || minimizeRefs.length > 0 || /\b(reload js|copy stack)\b/.test(text);
   const redBox =
     /\b(redbox|runtime error|reload js|copy stack|component stack|call stack)\b/.test(text) ||
+    hasControllessRedBoxText ||
     (hasReactNativeStackFrame && hasOverlayControl);
   const detected =
     collapsedRefs.length > 0 ||
     openDebuggerWarningNodes.length > 0 ||
+    hasControllessRedBoxText ||
     (hasOverlayControl && (hasKnownReactNativeOverlayText(text) || hasReactNativeStackFrame));
   return {
     detected,

src/compat/maestro/__tests__/replay-flow.test.ts

@@ -91,6 +91,7 @@ env:
   assert.equal(parsed.actions[4]?.flags.holdMs, 3000);
   assert.equal(parsed.actions[1]?.flags.maestro?.allowNonHittableCoordinateFallback, true);
   assert.equal(parsed.actions[6]?.flags?.maestro?.allowNonHittableCoordinateFallback, true);
+  assert.equal(parsed.actions[10]?.flags.maestro?.allowAlreadyPastLoading, true);
 });
 
 test('parseMaestroReplayFlow maps iOS openLink through the app id when available', () => {
@@ -106,6 +107,7 @@ test('parseMaestroReplayFlow maps iOS openLink through the app id when available
     parsed.actions.map((entry) => [entry.command, entry.positionals]),
     [['open', ['com.callstack.agentdevicelab', 'exp://localhost:8082']]],
   );
+  assert.equal(parsed.actions[0]?.flags.maestro?.prewarmRunnerBeforeOpen, true);
 });
 
 test('parseMaestroReplayFlow maps Android openLink through the app id when available', () => {