fix: harden maestro replay smoke tests

Author: thymikee

.github/workflows/ios.yml

@@ -54,9 +54,13 @@ jobs:
           runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
           preferred-device-name: iPhone 17 Pro
 
-      - name: Run iOS simulator smoke replay
+      - name: Prepare iOS runner
         run: |
           pnpm clean:daemon
+          node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout 240000
+
+      - name: Run iOS simulator smoke replay
+        run: |
           node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator/01-settings.ad --retries 2 --report-junit test/artifacts/replays-ios-simulator-smoke.junit.xml
 
       - name: Run iOS physical device smoke replay

.github/workflows/perf-nightly.yml

@@ -60,9 +60,13 @@ jobs:
           runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
           preferred-device-name: iPhone 17 Pro
 
-      - name: Run iOS command perf benchmark
+      - name: Prepare iOS runner
         run: |
           pnpm clean:daemon
+          node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout 240000
+
+      - name: Run iOS command perf benchmark
+        run: |
           node --experimental-strip-types scripts/perf/run.ts \
             --platform ios \
             --device "iPhone 17 Pro" \

.github/workflows/replays-nightly.yml

@@ -77,6 +77,11 @@ jobs:
           runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
           preferred-device-name: iPhone 17 Pro
 
+      - name: Prepare iOS runner
+        run: |
+          pnpm clean:daemon
+          node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout 240000
+
       - name: Run iOS simulator replay suite
         run: node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator --retries 2 --report-junit test/artifacts/replays-ios-simulator.junit.xml
 

ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -36,6 +36,21 @@ extension RunnerTests {
     case drag(DragVisualizationFrame)
   }
 
+  struct GestureFallback {
+    let strategy: String
+    let message: String
+    let hint: String?
+  }
+
+  private func gestureFallback(strategy: String, from outcome: RunnerInteractionOutcome) -> GestureFallback? {
+    switch outcome {
+    case .performed:
+      return nil
+    case .unsupported(let message, let hint):
+      return GestureFallback(strategy: strategy, message: message, hint: hint)
+    }
+  }
+
   /// Runs a gesture action with uniform timing capture. Touch gestures pass `idleTimeout: true`
   /// (the default) to run inside the scroll idle-timeout + quiescence-skip wrapper; synthesis
   /// gestures (pinch/rotate/transform) pass `false` because RunnerSynthesizedGesture governs its
@@ -64,15 +79,19 @@ extension RunnerTests {
   private func gestureResponse(
     message: String,
     timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double),
-    frame: GestureFrame = .none
+    frame: GestureFrame = .none,
+    fallback: GestureFallback? = nil
   ) -> Response {
     let data: DataPayload
     switch frame {
     case .none:
       data = DataPayload(
         message: message,
         gestureStartUptimeMs: timing.gestureStartUptimeMs,
-        gestureEndUptimeMs: timing.gestureEndUptimeMs
+        gestureEndUptimeMs: timing.gestureEndUptimeMs,
+        gestureFallback: fallback?.strategy,
+        gestureFallbackMessage: fallback?.message,
+        gestureFallbackHint: fallback?.hint
       )
     case .touch(let f):
       data = DataPayload(
@@ -82,7 +101,10 @@ extension RunnerTests {
         x: f?.x,
         y: f?.y,
         referenceWidth: f?.referenceWidth,
-        referenceHeight: f?.referenceHeight
+        referenceHeight: f?.referenceHeight,
+        gestureFallback: fallback?.strategy,
+        gestureFallbackMessage: fallback?.message,
+        gestureFallbackHint: fallback?.hint
       )
     case .drag(let f):
       data = DataPayload(
@@ -94,7 +116,10 @@ extension RunnerTests {
         x2: f.x2,
         y2: f.y2,
         referenceWidth: f.referenceWidth,
-        referenceHeight: f.referenceHeight
+        referenceHeight: f.referenceHeight,
+        gestureFallback: fallback?.strategy,
+        gestureFallbackMessage: fallback?.message,
+        gestureFallbackHint: fallback?.hint
       )
     }
     return Response(ok: true, data: data)
@@ -507,6 +532,7 @@ extension RunnerTests {
         x2: dragPoints.x2,
         y2: dragPoints.y2
       )
+      var fallback: GestureFallback?
       if command.synthesized == true {
         let durationMs = min(max(command.durationMs ?? 250, 16), 10000)
         let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
@@ -522,6 +548,7 @@ extension RunnerTests {
         if case .performed = outcome {
           return gestureResponse(message: "dragged", timing: timing, frame: .drag(dragFrame))
         }
+        fallback = gestureFallback(strategy: "xctest-coordinate-drag", from: outcome)
       }
       let holdDuration = command.synthesized == true
         ? synthesizedSwipeFallbackHoldDuration(durationMs: command.durationMs ?? 250)
@@ -539,7 +566,12 @@ extension RunnerTests {
       if let response = unsupportedResponse(for: outcome) {
         return response
       }
-      return gestureResponse(message: "dragged", timing: timing, frame: .drag(dragFrame))
+      return gestureResponse(
+        message: "dragged",
+        timing: timing,
+        frame: .drag(dragFrame),
+        fallback: fallback
+      )
     case .dragSeries:
       guard let x = command.x, let y = command.y, let x2 = command.x2, let y2 = command.y2 else {
         return Response(ok: false, error: ErrorPayload(message: "dragSeries requires x, y, x2, and y2"))

ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift

@@ -190,6 +190,9 @@ struct DataPayload: Codable {
   let wasVisible: Bool?
   let dismissed: Bool?
   let orientation: String?
+  let gestureFallback: String?
+  let gestureFallbackMessage: String?
+  let gestureFallbackHint: String?
 
   init(
     message: String? = nil,
@@ -218,7 +221,10 @@ struct DataPayload: Codable {
     visible: Bool? = nil,
     wasVisible: Bool? = nil,
     dismissed: Bool? = nil,
-    orientation: String? = nil
+    orientation: String? = nil,
+    gestureFallback: String? = nil,
+    gestureFallbackMessage: String? = nil,
+    gestureFallbackHint: String? = nil
   ) {
     self.message = message
     self.text = text
@@ -247,6 +253,9 @@ struct DataPayload: Codable {
     self.wasVisible = wasVisible
     self.dismissed = dismissed
     self.orientation = orientation
+    self.gestureFallback = gestureFallback
+    self.gestureFallbackMessage = gestureFallbackMessage
+    self.gestureFallbackHint = gestureFallbackHint
   }
 }
 

src/__tests__/cli-network.test.ts

@@ -212,6 +212,114 @@ test('test command --verbose prints step telemetry for passing tests without deb
   }
 });
 
+test('test command --verbose keeps nested retry and open step telemetry distinct', async () => {
+  const tmpDir = await fs.mkdtemp(
+    path.join(os.tmpdir(), 'agent-device-cli-test-verbose-retry-'),
+  );
+  const artifactsDir = path.join(tmpDir, 'material-top-tabs');
+  const attemptDir = path.join(artifactsDir, 'attempt-1');
+  await fs.mkdir(attemptDir, { recursive: true });
+  await fs.writeFile(
+    path.join(attemptDir, 'replay-timing.ndjson'),
+    [
+      {
+        type: 'replay_action_start',
+        step: 2,
+        line: 4,
+        command: 'retry',
+        positionals: ['3'],
+      },
+      {
+        type: 'replay_action_start',
+        step: 2,
+        line: 4,
+        command: 'open',
+        positionals: ['org.reactnavigation.playground', 'rne://material-top-tabs-basic'],
+      },
+      {
+        type: 'replay_action_stop',
+        step: 2,
+        line: 4,
+        command: 'open',
+        ok: true,
+        durationMs: 727,
+      },
+      {
+        type: 'replay_action_start',
+        step: 2.001,
+        line: 4,
+        command: '__maestroAssertVisible',
+        positionals: ['label="Chat" || text="Chat" || id="Chat"', '60000'],
+      },
+      {
+        type: 'replay_action_stop',
+        step: 2.001,
+        line: 4,
+        command: '__maestroAssertVisible',
+        ok: true,
+        durationMs: 2580,
+      },
+      {
+        type: 'replay_action_stop',
+        step: 2,
+        line: 4,
+        command: 'retry',
+        ok: true,
+        durationMs: 3310,
+      },
+    ]
+      .map((entry) => JSON.stringify(entry))
+      .join('\n'),
+  );
+
+  try {
+    const result = await runCliCapture(['test', './suite', '--verbose'], async () => ({
+      ok: true,
+      data: {
+        total: 1,
+        executed: 1,
+        passed: 1,
+        failed: 0,
+        skipped: 0,
+        notRun: 0,
+        durationMs: 3310,
+        failures: [],
+        tests: [
+          {
+            file: '/tmp/material-top-tabs.yml',
+            title: 'Material Top Tabs - Basic',
+            session: 'default:test:suite:1',
+            status: 'passed',
+            durationMs: 3310,
+            finalAttemptDurationMs: 3310,
+            attempts: 1,
+            artifactsDir,
+            replayed: 1,
+            healed: 0,
+          },
+        ],
+      },
+    }));
+
+    assert.equal(result.code, null);
+    assert.match(
+      result.stdout,
+      /open "org\.reactnavigation\.playground" "rne:\/\/material-top-tabs-basic" \(line 4, 0\.727s\)/,
+    );
+    assert.match(
+      result.stdout,
+      /assertVisible "label=\\"Chat\\" \|\| text=\\"Chat\\" \|\| id=\\"Chat\\"" "60000" \(line 4, 2\.58s\)/,
+    );
+    assert.match(result.stdout, /retry "3" \(line 4, 3\.31s\)/);
+    assert.doesNotMatch(
+      result.stdout,
+      /open "org\.reactnavigation\.playground" "rne:\/\/material-top-tabs-basic" \(line 4, 3\.31s\)/,
+    );
+  } finally {
+    await fs.rm(tmpDir, { recursive: true, force: true });
+  }
+});
+
 test('test command reports flaky passed-on-retry cases in the default summary', async () => {
   const result = await runCliCapture(['test', './suite'], async () => ({
     ok: true,

src/cli-test.ts

@@ -1,6 +1,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import type { ReplaySuiteResult, ReplaySuiteTestResult } from './daemon/types.ts';
+import { formatDurationSeconds } from './utils/duration-format.ts';
 import { AppError } from './utils/errors.ts';
 import { printJson } from './utils/output.ts';
 
@@ -142,20 +143,40 @@ function replayTestStepLines(result: ReplaySuiteTestResult): string[] {
   const events = readReplayTimingTrace(tracePath);
   if (events.length === 0) return [];
 
-  const starts = new Map<number, ReplayActionStartTrace>();
-  const stops: ReplayActionStopTrace[] = [];
+  const starts: ReplayActionStartTrace[] = [];
+  const stops: Array<{ stop: ReplayActionStopTrace; start: ReplayActionStartTrace | undefined }> =
+    [];
   for (const event of events) {
-    if (isReplayActionStartTrace(event)) starts.set(event.step, event);
-    if (isReplayActionStopTrace(event)) stops.push(event);
+    if (isReplayActionStartTrace(event)) {
+      starts.push(event);
+      continue;
+    }
+    if (isReplayActionStopTrace(event)) {
+      stops.push({ stop: event, start: consumeReplayActionStart(starts, event) });
+    }
   }
   if (stops.length === 0) return [];
 
   return [
     result.attempts > 1 ? `steps (attempt ${result.attempts}):` : 'steps:',
-    ...stops.map((stop) => renderReplayStepTrace(stop, starts.get(stop.step))),
+    ...stops.map(({ stop, start }) => renderReplayStepTrace(stop, start)),
   ];
 }
 
+function consumeReplayActionStart(
+  starts: ReplayActionStartTrace[],
+  stop: ReplayActionStopTrace,
+): ReplayActionStartTrace | undefined {
+  const stopCommand = stop.command;
+  const matchingIndex = starts.findIndex(
+    (start) =>
+      start.step === stop.step &&
+      (stopCommand === undefined || start.command === undefined || start.command === stopCommand),
+  );
+  if (matchingIndex < 0) return undefined;
+  return starts.splice(matchingIndex, 1)[0];
+}
+
 function replayTestTimingTracePath(
   result: Extract<ReplaySuiteTestResult, { status: 'passed' | 'failed' }>,
 ): string | undefined {
@@ -474,13 +495,6 @@ function formatJUnitSeconds(durationMs: number): string {
   return (Math.max(0, durationMs) / 1000).toFixed(3);
 }
 
-function formatDurationSeconds(durationMs: number): string {
-  const seconds = Math.max(0, durationMs) / 1000;
-  if (seconds >= 10) return `${seconds.toFixed(1)}s`;
-  if (seconds >= 1) return `${seconds.toFixed(2)}s`;
-  return `${seconds.toFixed(3).replace(/0+$/, '').replace(/\.$/, '')}s`;
-}
-
 function xmlEscape(value: string): string {
   return value
     .replaceAll('&', '&amp;')

src/commands/interaction-targeting.ts

@@ -3,6 +3,7 @@ import { centerOfRect } from '../utils/snapshot.ts';
 import { containsPoint, pickLargestRect } from '../utils/rect-visibility.ts';
 import { findNearestHittableAncestor, normalizeType } from '../utils/snapshot-processing.ts';
 import { normalizeRect, resolveRectCenter } from '../utils/rect-center.ts';
+import { intersectArea } from '../utils/screenshot-geometry.ts';
 
 const SEMANTIC_TOUCH_ROLE_FRAGMENTS = [
   'button',
@@ -142,7 +143,7 @@ function resolveRootViewportRect(nodes: SnapshotNode[], targetRect: Rect): Rect
 }
 
 function isRectViewportSized(rect: Rect, viewportRect: Rect): boolean {
-  const overlapArea = intersectionArea(rect, viewportRect);
+  const overlapArea = intersectArea(rect, viewportRect);
   const rectArea = rect.width * rect.height;
   const viewportArea = viewportRect.width * viewportRect.height;
   if (overlapArea <= 0 || rectArea <= 0 || viewportArea <= 0) return false;
@@ -151,15 +152,3 @@ function isRectViewportSized(rect: Rect, viewportRect: Rect): boolean {
   const rectCoverage = overlapArea / rectArea;
   return viewportCoverage >= 0.9 && rectCoverage >= 0.8;
 }
-
-function intersectionArea(left: Rect, right: Rect): number {
-  const xOverlap = Math.max(
-    0,
-    Math.min(left.x + left.width, right.x + right.width) - Math.max(left.x, right.x),
-  );
-  const yOverlap = Math.max(
-    0,
-    Math.min(left.y + left.height, right.y + right.height) - Math.max(left.y, right.y),
-  );
-  return xOverlap * yOverlap;
-}