fix: harden maestro replay smoke tests

Author: thymikee

.github/workflows/ios.yml

@@ -54,9 +54,13 @@ jobs:
           runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
           preferred-device-name: iPhone 17 Pro
 
-      - name: Run iOS simulator smoke replay
+      - name: Prepare iOS runner
         run: |
           pnpm clean:daemon
+          node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout 240000
+
+      - name: Run iOS simulator smoke replay
+        run: |
           node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator/01-settings.ad --retries 2 --report-junit test/artifacts/replays-ios-simulator-smoke.junit.xml
 
       - name: Run iOS physical device smoke replay

.github/workflows/perf-nightly.yml

@@ -60,9 +60,13 @@ jobs:
           runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
           preferred-device-name: iPhone 17 Pro
 
-      - name: Run iOS command perf benchmark
+      - name: Prepare iOS runner
         run: |
           pnpm clean:daemon
+          node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout 240000
+
+      - name: Run iOS command perf benchmark
+        run: |
           node --experimental-strip-types scripts/perf/run.ts \
             --platform ios \
             --device "iPhone 17 Pro" \

.github/workflows/replays-nightly.yml

@@ -77,6 +77,11 @@ jobs:
           runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
           preferred-device-name: iPhone 17 Pro
 
+      - name: Prepare iOS runner
+        run: |
+          pnpm clean:daemon
+          node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout 240000
+
       - name: Run iOS simulator replay suite
         run: node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator --retries 2 --report-junit test/artifacts/replays-ios-simulator.junit.xml
 

ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -36,6 +36,21 @@ extension RunnerTests {
     case drag(DragVisualizationFrame)
   }
 
+  struct GestureFallback {
+    let strategy: String
+    let message: String
+    let hint: String?
+  }
+
+  private func gestureFallback(strategy: String, from outcome: RunnerInteractionOutcome) -> GestureFallback? {
+    switch outcome {
+    case .performed:
+      return nil
+    case .unsupported(let message, let hint):
+      return GestureFallback(strategy: strategy, message: message, hint: hint)
+    }
+  }
+
   /// Runs a gesture action with uniform timing capture. Touch gestures pass `idleTimeout: true`
   /// (the default) to run inside the scroll idle-timeout + quiescence-skip wrapper; synthesis
   /// gestures (pinch/rotate/transform) pass `false` because RunnerSynthesizedGesture governs its
@@ -64,15 +79,19 @@ extension RunnerTests {
   private func gestureResponse(
     message: String,
     timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double),
-    frame: GestureFrame = .none
+    frame: GestureFrame = .none,
+    fallback: GestureFallback? = nil
   ) -> Response {
     let data: DataPayload
     switch frame {
     case .none:
       data = DataPayload(
         message: message,
         gestureStartUptimeMs: timing.gestureStartUptimeMs,
-        gestureEndUptimeMs: timing.gestureEndUptimeMs
+        gestureEndUptimeMs: timing.gestureEndUptimeMs,
+        gestureFallback: fallback?.strategy,
+        gestureFallbackMessage: fallback?.message,
+        gestureFallbackHint: fallback?.hint
       )
     case .touch(let f):
       data = DataPayload(
@@ -82,7 +101,10 @@ extension RunnerTests {
         x: f?.x,
         y: f?.y,
         referenceWidth: f?.referenceWidth,
-        referenceHeight: f?.referenceHeight
+        referenceHeight: f?.referenceHeight,
+        gestureFallback: fallback?.strategy,
+        gestureFallbackMessage: fallback?.message,
+        gestureFallbackHint: fallback?.hint
       )
     case .drag(let f):
       data = DataPayload(
@@ -94,7 +116,10 @@ extension RunnerTests {
         x2: f.x2,
         y2: f.y2,
         referenceWidth: f.referenceWidth,
-        referenceHeight: f.referenceHeight
+        referenceHeight: f.referenceHeight,
+        gestureFallback: fallback?.strategy,
+        gestureFallbackMessage: fallback?.message,
+        gestureFallbackHint: fallback?.hint
       )
     }
     return Response(ok: true, data: data)
@@ -507,6 +532,7 @@ extension RunnerTests {
         x2: dragPoints.x2,
         y2: dragPoints.y2
       )
+      var fallback: GestureFallback?
       if command.synthesized == true {
         let durationMs = min(max(command.durationMs ?? 250, 16), 10000)
         let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
@@ -522,6 +548,7 @@ extension RunnerTests {
         if case .performed = outcome {
           return gestureResponse(message: "dragged", timing: timing, frame: .drag(dragFrame))
         }
+        fallback = gestureFallback(strategy: "xctest-coordinate-drag", from: outcome)
       }
       let holdDuration = command.synthesized == true
         ? synthesizedSwipeFallbackHoldDuration(durationMs: command.durationMs ?? 250)
@@ -539,7 +566,12 @@ extension RunnerTests {
       if let response = unsupportedResponse(for: outcome) {
         return response
       }
-      return gestureResponse(message: "dragged", timing: timing, frame: .drag(dragFrame))
+      return gestureResponse(
+        message: "dragged",
+        timing: timing,
+        frame: .drag(dragFrame),
+        fallback: fallback
+      )
     case .dragSeries:
       guard let x = command.x, let y = command.y, let x2 = command.x2, let y2 = command.y2 else {
         return Response(ok: false, error: ErrorPayload(message: "dragSeries requires x, y, x2, and y2"))
@@ -550,8 +582,47 @@ extension RunnerTests {
       if pattern != "one-way" && pattern != "ping-pong" {
         return Response(ok: false, error: ErrorPayload(message: "dragSeries pattern must be one-way or ping-pong"))
       }
-      let holdDuration = min(max((command.durationMs ?? 60) / 1000.0, 0.016), 10.0)
       let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2)
+      var fallback: GestureFallback?
+      if command.synthesized == true {
+        let durationMs = min(max(command.durationMs ?? 250, 16), 10000)
+        let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
+          var outcome = RunnerInteractionOutcome.performed
+          runSeries(count: count, pauseMs: pauseMs) { idx in
+            guard case .performed = outcome else {
+              return
+            }
+            let reverse = pattern == "ping-pong" && (idx % 2 == 1)
+            if reverse {
+              outcome = synthesizedDragAt(
+                app: activeApp,
+                x: dragPoints.x2,
+                y: dragPoints.y2,
+                x2: dragPoints.x,
+                y2: dragPoints.y,
+                durationMs: durationMs
+              )
+            } else {
+              outcome = synthesizedDragAt(
+                app: activeApp,
+                x: dragPoints.x,
+                y: dragPoints.y,
+                x2: dragPoints.x2,
+                y2: dragPoints.y2,
+                durationMs: durationMs
+              )
+            }
+          }
+          return outcome
+        }
+        if case .performed = outcome {
+          return gestureResponse(message: "drag series", timing: timing)
+        }
+        fallback = gestureFallback(strategy: "xctest-coordinate-drag-series", from: outcome)
+      }
+      let holdDuration = command.synthesized == true
+        ? synthesizedSwipeFallbackHoldDuration(durationMs: command.durationMs ?? 250)
+        : min(max((command.durationMs ?? 60) / 1000.0, 0.016), 10.0)
       let (timing, outcome) = performGesture(activeApp) {
         var outcome = RunnerInteractionOutcome.performed
         runSeries(count: count, pauseMs: pauseMs) { idx in
@@ -584,7 +655,7 @@ extension RunnerTests {
       if let response = unsupportedResponse(for: outcome) {
         return response
       }
-      return gestureResponse(message: "drag series", timing: timing)
+      return gestureResponse(message: "drag series", timing: timing, fallback: fallback)
     case .remotePress:
       guard let button = tvRemoteButton(from: command.remoteButton) else {
         return Response(ok: false, error: ErrorPayload(message: "remotePress requires remoteButton"))

ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift

@@ -190,6 +190,9 @@ struct DataPayload: Codable {
   let wasVisible: Bool?
   let dismissed: Bool?
   let orientation: String?
+  let gestureFallback: String?
+  let gestureFallbackMessage: String?
+  let gestureFallbackHint: String?
 
   init(
     message: String? = nil,
@@ -218,7 +221,10 @@ struct DataPayload: Codable {
     visible: Bool? = nil,
     wasVisible: Bool? = nil,
     dismissed: Bool? = nil,
-    orientation: String? = nil
+    orientation: String? = nil,
+    gestureFallback: String? = nil,
+    gestureFallbackMessage: String? = nil,
+    gestureFallbackHint: String? = nil
   ) {
     self.message = message
     self.text = text
@@ -247,6 +253,9 @@ struct DataPayload: Codable {
     self.wasVisible = wasVisible
     self.dismissed = dismissed
     self.orientation = orientation
+    self.gestureFallback = gestureFallback
+    self.gestureFallbackMessage = gestureFallbackMessage
+    self.gestureFallbackHint = gestureFallbackHint
   }
 }
 

src/__tests__/cli-network.test.ts

@@ -212,6 +212,114 @@ test('test command --verbose prints step telemetry for passing tests without deb
   }
 });
 
+test('test command --verbose keeps nested retry and open step telemetry distinct', async () => {
+  const tmpDir = await fs.mkdtemp(
+    path.join(os.tmpdir(), 'agent-device-cli-test-verbose-retry-'),
+  );
+  const artifactsDir = path.join(tmpDir, 'material-top-tabs');
+  const attemptDir = path.join(artifactsDir, 'attempt-1');
+  await fs.mkdir(attemptDir, { recursive: true });
+  await fs.writeFile(
+    path.join(attemptDir, 'replay-timing.ndjson'),
+    [
+      {
+        type: 'replay_action_start',
+        step: 2,
+        line: 4,
+        command: 'retry',
+        positionals: ['3'],
+      },
+      {
+        type: 'replay_action_start',
+        step: 2,
+        line: 4,
+        command: 'open',
+        positionals: ['org.reactnavigation.playground', 'rne://material-top-tabs-basic'],
+      },
+      {
+        type: 'replay_action_stop',
+        step: 2,
+        line: 4,
+        command: 'open',
+        ok: true,
+        durationMs: 727,
+      },
+      {
+        type: 'replay_action_start',
+        step: 2.001,
+        line: 4,
+        command: '__maestroAssertVisible',
+        positionals: ['label="Chat" || text="Chat" || id="Chat"', '60000'],
+      },
+      {
+        type: 'replay_action_stop',
+        step: 2.001,
+        line: 4,
+        command: '__maestroAssertVisible',
+        ok: true,
+        durationMs: 2580,
+      },
+      {
+        type: 'replay_action_stop',
+        step: 2,
+        line: 4,
+        command: 'retry',
+        ok: true,
+        durationMs: 3310,
+      },
+    ]
+      .map((entry) => JSON.stringify(entry))
+      .join('\n'),
+  );
+
+  try {
+    const result = await runCliCapture(['test', './suite', '--verbose'], async () => ({
+      ok: true,
+      data: {
+        total: 1,
+        executed: 1,
+        passed: 1,
+        failed: 0,
+        skipped: 0,
+        notRun: 0,
+        durationMs: 3310,
+        failures: [],
+        tests: [
+          {
+            file: '/tmp/material-top-tabs.yml',
+            title: 'Material Top Tabs - Basic',
+            session: 'default:test:suite:1',
+            status: 'passed',
+            durationMs: 3310,
+            finalAttemptDurationMs: 3310,
+            attempts: 1,
+            artifactsDir,
+            replayed: 1,
+            healed: 0,
+          },
+        ],
+      },
+    }));
+
+    assert.equal(result.code, null);
+    assert.match(
+      result.stdout,
+      /open "org\.reactnavigation\.playground" "rne:\/\/material-top-tabs-basic" \(line 4, 0\.727s\)/,
+    );
+    assert.match(
+      result.stdout,
+      /assertVisible "label=\\"Chat\\" \|\| text=\\"Chat\\" \|\| id=\\"Chat\\"" "60000" \(line 4, 2\.58s\)/,
+    );
+    assert.match(result.stdout, /retry "3" \(line 4, 3\.31s\)/);
+    assert.doesNotMatch(
+      result.stdout,
+      /open "org\.reactnavigation\.playground" "rne:\/\/material-top-tabs-basic" \(line 4, 3\.31s\)/,
+    );
+  } finally {
+    await fs.rm(tmpDir, { recursive: true, force: true });
+  }
+});
+
 test('test command reports flaky passed-on-retry cases in the default summary', async () => {
   const result = await runCliCapture(['test', './suite'], async () => ({
     ok: true,