fix: harden maestro replay smoke tests

Author: thymikee

.github/workflows/ios.yml

@@ -54,9 +54,13 @@ jobs:
           runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
           preferred-device-name: iPhone 17 Pro
 
-      - name: Run iOS simulator smoke replay
+      - name: Prepare iOS runner
         run: |
           pnpm clean:daemon
+          node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout 240000
+
+      - name: Run iOS simulator smoke replay
+        run: |
           node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator/01-settings.ad --retries 2 --report-junit test/artifacts/replays-ios-simulator-smoke.junit.xml
 
       - name: Run iOS physical device smoke replay

.github/workflows/perf-nightly.yml

@@ -60,9 +60,13 @@ jobs:
           runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
           preferred-device-name: iPhone 17 Pro
 
-      - name: Run iOS command perf benchmark
+      - name: Prepare iOS runner
         run: |
           pnpm clean:daemon
+          node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout 240000
+
+      - name: Run iOS command perf benchmark
+        run: |
           node --experimental-strip-types scripts/perf/run.ts \
             --platform ios \
             --device "iPhone 17 Pro" \

.github/workflows/replays-nightly.yml

@@ -77,6 +77,11 @@ jobs:
           runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
           preferred-device-name: iPhone 17 Pro
 
+      - name: Prepare iOS runner
+        run: |
+          pnpm clean:daemon
+          node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout 240000
+
       - name: Run iOS simulator replay suite
         run: node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator --retries 2 --report-junit test/artifacts/replays-ios-simulator.junit.xml
 

src/__tests__/cli-network.test.ts

@@ -212,6 +212,114 @@ test('test command --verbose prints step telemetry for passing tests without deb
   }
 });
 
+test('test command --verbose keeps nested retry and open step telemetry distinct', async () => {
+  const tmpDir = await fs.mkdtemp(
+    path.join(os.tmpdir(), 'agent-device-cli-test-verbose-retry-'),
+  );
+  const artifactsDir = path.join(tmpDir, 'material-top-tabs');
+  const attemptDir = path.join(artifactsDir, 'attempt-1');
+  await fs.mkdir(attemptDir, { recursive: true });
+  await fs.writeFile(
+    path.join(attemptDir, 'replay-timing.ndjson'),
+    [
+      {
+        type: 'replay_action_start',
+        step: 2,
+        line: 4,
+        command: 'retry',
+        positionals: ['3'],
+      },
+      {
+        type: 'replay_action_start',
+        step: 2,
+        line: 4,
+        command: 'open',
+        positionals: ['org.reactnavigation.playground', 'rne://material-top-tabs-basic'],
+      },
+      {
+        type: 'replay_action_stop',
+        step: 2,
+        line: 4,
+        command: 'open',
+        ok: true,
+        durationMs: 727,
+      },
+      {
+        type: 'replay_action_start',
+        step: 2.001,
+        line: 4,
+        command: '__maestroAssertVisible',
+        positionals: ['label="Chat" || text="Chat" || id="Chat"', '60000'],
+      },
+      {
+        type: 'replay_action_stop',
+        step: 2.001,
+        line: 4,
+        command: '__maestroAssertVisible',
+        ok: true,
+        durationMs: 2580,
+      },
+      {
+        type: 'replay_action_stop',
+        step: 2,
+        line: 4,
+        command: 'retry',
+        ok: true,
+        durationMs: 3310,
+      },
+    ]
+      .map((entry) => JSON.stringify(entry))
+      .join('\n'),
+  );
+
+  try {
+    const result = await runCliCapture(['test', './suite', '--verbose'], async () => ({
+      ok: true,
+      data: {
+        total: 1,
+        executed: 1,
+        passed: 1,
+        failed: 0,
+        skipped: 0,
+        notRun: 0,
+        durationMs: 3310,
+        failures: [],
+        tests: [
+          {
+            file: '/tmp/material-top-tabs.yml',
+            title: 'Material Top Tabs - Basic',
+            session: 'default:test:suite:1',
+            status: 'passed',
+            durationMs: 3310,
+            finalAttemptDurationMs: 3310,
+            attempts: 1,
+            artifactsDir,
+            replayed: 1,
+            healed: 0,
+          },
+        ],
+      },
+    }));
+
+    assert.equal(result.code, null);
+    assert.match(
+      result.stdout,
+      /open "org\.reactnavigation\.playground" "rne:\/\/material-top-tabs-basic" \(line 4, 0\.727s\)/,
+    );
+    assert.match(
+      result.stdout,
+      /assertVisible "label=\\"Chat\\" \|\| text=\\"Chat\\" \|\| id=\\"Chat\\"" "60000" \(line 4, 2\.58s\)/,
+    );
+    assert.match(result.stdout, /retry "3" \(line 4, 3\.31s\)/);
+    assert.doesNotMatch(
+      result.stdout,
+      /open "org\.reactnavigation\.playground" "rne:\/\/material-top-tabs-basic" \(line 4, 3\.31s\)/,
+    );
+  } finally {
+    await fs.rm(tmpDir, { recursive: true, force: true });
+  }
+});
+
 test('test command reports flaky passed-on-retry cases in the default summary', async () => {
   const result = await runCliCapture(['test', './suite'], async () => ({
     ok: true,

src/cli-test.ts

@@ -1,6 +1,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import type { ReplaySuiteResult, ReplaySuiteTestResult } from './daemon/types.ts';
+import { formatDurationSeconds } from './utils/duration-format.ts';
 import { AppError } from './utils/errors.ts';
 import { printJson } from './utils/output.ts';
 
@@ -142,20 +143,40 @@ function replayTestStepLines(result: ReplaySuiteTestResult): string[] {
   const events = readReplayTimingTrace(tracePath);
   if (events.length === 0) return [];
 
-  const starts = new Map<number, ReplayActionStartTrace>();
-  const stops: ReplayActionStopTrace[] = [];
+  const starts: ReplayActionStartTrace[] = [];
+  const stops: Array<{ stop: ReplayActionStopTrace; start: ReplayActionStartTrace | undefined }> =
+    [];
   for (const event of events) {
-    if (isReplayActionStartTrace(event)) starts.set(event.step, event);
-    if (isReplayActionStopTrace(event)) stops.push(event);
+    if (isReplayActionStartTrace(event)) {
+      starts.push(event);
+      continue;
+    }
+    if (isReplayActionStopTrace(event)) {
+      stops.push({ stop: event, start: consumeReplayActionStart(starts, event) });
+    }
   }
   if (stops.length === 0) return [];
 
   return [
     result.attempts > 1 ? `steps (attempt ${result.attempts}):` : 'steps:',
-    ...stops.map((stop) => renderReplayStepTrace(stop, starts.get(stop.step))),
+    ...stops.map(({ stop, start }) => renderReplayStepTrace(stop, start)),
   ];
 }
 
+function consumeReplayActionStart(
+  starts: ReplayActionStartTrace[],
+  stop: ReplayActionStopTrace,
+): ReplayActionStartTrace | undefined {
+  const stopCommand = stop.command;
+  const matchingIndex = starts.findIndex(
+    (start) =>
+      start.step === stop.step &&
+      (stopCommand === undefined || start.command === undefined || start.command === stopCommand),
+  );
+  if (matchingIndex < 0) return undefined;
+  return starts.splice(matchingIndex, 1)[0];
+}
+
 function replayTestTimingTracePath(
   result: Extract<ReplaySuiteTestResult, { status: 'passed' | 'failed' }>,
 ): string | undefined {
@@ -474,13 +495,6 @@ function formatJUnitSeconds(durationMs: number): string {
   return (Math.max(0, durationMs) / 1000).toFixed(3);
 }
 
-function formatDurationSeconds(durationMs: number): string {
-  const seconds = Math.max(0, durationMs) / 1000;
-  if (seconds >= 10) return `${seconds.toFixed(1)}s`;
-  if (seconds >= 1) return `${seconds.toFixed(2)}s`;
-  return `${seconds.toFixed(3).replace(/0+$/, '').replace(/\.$/, '')}s`;
-}
-
 function xmlEscape(value: string): string {
   return value
     .replaceAll('&', '&amp;')

src/commands/interaction-targeting.ts

@@ -3,6 +3,7 @@ import { centerOfRect } from '../utils/snapshot.ts';
 import { containsPoint, pickLargestRect } from '../utils/rect-visibility.ts';
 import { findNearestHittableAncestor, normalizeType } from '../utils/snapshot-processing.ts';
 import { normalizeRect, resolveRectCenter } from '../utils/rect-center.ts';
+import { intersectArea } from '../utils/screenshot-geometry.ts';
 
 const SEMANTIC_TOUCH_ROLE_FRAGMENTS = [
   'button',
@@ -142,7 +143,7 @@ function resolveRootViewportRect(nodes: SnapshotNode[], targetRect: Rect): Rect
 }
 
 function isRectViewportSized(rect: Rect, viewportRect: Rect): boolean {
-  const overlapArea = intersectionArea(rect, viewportRect);
+  const overlapArea = intersectArea(rect, viewportRect);
   const rectArea = rect.width * rect.height;
   const viewportArea = viewportRect.width * viewportRect.height;
   if (overlapArea <= 0 || rectArea <= 0 || viewportArea <= 0) return false;
@@ -151,15 +152,3 @@ function isRectViewportSized(rect: Rect, viewportRect: Rect): boolean {
   const rectCoverage = overlapArea / rectArea;
   return viewportCoverage >= 0.9 && rectCoverage >= 0.8;
 }
-
-function intersectionArea(left: Rect, right: Rect): number {
-  const xOverlap = Math.max(
-    0,
-    Math.min(left.x + left.width, right.x + right.width) - Math.max(left.x, right.x),
-  );
-  const yOverlap = Math.max(
-    0,
-    Math.min(left.y + left.height, right.y + right.height) - Math.max(left.y, right.y),
-  );
-  return xOverlap * yOverlap;
-}

src/compat/maestro/__tests__/runtime-assertions.test.ts

@@ -151,6 +151,85 @@ test('invokeMaestroAssertVisible uses snapshot resolution for short iOS assertio
   assert.deepEqual(calls, [['snapshot', []]]);
 });
 
+test('invokeMaestroAssertVisible falls back to raw snapshot shaping when optimized snapshot misses', async () => {
+  const snapshotFlags: Array<DaemonRequest['flags']> = [];
+  const response = await invokeMaestroAssertVisible({
+    baseReq: {
+      token: 't',
+      session: 's',
+      flags: { platform: 'ios' },
+    },
+    positionals: ['id="chat"', '1000'],
+    invoke: async (req): Promise<DaemonResponse> => {
+      if (req.command === 'snapshot') {
+        snapshotFlags.push(req.flags);
+        return {
+          ok: true,
+          data:
+            req.flags?.snapshotRaw === true
+              ? snapshot([node('Chat', { identifier: 'chat' })])
+              : snapshot([]),
+        };
+      }
+      return { ok: false, error: { code: 'UNEXPECTED_COMMAND', message: req.command } };
+    },
+  });
+
+  assert.equal(response.ok, true);
+  assert.equal(snapshotFlags.length, 2);
+  assert.equal(snapshotFlags[0]?.snapshotRaw, undefined);
+  assert.equal(snapshotFlags[1]?.snapshotRaw, true);
+  assert.equal(snapshotFlags[1]?.snapshotForceFull, undefined);
+});
+
+test('invokeMaestroAssertVisible does not use raw fallback for Android identifiers', async () => {
+  const snapshotFlags: Array<DaemonRequest['flags']> = [];
+  const response = await invokeMaestroAssertVisible({
+    baseReq: {
+      token: 't',
+      session: 's',
+      flags: { platform: 'android' },
+    },
+    positionals: ['id="album-0"', '1000'],
+    invoke: async (req): Promise<DaemonResponse> => {
+      if (req.command === 'snapshot') {
+        snapshotFlags.push(req.flags);
+        return {
+          ok: true,
+          data: snapshot([node('Album item', { identifier: 'album-0' })]),
+        };
+      }
+      return { ok: false, error: { code: 'UNEXPECTED_COMMAND', message: req.command } };
+    },
+  });
+
+  assert.equal(response.ok, true);
+  assert.equal(snapshotFlags.length, 1);
+  assert.equal(snapshotFlags[0]?.snapshotRaw, undefined);
+});
+
+test('invokeMaestroAssertVisible does not use Android raw fallback for generated text selectors', async () => {
+  const snapshotFlags: Array<DaemonRequest['flags']> = [];
+  const response = await invokeMaestroAssertVisible({
+    baseReq: {
+      token: 't',
+      session: 's',
+      flags: { platform: 'android' },
+    },
+    positionals: ['label="Chat" || text="Chat" || id="Chat"', '0'],
+    invoke: async (req): Promise<DaemonResponse> => {
+      if (req.command === 'snapshot') {
+        snapshotFlags.push(req.flags);
+        return { ok: true, data: snapshot([]) };
+      }
+      return { ok: false, error: { code: 'UNEXPECTED_COMMAND', message: req.command } };
+    },
+  });
+
+  assert.equal(response.ok, false);
+  assert.equal(snapshotFlags.some((flags) => flags?.snapshotRaw === true), false);
+});
+
 test('invokeMaestroAssertVisible treats an elapsed ellipsis loading gate as already past loading', async () => {
   vi.spyOn(Date, 'now').mockReturnValueOnce(0).mockReturnValueOnce(0).mockReturnValueOnce(250);
 

src/compat/maestro/__tests__/runtime-geometry.test.ts

@@ -15,11 +15,31 @@ test('pointForMaestroTapOnTarget biases large scroll-area text containers toward
       frame: { referenceWidth: 402, referenceHeight: 874 },
     },
     true,
+    { allowLargeContainerBias: true },
   );
 
   expect(point).toEqual({ x: 84, y: 141 });
 });
 
+test('pointForMaestroTapOnTarget centers optimized broad text containers by default', () => {
+  const point = pointForMaestroTapOnTarget(
+    {
+      node: {
+        index: 5,
+        ref: 'e5',
+        type: 'scroll-area',
+        label: 'Article',
+        rect: { x: 0, y: 117, width: 402, height: 180 },
+      },
+      rect: { x: 0, y: 117, width: 402, height: 180 },
+      frame: { referenceWidth: 402, referenceHeight: 874 },
+    },
+    true,
+  );
+
+  expect(point).toEqual({ x: 201, y: 207 });
+});
+
 test('pointForMaestroTapOnTarget centers tall Android bottom-tab containers', () => {
   const point = pointForMaestroTapOnTarget(
     {