fix: improve maestro test output

Author: thymikee

src/__tests__/cli-network.test.ts

@@ -8,6 +8,7 @@ import { runCliCapture } from './cli-capture.ts';
 function makeFailedReplayResult() {
   return {
     file: '/tmp/02-fail.ad',
+    title: 'Checkout failure',
     session: 'default:test:suite:2',
     status: 'failed',
     durationMs: 5,
@@ -106,11 +107,14 @@ test('test command prints suite summary and exits non-zero on failures', async (
   assert.equal(result.calls[0]?.meta?.requestProgress, 'replay-test');
   assert.match(result.stderr, /Running replay suite\.\.\./);
   assert.doesNotMatch(result.stdout, /PASS \/tmp\/01-pass\.ad/);
-  assert.match(result.stdout, /FAIL \/tmp\/02-fail\.ad after 2 attempts \(5ms\)/);
+  assert.match(
+    result.stdout,
+    /FAIL "Checkout failure" in 02-fail\.ad after 2 attempts \(total 0\.005s\)/,
+  );
   assert.match(result.stdout, /Replay failed at step 1 \(open Demo\): boom/);
   assert.match(result.stdout, /artifacts: \/tmp\/test-artifacts\/02-fail/);
   assert.doesNotMatch(result.stdout, /SKIP \/tmp\/03-skip\.ad/);
-  assert.match(result.stdout, /Test summary: 1 passed, 1 failed in 25ms/);
+  assert.match(result.stdout, /Test summary: 1 passed, 1 failed in 0\.025s/);
 });
 
 test('test command --verbose prints all test statuses', async () => {
@@ -120,8 +124,8 @@ test('test command --verbose prints all test statuses', async () => {
 
   assert.equal(result.code, 1);
   assert.match(result.stderr, /Running replay suite\.\.\./);
-  assert.match(result.stdout, /PASS \/tmp\/01-pass\.ad \(10ms\)/);
-  assert.match(result.stdout, /SKIP \/tmp\/03-skip\.ad/);
+  assert.match(result.stdout, /PASS 01-pass\.ad \(0\.01s\)/);
+  assert.match(result.stdout, /SKIP 03-skip\.ad/);
 });
 
 test('test command reports flaky passed-on-retry cases in the default summary', async () => {
@@ -138,20 +142,127 @@ test('test command reports flaky passed-on-retry cases in the default summary',
       failures: [],
       tests: [
         {
-          file: '/tmp/01-flaky.ad',
+          file: '/tmp/auth-flow.yml',
+          title: 'Authentication flow',
           session: 'default:test:suite:1',
           status: 'passed',
-          durationMs: 10,
+          durationMs: 112151,
+          finalAttemptDurationMs: 17492,
           attempts: 2,
+          attemptFailures: [
+            {
+              attempt: 1,
+              message: 'Replay failed at step 3 (tapOn "Log in"): selector not found',
+              durationMs: 94659,
+            },
+          ],
         },
       ],
     },
   }));
 
   assert.equal(result.code, null);
   assert.match(result.stderr, /Running replay suite\.\.\./);
-  assert.match(result.stdout, /FLAKY \/tmp\/01-flaky\.ad after 2 attempts \(10ms\)/);
-  assert.match(result.stdout, /Test summary: 1 passed, 0 failed, 1 flaky in 25ms/);
+  assert.doesNotMatch(result.stdout, /FLAKY/);
+  assert.match(result.stdout, /Test summary: 1 passed, 0 failed, 1 flaky in 0\.025s/);
+  assert.match(result.stdout, /Flaky tests:/);
+  assert.match(
+    result.stdout,
+    /PASS "Authentication flow" after 2 attempts \(passed attempt 17\.5s, total 112\.2s\)/,
+  );
+  assert.match(
+    result.stdout,
+    /attempt 1 failed \(94\.7s\): Replay failed at step 3 \(tapOn "Log in"\): selector not found/,
+  );
+});
+
+test('test command prints failed attempt step telemetry when timing trace exists', async () => {
+  const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-cli-test-steps-'));
+  const artifactsDir = path.join(tmpDir, 'checkout-flow');
+  const attemptDir = path.join(artifactsDir, 'attempt-2');
+  await fs.mkdir(attemptDir, { recursive: true });
+  await fs.writeFile(
+    path.join(attemptDir, 'replay-timing.ndjson'),
+    [
+      {
+        type: 'replay_action_start',
+        step: 1,
+        line: 3,
+        command: 'open',
+        positionals: ['Demo'],
+      },
+      {
+        type: 'replay_action_stop',
+        step: 1,
+        line: 3,
+        command: 'open',
+        ok: true,
+        durationMs: 125,
+        resultTiming: { launchMs: 100 },
+      },
+      {
+        type: 'replay_action_start',
+        step: 2,
+        line: 4,
+        command: '__maestroTapOn',
+        positionals: ['text="Pay"'],
+      },
+      {
+        type: 'replay_action_stop',
+        step: 2,
+        line: 4,
+        command: '__maestroTapOn',
+        ok: false,
+        durationMs: 1500,
+        errorCode: 'ASSERTION_FAILED',
+      },
+    ]
+      .map((entry) => JSON.stringify(entry))
+      .join('\n'),
+  );
+
+  try {
+    const failedReplayResult = {
+      file: '/tmp/checkout-flow.yml',
+      title: 'Checkout flow',
+      session: 'default:test:suite:1',
+      status: 'failed',
+      durationMs: 2000,
+      attempts: 2,
+      artifactsDir,
+      error: {
+        code: 'ASSERTION_FAILED',
+        message: 'Replay failed at step 2 (click "Pay"): selector not found',
+      },
+    };
+    const result = await runCliCapture(['test', './suite'], async () => ({
+      ok: true,
+      data: {
+        total: 1,
+        executed: 1,
+        passed: 0,
+        failed: 1,
+        skipped: 0,
+        notRun: 0,
+        durationMs: 2000,
+        failures: [failedReplayResult],
+        tests: [failedReplayResult],
+      },
+    }));
+
+    assert.equal(result.code, 1);
+    assert.match(result.stdout, /steps \(attempt 2\):/);
+    assert.match(
+      result.stdout,
+      /\[ok\] open "Demo" \(line 3, 0\.125s, timing \{"launchMs":100\}\)/,
+    );
+    assert.match(
+      result.stdout,
+      /\[FAIL\] tapOn "text=\\"Pay\\"" \(line 4, 1\.50s, ASSERTION_FAILED\)/,
+    );
+  } finally {
+    await fs.rm(tmpDir, { recursive: true, force: true });
+  }
 });
 
 test('test --maestro forwards Maestro backend and platform for directory suites', async () => {