fix: improve maestro test output

Author: thymikee

src/__tests__/cli-network.test.ts

@@ -106,11 +106,11 @@ test('test command prints suite summary and exits non-zero on failures', async (
   assert.equal(result.calls[0]?.meta?.requestProgress, 'replay-test');
   assert.match(result.stderr, /Running replay suite\.\.\./);
   assert.doesNotMatch(result.stdout, /PASS \/tmp\/01-pass\.ad/);
-  assert.match(result.stdout, /FAIL \/tmp\/02-fail\.ad after 2 attempts \(5ms\)/);
+  assert.match(result.stdout, /FAIL 02-fail\.ad after 2 attempts \(total 0\.005s\)/);
   assert.match(result.stdout, /Replay failed at step 1 \(open Demo\): boom/);
   assert.match(result.stdout, /artifacts: \/tmp\/test-artifacts\/02-fail/);
   assert.doesNotMatch(result.stdout, /SKIP \/tmp\/03-skip\.ad/);
-  assert.match(result.stdout, /Test summary: 1 passed, 1 failed in 25ms/);
+  assert.match(result.stdout, /Test summary: 1 passed, 1 failed in 0\.025s/);
 });
 
 test('test command --verbose prints all test statuses', async () => {
@@ -120,8 +120,8 @@ test('test command --verbose prints all test statuses', async () => {
 
   assert.equal(result.code, 1);
   assert.match(result.stderr, /Running replay suite\.\.\./);
-  assert.match(result.stdout, /PASS \/tmp\/01-pass\.ad \(10ms\)/);
-  assert.match(result.stdout, /SKIP \/tmp\/03-skip\.ad/);
+  assert.match(result.stdout, /PASS 01-pass\.ad \(0\.01s\)/);
+  assert.match(result.stdout, /SKIP 03-skip\.ad/);
 });
 
 test('test command reports flaky passed-on-retry cases in the default summary', async () => {
@@ -138,20 +138,37 @@ test('test command reports flaky passed-on-retry cases in the default summary',
       failures: [],
       tests: [
         {
-          file: '/tmp/01-flaky.ad',
+          file: '/tmp/auth-flow.yml',
           session: 'default:test:suite:1',
           status: 'passed',
-          durationMs: 10,
+          durationMs: 112151,
+          finalAttemptDurationMs: 17492,
           attempts: 2,
+          attemptFailures: [
+            {
+              attempt: 1,
+              message: 'Replay failed at step 3 (tapOn "Log in"): selector not found',
+              durationMs: 94659,
+            },
+          ],
         },
       ],
     },
   }));
 
   assert.equal(result.code, null);
   assert.match(result.stderr, /Running replay suite\.\.\./);
-  assert.match(result.stdout, /FLAKY \/tmp\/01-flaky\.ad after 2 attempts \(10ms\)/);
-  assert.match(result.stdout, /Test summary: 1 passed, 0 failed, 1 flaky in 25ms/);
+  assert.doesNotMatch(result.stdout, /FLAKY/);
+  assert.match(result.stdout, /Test summary: 1 passed, 0 failed, 1 flaky in 0\.025s/);
+  assert.match(result.stdout, /Flaky tests:/);
+  assert.match(
+    result.stdout,
+    /PASS auth-flow\.yml after 2 attempts \(passed attempt 17\.5s, total 112\.2s\)/,
+  );
+  assert.match(
+    result.stdout,
+    /attempt 1 failed \(94\.7s\): Replay failed at step 3 \(tapOn "Log in"\): selector not found/,
+  );
 });
 
 test('test --maestro forwards Maestro backend and platform for directory suites', async () => {

src/cli-test.ts

@@ -40,16 +40,15 @@ function renderReplayTestSummary(
     for (const entry of data.failures) {
       renderFailedTestResult(entry);
     }
-    for (const entry of flaky) {
-      renderFlakyTestResult(entry);
-    }
   }
 
   const durationMs = typeof data.durationMs === 'number' ? data.durationMs : undefined;
   const flakySuffix = flaky.length > 0 ? `, ${flaky.length} flaky` : '';
+  const durationSuffix = durationMs !== undefined ? ` in ${formatDurationSeconds(durationMs)}` : '';
   process.stdout.write(
-    `Test summary: ${data.passed} passed, ${data.failed} failed${flakySuffix}${durationMs !== undefined ? ` in ${durationMs}ms` : ''}\n`,
+    `Test summary: ${data.passed} passed, ${data.failed} failed${flakySuffix}${durationSuffix}\n`,
   );
+  renderFlakyTestSummary(flaky);
   return getReplayTestExitCode(data);
 }
 
@@ -59,11 +58,10 @@ function renderVerboseTestResult(result: ReplaySuiteTestResult): void {
     return;
   }
 
-  const prefix = replayResultPrefix(result);
-  const attemptSuffix =
-    'attempts' in result && result.attempts > 1 ? ` after ${result.attempts} attempts` : '';
-  const durationSuffix = result.durationMs > 0 ? ` (${result.durationMs}ms)` : '';
-  process.stdout.write(`${prefix} ${result.file}${attemptSuffix}${durationSuffix}\n`);
+  const durationSuffix = formatReplayTestDurationSuffix(result);
+  process.stdout.write(
+    `${replayResultPrefix(result)} ${replayTestDisplayName(result)}${durationSuffix}\n`,
+  );
   if (result.status === 'skipped') {
     process.stdout.write(`  ${result.message ?? 'skipped'}\n`);
   }
@@ -73,16 +71,16 @@ function renderFailedTestResult(
   result: Extract<ReplaySuiteTestResult, { status: 'failed' }>,
 ): void {
   const attemptSuffix = result.attempts > 1 ? ` after ${result.attempts} attempts` : '';
-  const durationSuffix = result.durationMs > 0 ? ` (${result.durationMs}ms)` : '';
-  process.stdout.write(`FAIL ${result.file}${attemptSuffix}${durationSuffix}\n`);
+  const durationSuffix = formatReplayTestDurationSuffix(result);
+  process.stdout.write(`FAIL ${replayTestDisplayName(result)}${attemptSuffix}${durationSuffix}\n`);
   process.stdout.write(`  ${result.error?.message ?? 'Unknown test failure'}\n`);
   for (const line of replayFailureConsoleLines(result)) {
     process.stdout.write(`  ${line}\n`);
   }
 }
 
 function replayResultPrefix(result: ReplaySuiteTestResult): string {
-  if (result.status === 'passed') return result.attempts > 1 ? 'FLAKY' : 'PASS';
+  if (result.status === 'passed') return 'PASS';
   if (result.status === 'skipped') return 'SKIP';
   return 'INFO';
 }
@@ -98,17 +96,64 @@ function replayFailureConsoleLines(
   ].filter(Boolean);
 }
 
-function renderFlakyTestResult(result: Extract<ReplaySuiteTestResult, { status: 'passed' }>): void {
-  const durationSuffix = result.durationMs > 0 ? ` (${result.durationMs}ms)` : '';
-  process.stdout.write(`FLAKY ${result.file} after ${result.attempts} attempts${durationSuffix}\n`);
-}
-
 function isFlakyReplayTestResult(
   result: ReplaySuiteTestResult,
 ): result is Extract<ReplaySuiteTestResult, { status: 'passed' }> {
   return result.status === 'passed' && result.attempts > 1;
 }
 
+function renderFlakyTestSummary(
+  results: Array<Extract<ReplaySuiteTestResult, { status: 'passed' }>>,
+): void {
+  if (results.length === 0) return;
+  process.stdout.write('Flaky tests:\n');
+  for (const result of results) {
+    process.stdout.write(
+      `  PASS ${replayTestDisplayName(result)} after ${result.attempts} attempts${formatFlakyReplayDurationSuffix(result)}\n`,
+    );
+    for (const failure of result.attemptFailures ?? []) {
+      const attemptDuration =
+        typeof failure.durationMs === 'number'
+          ? ` (${formatDurationSeconds(failure.durationMs)})`
+          : '';
+      process.stdout.write(
+        `    attempt ${failure.attempt} failed${attemptDuration}: ${failure.message}\n`,
+      );
+    }
+  }
+}
+
+function replayTestDisplayName(result: ReplaySuiteTestResult): string {
+  return path.basename(result.file);
+}
+
+function formatReplayTestDurationSuffix(result: ReplaySuiteTestResult): string {
+  if (result.status === 'passed' && result.attempts > 1) {
+    return formatFlakyReplayDurationSuffix(result);
+  }
+  if (result.status === 'failed' && result.attempts > 1 && result.durationMs > 0) {
+    return ` (total ${formatDurationSeconds(result.durationMs)})`;
+  }
+
+  const durationMs =
+    result.status === 'passed' && typeof result.finalAttemptDurationMs === 'number'
+      ? result.finalAttemptDurationMs
+      : result.durationMs;
+  return durationMs > 0 ? ` (${formatDurationSeconds(durationMs)})` : '';
+}
+
+function formatFlakyReplayDurationSuffix(
+  result: Extract<ReplaySuiteTestResult, { status: 'passed' }>,
+): string {
+  const timings = [
+    typeof result.finalAttemptDurationMs === 'number'
+      ? `passed attempt ${formatDurationSeconds(result.finalAttemptDurationMs)}`
+      : '',
+    result.durationMs > 0 ? `total ${formatDurationSeconds(result.durationMs)}` : '',
+  ].filter(Boolean);
+  return timings.length > 0 ? ` (${timings.join(', ')})` : '';
+}
+
 function getReplayTestExitCode(data: ReplaySuiteResult): number {
   return data.failed > 0 ? 1 : 0;
 }
@@ -144,7 +189,7 @@ function buildReplayJunitXml(suite: ReplaySuiteResult): string {
 }
 
 function renderJUnitTestCase(test: ReplaySuiteTestResult): string[] {
-  const name = xmlEscape(path.basename(test.file));
+  const name = xmlEscape(replayTestDisplayName(test));
   const className = xmlEscape(
     path.dirname(test.file) === '.' ? test.file : path.dirname(test.file),
   );
@@ -239,6 +284,13 @@ function formatJUnitSeconds(durationMs: number): string {
   return (Math.max(0, durationMs) / 1000).toFixed(3);
 }
 
+function formatDurationSeconds(durationMs: number): string {
+  const seconds = Math.max(0, durationMs) / 1000;
+  if (seconds >= 10) return `${seconds.toFixed(1)}s`;
+  if (seconds >= 1) return `${seconds.toFixed(2)}s`;
+  return `${seconds.toFixed(3).replace(/0+$/, '').replace(/\.$/, '')}s`;
+}
+
 function xmlEscape(value: string): string {
   return value
     .replaceAll('&', '&amp;')

src/daemon/handlers/__tests__/session-test-suite.test.ts

@@ -130,6 +130,13 @@ test('test emits progress when attempts retry and pass', async () => {
 
   const data = expectOkData(response);
   expect(data.passed).toBe(1);
+  expect((data.tests as Array<Record<string, unknown>>)[0]?.attemptFailures).toEqual([
+    {
+      attempt: 1,
+      message: 'Replay failed at step 1 (open "Demo"): first attempt failed',
+      durationMs: expect.any(Number),
+    },
+  ]);
   expect(events.map((event) => event.status)).toEqual(['fail', 'pass']);
   expect(events[0]).toMatchObject({
     type: 'replay-test',

src/daemon/handlers/session-test.ts

@@ -150,9 +150,14 @@ async function runReplayTestCase(
   let finalResponse: DaemonResponse | undefined;
   let finalSessionName = '';
   let attempts = 0;
+  let finalAttemptDurationMs = 0;
+  const attemptFailures: NonNullable<
+    Extract<ReplaySuiteTestResult, { status: 'passed' }>['attemptFailures']
+  > = [];
 
   for (let attemptIndex = 0; attemptIndex <= retries; attemptIndex += 1) {
     attempts = attemptIndex + 1;
+    const attemptStartedAt = Date.now();
     const testSessionName = buildReplayTestSessionName(
       sessionName,
       suiteInvocationId,
@@ -181,6 +186,7 @@ async function runReplayTestCase(
       runReplay,
       cleanupSession,
     });
+    finalAttemptDurationMs = Date.now() - attemptStartedAt;
     materializeReplayTestAttemptArtifacts({
       response,
       filePath: entry.path,
@@ -192,6 +198,11 @@ async function runReplayTestCase(
     finalResponse = response;
     finalSessionName = testSessionName;
     if (response.ok) break;
+    attemptFailures.push({
+      attempt: attempts,
+      message: response.error.message,
+      durationMs: finalAttemptDurationMs,
+    });
     if (isReplayInfrastructureFailure(response)) break;
     if (attemptIndex >= retries) break;
     emitRequestProgress({
@@ -225,10 +236,12 @@ async function runReplayTestCase(
       session: finalSessionName,
       status: 'passed',
       durationMs,
+      finalAttemptDurationMs,
       attempts,
       artifactsDir: testArtifactsDir,
       replayed: typeof finalResponse.data?.replayed === 'number' ? finalResponse.data.replayed : 0,
       healed: typeof finalResponse.data?.healed === 'number' ? finalResponse.data.healed : 0,
+      ...(attemptFailures.length > 0 ? { attemptFailures } : {}),
     };
   }
 

src/daemon/types.ts

@@ -41,10 +41,12 @@ export type ReplaySuiteTestPassed = {
   session: string;
   status: 'passed';
   durationMs: number;
+  finalAttemptDurationMs?: number;
   attempts: number;
   artifactsDir?: string;
   replayed: number;
   healed: number;
+  attemptFailures?: ReplaySuiteAttemptFailure[];
 };
 
 export type ReplaySuiteTestFailed = {
@@ -72,6 +74,12 @@ export type ReplaySuiteTestSkipped = {
   message: string;
 };
 
+export type ReplaySuiteAttemptFailure = {
+  attempt: number;
+  message: string;
+  durationMs?: number;
+};
+
 export type ReplaySuiteTestResult =
   | ReplaySuiteTestPassed
   | ReplaySuiteTestFailed