fix: improve maestro test output (#652)

Author: thymikee

src/__tests__/cli-network.test.ts

@@ -106,7 +106,7 @@ test('test command prints suite summary and exits non-zero on failures', async (
   assert.equal(result.calls.length, 1);
   assert.equal(result.calls[0]?.meta?.requestProgress, 'replay-test');
   assert.match(result.stderr, /Running replay suite\.\.\./);
-  assert.doesNotMatch(result.stdout, /PASS \/tmp\/01-pass\.ad/);
+  assert.match(result.stdout, /PASS 01-pass\.ad \(0\.01s\)/);
   assert.match(
     result.stdout,
     /FAIL "Checkout failure" in 02-fail\.ad after 2 attempts \(total 0\.005s\)/,
@@ -204,9 +204,9 @@ test('test command --verbose prints step telemetry for passing tests without deb
     assert.equal(result.code, null);
     assert.equal(result.calls[0]?.meta?.debug, false);
     assert.match(result.stdout, /PASS "Authentication flow" \(0\.5s\)/);
-    assert.match(result.stdout, /steps \(attempt 1\):/);
-    assert.match(result.stdout, /\[ok\] tapOn "text=\\"Log in\\"" \(line 3, 0\.25s\)/);
-    assert.match(result.stdout, /\[ok\] assertVisible "text=\\"Home\\"" \(line 4, 0\.075s\)/);
+    assert.match(result.stdout, /steps:/);
+    assert.match(result.stdout, /tapOn "text=\\"Log in\\"" \(line 3, 0\.25s\)/);
+    assert.match(result.stdout, /assertVisible "text=\\"Home\\"" \(line 4, 0\.075s\)/);
   } finally {
     await fs.rm(tmpDir, { recursive: true, force: true });
   }
@@ -248,6 +248,10 @@ test('test command reports flaky passed-on-retry cases in the default summary',
   assert.equal(result.code, null);
   assert.match(result.stderr, /Running replay suite\.\.\./);
   assert.doesNotMatch(result.stdout, /FLAKY/);
+  assert.match(
+    result.stdout,
+    /PASS "Authentication flow" after 2 attempts \(passed attempt 17\.5s, total 112\.2s\)/,
+  );
   assert.match(result.stdout, /Test summary: 1 passed, 0 failed, 1 flaky in 0\.025s/);
   assert.match(result.stdout, /Flaky tests:/);
   assert.match(
@@ -336,10 +340,7 @@ test('test command prints failed attempt step telemetry when timing trace exists
 
     assert.equal(result.code, 1);
     assert.match(result.stdout, /steps \(attempt 2\):/);
-    assert.match(
-      result.stdout,
-      /\[ok\] open "Demo" \(line 3, 0\.125s, timing \{"launchMs":100\}\)/,
-    );
+    assert.match(result.stdout, /open "Demo" \(line 3, 0\.125s, timing \{"launchMs":100\}\)/);
     assert.match(
       result.stdout,
       /\[FAIL\] tapOn "text=\\"Pay\\"" \(line 4, 1\.50s, ASSERTION_FAILED\)/,
@@ -381,6 +382,7 @@ test('test --maestro forwards Maestro backend and platform for directory suites'
     assert.deepEqual(result.calls[0]?.positionals, [tmpDir]);
     assert.equal(result.calls[0]?.flags?.replayBackend, 'maestro');
     assert.equal(result.calls[0]?.flags?.platform, 'android');
+    assert.equal(result.calls[0]?.meta?.requestProgress, 'replay-test');
     assert.match(result.stderr, /Running replay suite\.\.\./);
   } finally {
     await fs.rm(tmpDir, { recursive: true, force: true });

src/cli-test.ts

@@ -37,8 +37,8 @@ function renderReplayTestSummary(
       renderVerboseTestResult(entry);
     }
   } else {
-    for (const entry of data.failures) {
-      renderFailedTestResult(entry);
+    for (const entry of data.tests) {
+      renderDefaultTestResult(entry);
     }
   }
 
@@ -52,6 +52,18 @@ function renderReplayTestSummary(
   return getReplayTestExitCode(data);
 }
 
+function renderDefaultTestResult(result: ReplaySuiteTestResult): void {
+  if (result.status === 'failed') {
+    renderFailedTestResult(result);
+    return;
+  }
+  if (result.status !== 'passed') return;
+
+  process.stdout.write(
+    `PASS ${replayTestDisplayName(result)}${formatReplayTestDurationSuffix(result)}\n`,
+  );
+}
+
 function renderVerboseTestResult(result: ReplaySuiteTestResult): void {
   if (result.status === 'failed') {
     renderFailedTestResult(result);
@@ -139,7 +151,7 @@ function replayTestStepLines(result: ReplaySuiteTestResult): string[] {
   if (stops.length === 0) return [];
 
   return [
-    `steps (attempt ${result.attempts}):`,
+    result.attempts > 1 ? `steps (attempt ${result.attempts}):` : 'steps:',
     ...stops.map((stop) => renderReplayStepTrace(stop, starts.get(stop.step))),
   ];
 }
@@ -211,8 +223,8 @@ function renderReplayStepTrace(
   start: ReplayActionStartTrace | undefined,
 ): string {
   const failed = stop.ok === false;
-  const status = failed ? '[FAIL]' : stop.ok === true ? '[ok]' : '[info]';
-  return `  ${status} ${formatReplayStepCommand(start, stop)}${formatReplayStepDetails(stop, start)}`;
+  const status = failed ? '[FAIL] ' : stop.ok === true ? '' : '[info] ';
+  return `  ${status}${formatReplayStepCommand(start, stop)}${formatReplayStepDetails(stop, start)}`;
 }
 
 function formatReplayStepDetails(

src/daemon/handlers/__tests__/session-test-suite.test.ts

@@ -141,16 +141,19 @@ test('test emits progress when attempts retry and pass', async () => {
   expect(events.map((event) => event.status)).toEqual(['fail', 'pass']);
   expect(events[0]).toMatchObject({
     type: 'replay-test',
+    title: undefined,
     status: 'fail',
     index: 1,
     total: 1,
     attempt: 1,
     maxAttempts: 2,
+    durationMs: expect.any(Number),
     retrying: true,
     message: 'Replay failed at step 1 (open "Demo"): first attempt failed',
   });
   expect(events[1]).toMatchObject({
     type: 'replay-test',
+    title: undefined,
     status: 'pass',
     index: 1,
     total: 1,

src/daemon/handlers/session-test.ts

@@ -208,11 +208,13 @@ async function runReplayTestCase(
     emitRequestProgress({
       type: 'replay-test',
       file: entry.path,
+      title: entry.title,
       status: 'fail',
       index: suiteIndex,
       total: suiteTotal,
       attempt: attempts,
       maxAttempts: retries + 1,
+      durationMs: finalAttemptDurationMs,
       retrying: true,
       message: response.error.message,
     });
@@ -223,6 +225,7 @@ async function runReplayTestCase(
     emitRequestProgress({
       type: 'replay-test',
       file: entry.path,
+      title: entry.title,
       status: 'pass',
       index: suiteIndex,
       total: suiteTotal,
@@ -255,6 +258,7 @@ async function runReplayTestCase(
   emitRequestProgress({
     type: 'replay-test',
     file: entry.path,
+    title: entry.title,
     status: 'fail',
     index: suiteIndex,
     total: suiteTotal,

src/daemon/request-progress-protocol.ts

@@ -1,3 +1,4 @@
+import path from 'node:path';
 import type { DaemonRequest, DaemonResponse } from './types.ts';
 import type { RequestProgressEvent } from './request-progress.ts';
 
@@ -49,19 +50,51 @@ export function serializeDaemonRpcResponseEnvelope(response: unknown): string {
 
 export function formatRequestProgressEvent(event: RequestProgressEvent): string | undefined {
   if (event.type !== 'replay-test') return undefined;
-  const parts = [event.status, `${event.index}/${event.total}`, event.file];
-  if (event.attempt !== undefined && event.maxAttempts !== undefined) {
-    parts.push(`attempt=${event.attempt}/${event.maxAttempts}`);
-  }
-  if (event.retrying) parts.push('retry=true');
-  if (event.durationMs !== undefined)
-    parts.push(`duration=${formatDurationSeconds(event.durationMs)}`);
-  if (event.artifactsDir && event.status === 'fail') parts.push(`artifacts=${event.artifactsDir}`);
+  const name = formatReplayTestProgressName(event);
+  const durationSuffix =
+    event.durationMs !== undefined ? ` (${formatReplayProgressDuration(event)})` : '';
+  const attemptSuffix = formatReplayProgressAttemptSuffix(event);
   const message = event.message?.replace(/\s+/g, ' ').trim();
-  if (message) parts.push(message);
-  return parts.join(' ');
+
+  if (event.status === 'pass') {
+    return `PASS ${name}${attemptSuffix}${durationSuffix}`;
+  }
+  if (event.status === 'skip') {
+    return [`SKIP ${name}`, message ? `  ${message}` : ''].filter(Boolean).join('\n');
+  }
+
+  return [
+    `FAIL ${name}${attemptSuffix}${durationSuffix}`,
+    message ? `  ${message}` : '',
+    event.artifactsDir ? `  artifacts: ${event.artifactsDir}` : '',
+  ]
+    .filter(Boolean)
+    .join('\n');
+}
+
+function formatReplayTestProgressName(event: RequestProgressEvent): string {
+  const title = event.title?.trim();
+  if (title) return JSON.stringify(title);
+  return path.basename(event.file);
+}
+
+function formatReplayProgressAttemptSuffix(event: RequestProgressEvent): string {
+  if (event.attempt === undefined) return '';
+  if (event.status === 'fail' && event.retrying && event.maxAttempts !== undefined) {
+    return ` attempt ${event.attempt}/${event.maxAttempts} retrying`;
+  }
+  if (event.attempt > 1) return ` after ${event.attempt} attempts`;
+  return '';
+}
+
+function formatReplayProgressDuration(event: RequestProgressEvent): string {
+  const duration = formatDurationSeconds(event.durationMs ?? 0);
+  return event.attempt && event.attempt > 1 && !event.retrying ? `total ${duration}` : duration;
 }
 
 function formatDurationSeconds(durationMs: number): string {
-  return `${(Math.max(0, durationMs) / 1000).toFixed(2)}s`;
+  const seconds = Math.max(0, durationMs) / 1000;
+  if (seconds >= 10) return `${seconds.toFixed(1)}s`;
+  if (seconds >= 1) return `${seconds.toFixed(2)}s`;
+  return `${seconds.toFixed(3).replace(/0+$/, '').replace(/\.$/, '')}s`;
 }

src/daemon/request-progress.ts

@@ -3,6 +3,7 @@ import { AsyncLocalStorage } from 'node:async_hooks';
 export type ReplayTestProgressEvent = {
   type: 'replay-test';
   file: string;
+  title?: string;
   status: 'pass' | 'fail' | 'skip';
   index: number;
   total: number;

src/utils/__tests__/daemon-client.test.ts

@@ -397,6 +397,7 @@ test('sendToDaemon prints replay test progress before the socket response', asyn
               event: {
                 type: 'replay-test',
                 file: '/tmp/01-login.ad',
+                title: 'Login flow',
                 status: 'fail',
                 index: 1,
                 total: 2,
@@ -440,10 +441,8 @@ test('sendToDaemon prints replay test progress before the socket response', asyn
     });
 
     assert.deepEqual(response, { ok: true, data: { via: 'socket' } });
-    assert.match(
-      stderr,
-      /fail 1\/2 \/tmp\/01-login\.ad attempt=1\/2 retry=true duration=1\.23s first attempt failed/,
-    );
+    assert.match(stderr, /FAIL "Login flow" attempt 1\/2 retrying \(1\.23s\)/);
+    assert.match(stderr, /  first attempt failed/);
   } finally {
     (net as unknown as { createConnection: typeof net.createConnection }).createConnection =
       originalCreateConnection;
@@ -498,6 +497,7 @@ test('sendToDaemon prints replay test progress before the HTTP NDJSON response',
             event: {
               type: 'replay-test',
               file: '/tmp/02-payments.ad',
+              title: 'Payments flow',
               status: 'pass',
               index: 2,
               total: 3,
@@ -543,7 +543,7 @@ test('sendToDaemon prints replay test progress before the HTTP NDJSON response',
       assert.deepEqual(response, { ok: true, data: { via: 'http-progress' } });
     });
     assert.deepEqual(seenPaths, ['GET /agent-device/health', 'POST /agent-device/rpc']);
-    assert.match(stderr, /pass 2\/3 \/tmp\/02-payments\.ad attempt=1\/1 duration=2\.50s/);
+    assert.match(stderr, /PASS "Payments flow" \(2\.50s\)/);
   } finally {
     (http as unknown as { request: typeof http.request }).request = originalHttpRequest;
     process.stderr.write = originalStderrWrite;