fix: improve maestro test output

Author: thymikee

src/__tests__/cli-network.test.ts

@@ -106,11 +106,11 @@ test('test command prints suite summary and exits non-zero on failures', async (
   assert.equal(result.calls[0]?.meta?.requestProgress, 'replay-test');
   assert.match(result.stderr, /Running replay suite\.\.\./);
   assert.doesNotMatch(result.stdout, /PASS \/tmp\/01-pass\.ad/);
-  assert.match(result.stdout, /FAIL \/tmp\/02-fail\.ad after 2 attempts \(5ms\)/);
+  assert.match(result.stdout, /FAIL 02-fail\.ad after 2 attempts \(0\.005s\)/);
   assert.match(result.stdout, /Replay failed at step 1 \(open Demo\): boom/);
   assert.match(result.stdout, /artifacts: \/tmp\/test-artifacts\/02-fail/);
   assert.doesNotMatch(result.stdout, /SKIP \/tmp\/03-skip\.ad/);
-  assert.match(result.stdout, /Test summary: 1 passed, 1 failed in 25ms/);
+  assert.match(result.stdout, /Test summary: 1 passed, 1 failed in 0\.025s/);
 });
 
 test('test command --verbose prints all test statuses', async () => {
@@ -120,8 +120,8 @@ test('test command --verbose prints all test statuses', async () => {
 
   assert.equal(result.code, 1);
   assert.match(result.stderr, /Running replay suite\.\.\./);
-  assert.match(result.stdout, /PASS \/tmp\/01-pass\.ad \(10ms\)/);
-  assert.match(result.stdout, /SKIP \/tmp\/03-skip\.ad/);
+  assert.match(result.stdout, /PASS 01-pass\.ad \(0\.01s\)/);
+  assert.match(result.stdout, /SKIP 03-skip\.ad/);
 });
 
 test('test command reports flaky passed-on-retry cases in the default summary', async () => {
@@ -138,20 +138,35 @@ test('test command reports flaky passed-on-retry cases in the default summary',
       failures: [],
       tests: [
         {
-          file: '/tmp/01-flaky.ad',
+          file: '/tmp/auth-flow.yml',
+          title: 'Authentication flow',
           session: 'default:test:suite:1',
           status: 'passed',
-          durationMs: 10,
+          durationMs: 112151,
+          finalAttemptDurationMs: 17492,
           attempts: 2,
+          attemptFailures: [
+            {
+              attempt: 1,
+              message: 'Replay failed at step 3 (tapOn "Log in"): selector not found',
+              durationMs: 94659,
+            },
+          ],
         },
       ],
     },
   }));
 
   assert.equal(result.code, null);
   assert.match(result.stderr, /Running replay suite\.\.\./);
-  assert.match(result.stdout, /FLAKY \/tmp\/01-flaky\.ad after 2 attempts \(10ms\)/);
-  assert.match(result.stdout, /Test summary: 1 passed, 0 failed, 1 flaky in 25ms/);
+  assert.doesNotMatch(result.stdout, /FLAKY/);
+  assert.match(result.stdout, /Test summary: 1 passed, 0 failed, 1 flaky in 0\.025s/);
+  assert.match(result.stdout, /Flaky tests:/);
+  assert.match(result.stdout, /PASS Authentication flow after 2 attempts, total 112\.2s/);
+  assert.match(
+    result.stdout,
+    /attempt 1 failed \(94\.7s\): Replay failed at step 3 \(tapOn "Log in"\): selector not found/,
+  );
 });
 
 test('test --maestro forwards Maestro backend and platform for directory suites', async () => {

src/cli-test.ts

@@ -40,16 +40,15 @@ function renderReplayTestSummary(
     for (const entry of data.failures) {
       renderFailedTestResult(entry);
     }
-    for (const entry of flaky) {
-      renderFlakyTestResult(entry);
-    }
   }
 
   const durationMs = typeof data.durationMs === 'number' ? data.durationMs : undefined;
   const flakySuffix = flaky.length > 0 ? `, ${flaky.length} flaky` : '';
+  const durationSuffix = durationMs !== undefined ? ` in ${formatDurationSeconds(durationMs)}` : '';
   process.stdout.write(
-    `Test summary: ${data.passed} passed, ${data.failed} failed${flakySuffix}${durationMs !== undefined ? ` in ${durationMs}ms` : ''}\n`,
+    `Test summary: ${data.passed} passed, ${data.failed} failed${flakySuffix}${durationSuffix}\n`,
   );
+  renderFlakyTestSummary(flaky);
   return getReplayTestExitCode(data);
 }
 
@@ -59,11 +58,10 @@ function renderVerboseTestResult(result: ReplaySuiteTestResult): void {
     return;
   }
 
-  const prefix = replayResultPrefix(result);
-  const attemptSuffix =
-    'attempts' in result && result.attempts > 1 ? ` after ${result.attempts} attempts` : '';
-  const durationSuffix = result.durationMs > 0 ? ` (${result.durationMs}ms)` : '';
-  process.stdout.write(`${prefix} ${result.file}${attemptSuffix}${durationSuffix}\n`);
+  const durationSuffix = formatReplayTestDurationSuffix(result);
+  process.stdout.write(
+    `${replayResultPrefix(result)} ${replayTestDisplayName(result)}${durationSuffix}\n`,
+  );
   if (result.status === 'skipped') {
     process.stdout.write(`  ${result.message ?? 'skipped'}\n`);
   }
@@ -73,16 +71,16 @@ function renderFailedTestResult(
   result: Extract<ReplaySuiteTestResult, { status: 'failed' }>,
 ): void {
   const attemptSuffix = result.attempts > 1 ? ` after ${result.attempts} attempts` : '';
-  const durationSuffix = result.durationMs > 0 ? ` (${result.durationMs}ms)` : '';
-  process.stdout.write(`FAIL ${result.file}${attemptSuffix}${durationSuffix}\n`);
+  const durationSuffix = formatReplayTestDurationSuffix(result);
+  process.stdout.write(`FAIL ${replayTestDisplayName(result)}${attemptSuffix}${durationSuffix}\n`);
   process.stdout.write(`  ${result.error?.message ?? 'Unknown test failure'}\n`);
   for (const line of replayFailureConsoleLines(result)) {
     process.stdout.write(`  ${line}\n`);
   }
 }
 
 function replayResultPrefix(result: ReplaySuiteTestResult): string {
-  if (result.status === 'passed') return result.attempts > 1 ? 'FLAKY' : 'PASS';
+  if (result.status === 'passed') return 'PASS';
   if (result.status === 'skipped') return 'SKIP';
   return 'INFO';
 }
@@ -98,17 +96,48 @@ function replayFailureConsoleLines(
   ].filter(Boolean);
 }
 
-function renderFlakyTestResult(result: Extract<ReplaySuiteTestResult, { status: 'passed' }>): void {
-  const durationSuffix = result.durationMs > 0 ? ` (${result.durationMs}ms)` : '';
-  process.stdout.write(`FLAKY ${result.file} after ${result.attempts} attempts${durationSuffix}\n`);
-}
-
 function isFlakyReplayTestResult(
   result: ReplaySuiteTestResult,
 ): result is Extract<ReplaySuiteTestResult, { status: 'passed' }> {
   return result.status === 'passed' && result.attempts > 1;
 }
 
+function renderFlakyTestSummary(
+  results: Array<Extract<ReplaySuiteTestResult, { status: 'passed' }>>,
+): void {
+  if (results.length === 0) return;
+  process.stdout.write('Flaky tests:\n');
+  for (const result of results) {
+    const durationSuffix =
+      result.durationMs > 0 ? `, total ${formatDurationSeconds(result.durationMs)}` : '';
+    process.stdout.write(
+      `  PASS ${replayTestDisplayName(result)} after ${result.attempts} attempts${durationSuffix}\n`,
+    );
+    for (const failure of result.attemptFailures ?? []) {
+      const attemptDuration =
+        typeof failure.durationMs === 'number'
+          ? ` (${formatDurationSeconds(failure.durationMs)})`
+          : '';
+      process.stdout.write(
+        `    attempt ${failure.attempt} failed${attemptDuration}: ${failure.message}\n`,
+      );
+    }
+  }
+}
+
+function replayTestDisplayName(result: ReplaySuiteTestResult): string {
+  const title = result.title?.trim();
+  return title && title.length > 0 ? title : path.basename(result.file);
+}
+
+function formatReplayTestDurationSuffix(result: ReplaySuiteTestResult): string {
+  const durationMs =
+    result.status === 'passed' && typeof result.finalAttemptDurationMs === 'number'
+      ? result.finalAttemptDurationMs
+      : result.durationMs;
+  return durationMs > 0 ? ` (${formatDurationSeconds(durationMs)})` : '';
+}
+
 function getReplayTestExitCode(data: ReplaySuiteResult): number {
   return data.failed > 0 ? 1 : 0;
 }
@@ -144,7 +173,7 @@ function buildReplayJunitXml(suite: ReplaySuiteResult): string {
 }
 
 function renderJUnitTestCase(test: ReplaySuiteTestResult): string[] {
-  const name = xmlEscape(path.basename(test.file));
+  const name = xmlEscape(replayTestDisplayName(test));
   const className = xmlEscape(
     path.dirname(test.file) === '.' ? test.file : path.dirname(test.file),
   );
@@ -239,6 +268,13 @@ function formatJUnitSeconds(durationMs: number): string {
   return (Math.max(0, durationMs) / 1000).toFixed(3);
 }
 
+function formatDurationSeconds(durationMs: number): string {
+  const seconds = Math.max(0, durationMs) / 1000;
+  if (seconds >= 10) return `${seconds.toFixed(1)}s`;
+  if (seconds >= 1) return `${seconds.toFixed(2)}s`;
+  return `${seconds.toFixed(3).replace(/0+$/, '').replace(/\.$/, '')}s`;
+}
+
 function xmlEscape(value: string): string {
   return value
     .replaceAll('&', '&amp;')

src/compat/maestro/replay-flow.ts

@@ -21,6 +21,12 @@ export function parseMaestroReplayFlow(
   return parseMaestroReplayFlowInternal(script, createParseContext(options));
 }
 
+export function readMaestroFlowName(script: string): string | undefined {
+  const values = parseYamlDocuments(script);
+  const { config } = splitMaestroDocuments(values);
+  return config.name;
+}
+
 function parseMaestroReplayFlowInternal(
   script: string,
   context: MaestroParseContext,
@@ -259,6 +265,7 @@ function normalizeConfig(value: unknown): MaestroFlowConfig {
     throw new AppError('INVALID_ARGS', 'Maestro flow config must be a YAML map.');
   }
   return {
+    ...(typeof value.name === 'string' && value.name.length > 0 ? { name: value.name } : {}),
     ...(typeof value.appId === 'string' && value.appId.length > 0 ? { appId: value.appId } : {}),
     ...(isPlainRecord(value.env) ? { env: readEnvMap(value.env, 'env') } : {}),
     ...(Array.isArray(value.onFlowStart)

src/compat/maestro/types.ts

@@ -1,6 +1,7 @@
 import type { ParsedReplayScript, ReplayScriptMetadata } from '../../replay/script.ts';
 
 export type MaestroFlowConfig = {
+  name?: string;
   appId?: string;
   env?: Record<string, string>;
   onFlowStart?: MaestroCommand[];

src/daemon/handlers/__tests__/session-test-discovery.test.ts

@@ -57,7 +57,10 @@ test('discoverReplayTestEntries rejects empty post-filter suites', () => {
 
 test('discoverReplayTestEntries includes Maestro yaml flows for Maestro test suites', () => {
   const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-discovery-maestro-'));
-  fs.writeFileSync(path.join(root, '01-flow.yaml'), 'appId: demo\n---\n- launchApp\n');
+  fs.writeFileSync(
+    path.join(root, '01-flow.yaml'),
+    'appId: demo\nname: Authentication flow\n---\n- launchApp\n',
+  );
   fs.writeFileSync(path.join(root, '02-flow.yml'), 'appId: demo\n---\n- launchApp\n');
   fs.writeFileSync(path.join(root, '03-flow.ad'), 'open "Demo"\n');
 
@@ -76,4 +79,8 @@ test('discoverReplayTestEntries includes Maestro yaml flows for Maestro test sui
     entries.map((entry) => entry.kind),
     ['run', 'run', 'run'],
   );
+  assert.equal(entries[0]?.kind, 'run');
+  if (entries[0]?.kind === 'run') {
+    assert.equal(entries[0].title, 'Authentication flow');
+  }
 });

src/daemon/handlers/__tests__/session-test-suite.test.ts

@@ -65,7 +65,7 @@ test('test discovers Maestro YAML suites when replay backend is set', async () =
   const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-suite-maestro-'));
   fs.writeFileSync(
     path.join(root, 'auth-flow.yml'),
-    ['appId: demo.app', '---', '- launchApp', ''].join('\n'),
+    ['appId: demo.app', 'name: Authentication flow', '---', '- launchApp', ''].join('\n'),
   );
 
   const invoked: DaemonRequest[] = [];
@@ -91,6 +91,7 @@ test('test discovers Maestro YAML suites when replay backend is set', async () =
   expect(invoked.map((req) => [req.command, req.positionals])).toEqual([['open', ['demo.app']]]);
   expect(data.passed).toBe(1);
   expect(data.failed).toBe(0);
+  expect((data.tests as Array<Record<string, unknown>>)[0]?.title).toBe('Authentication flow');
 });
 
 test('test emits progress when attempts retry and pass', async () => {
@@ -130,6 +131,13 @@ test('test emits progress when attempts retry and pass', async () => {
 
   const data = expectOkData(response);
   expect(data.passed).toBe(1);
+  expect((data.tests as Array<Record<string, unknown>>)[0]?.attemptFailures).toEqual([
+    {
+      attempt: 1,
+      message: 'Replay failed at step 1 (open "Demo"): first attempt failed',
+      durationMs: expect.any(Number),
+    },
+  ]);
   expect(events.map((event) => event.status)).toEqual(['fail', 'pass']);
   expect(events[0]).toMatchObject({
     type: 'replay-test',

src/daemon/handlers/session-test-discovery.ts

@@ -5,6 +5,7 @@ import type { PlatformSelector } from '../../utils/device.ts';
 import { resolveRequestTrackingId } from '../request-cancel.ts';
 import { SessionStore } from '../session-store.ts';
 import { readReplayScriptMetadata, type ReplayScriptMetadata } from '../../replay/script.ts';
+import { readMaestroFlowName } from '../../compat/maestro/replay-flow.ts';
 
 const GLOB_PATTERN_CHARS = /[*?[\]{}]/;
 
@@ -14,6 +15,7 @@ export type ReplayTestDiscoveryEntry =
   | {
       kind: 'run';
       path: string;
+      title?: string;
       metadata: ReplayScriptMetadata;
     }
   | {
@@ -42,13 +44,14 @@ export function discoverReplayTestEntries(params: {
   for (const filePath of filePaths) {
     const script = fs.readFileSync(filePath, 'utf8');
     const metadata = readReplayScriptMetadata(script);
+    const title = readReplayTestTitle(script, filePath, replayBackend);
     if (!platformFilter) {
-      entries.push({ kind: 'run', path: filePath, metadata });
+      entries.push({ kind: 'run', path: filePath, title, metadata });
       continue;
     }
     if (!metadata.platform) {
       if (isMaestroReplayBackend(replayBackend)) {
-        entries.push({ kind: 'run', path: filePath, metadata });
+        entries.push({ kind: 'run', path: filePath, title, metadata });
       } else {
         entries.push({
           kind: 'skip',
@@ -62,7 +65,7 @@ export function discoverReplayTestEntries(params: {
     if (!matchesPlatformFilter(platformFilter, metadata.platform)) {
       continue;
     }
-    entries.push({ kind: 'run', path: filePath, metadata });
+    entries.push({ kind: 'run', path: filePath, title, metadata });
   }
 
   const runnableCount = entries.filter((entry) => entry.kind === 'run').length;
@@ -177,6 +180,16 @@ function isMaestroReplayBackend(replayBackend: string | undefined): boolean {
   return replayBackend === 'maestro';
 }
 
+function readReplayTestTitle(
+  script: string,
+  filePath: string,
+  replayBackend: string | undefined,
+): string | undefined {
+  return isMaestroReplayBackend(replayBackend) && path.extname(filePath) !== '.ad'
+    ? readMaestroFlowName(script)
+    : undefined;
+}
+
 function looksLikeGlob(value: string): boolean {
   return GLOB_PATTERN_CHARS.test(value);
 }

src/daemon/handlers/session-test.ts

@@ -150,9 +150,14 @@ async function runReplayTestCase(
   let finalResponse: DaemonResponse | undefined;
   let finalSessionName = '';
   let attempts = 0;
+  let finalAttemptDurationMs = 0;
+  const attemptFailures: NonNullable<
+    Extract<ReplaySuiteTestResult, { status: 'passed' }>['attemptFailures']
+  > = [];
 
   for (let attemptIndex = 0; attemptIndex <= retries; attemptIndex += 1) {
     attempts = attemptIndex + 1;
+    const attemptStartedAt = Date.now();
     const testSessionName = buildReplayTestSessionName(
       sessionName,
       suiteInvocationId,
@@ -181,6 +186,7 @@ async function runReplayTestCase(
       runReplay,
       cleanupSession,
     });
+    finalAttemptDurationMs = Date.now() - attemptStartedAt;
     materializeReplayTestAttemptArtifacts({
       response,
       filePath: entry.path,
@@ -192,6 +198,11 @@ async function runReplayTestCase(
     finalResponse = response;
     finalSessionName = testSessionName;
     if (response.ok) break;
+    attemptFailures.push({
+      attempt: attempts,
+      message: response.error.message,
+      durationMs: finalAttemptDurationMs,
+    });
     if (isReplayInfrastructureFailure(response)) break;
     if (attemptIndex >= retries) break;
     emitRequestProgress({
@@ -222,13 +233,16 @@ async function runReplayTestCase(
     });
     return {
       file: entry.path,
+      title: entry.title,
       session: finalSessionName,
       status: 'passed',
       durationMs,
+      finalAttemptDurationMs,
       attempts,
       artifactsDir: testArtifactsDir,
       replayed: typeof finalResponse.data?.replayed === 'number' ? finalResponse.data.replayed : 0,
       healed: typeof finalResponse.data?.healed === 'number' ? finalResponse.data.healed : 0,
+      ...(attemptFailures.length > 0 ? { attemptFailures } : {}),
     };
   }
 
@@ -252,6 +266,7 @@ async function runReplayTestCase(
   });
   return {
     file: entry.path,
+    title: entry.title,
     session: finalSessionName,
     status: 'failed',
     durationMs,