fix: improve Maestro test suite replay (#601)

Author: thymikee

src/__tests__/cli-network.test.ts

@@ -153,6 +153,44 @@ test('test command reports flaky passed-on-retry cases in the default summary',
   assert.match(result.stdout, /Test summary: 1 passed, 0 failed, 1 flaky in 25ms/);
 });
 
+test('test --maestro forwards Maestro backend and platform for directory suites', async () => {
+  const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-cli-maestro-suite-'));
+  await fs.writeFile(
+    path.join(tmpDir, 'auth-flow.yml'),
+    ['appId: demo.app', '---', '- launchApp', ''].join('\n'),
+  );
+
+  try {
+    const result = await runCliCapture(
+      ['test', '--maestro', '--platform', 'android', tmpDir],
+      async () => ({
+        ok: true,
+        data: {
+          total: 1,
+          executed: 1,
+          passed: 1,
+          failed: 0,
+          skipped: 0,
+          notRun: 0,
+          durationMs: 5,
+          failures: [],
+          tests: [],
+        },
+      }),
+    );
+
+    assert.equal(result.code, null);
+    assert.equal(result.calls.length, 1);
+    assert.equal(result.calls[0]?.command, 'test');
+    assert.deepEqual(result.calls[0]?.positionals, [tmpDir]);
+    assert.equal(result.calls[0]?.flags?.replayBackend, 'maestro');
+    assert.equal(result.calls[0]?.flags?.platform, 'android');
+    assert.match(result.stderr, /Running replay suite\.\.\./);
+  } finally {
+    await fs.rm(tmpDir, { recursive: true, force: true });
+  }
+});
+
 test('test command writes JUnit report with failure metadata', async () => {
   const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-junit-test-'));
   const reportPath = path.join(tmpDir, 'replays.junit.xml');
@@ -179,10 +217,12 @@ test('test command writes JUnit report with failure metadata', async () => {
               attempts: 2,
               artifactsDir: '/tmp/test-artifacts/02-fail',
               error: {
+                code: 'COMMAND_FAILED',
                 message: 'Replay failed at step 1 (open Demo): boom',
                 hint: 'retry me',
                 diagnosticId: 'diag-123',
                 logPath: '/tmp/diag.ndjson',
+                details: { command: 'open', reason: 'selector_not_found' },
               },
             },
           ],
@@ -204,10 +244,12 @@ test('test command writes JUnit report with failure metadata', async () => {
               attempts: 2,
               artifactsDir: '/tmp/test-artifacts/02-fail',
               error: {
+                code: 'COMMAND_FAILED',
                 message: 'Replay failed at step 1 (open Demo): boom',
                 hint: 'retry me',
                 diagnosticId: 'diag-123',
                 logPath: '/tmp/diag.ndjson',
+                details: { command: 'open', reason: 'selector_not_found' },
               },
             },
             {
@@ -236,6 +278,12 @@ test('test command writes JUnit report with failure metadata', async () => {
     assert.match(xml, /diagnosticId: diag-123/);
     assert.match(xml, /logPath: \/tmp\/diag\.ndjson/);
     assert.match(xml, /artifactsDir: \/tmp\/test-artifacts\/02-fail/);
+    assert.match(xml, /errorCode: COMMAND_FAILED/);
+    assert.match(xml, /errorMessage: Replay failed at step 1 \(open Demo\): boom/);
+    assert.match(
+      xml,
+      /details: \{"command":"open","reason":"selector_not_found"\}/,
+    );
     assert.match(xml, /flaky: true/);
     assert.match(xml, /<skipped message="not runnable" \/>/);
   } finally {

src/__tests__/client.test.ts

@@ -418,6 +418,45 @@ test('replay.test keeps backend alias for suite discovery', async () => {
   assert.equal(setup.calls[0]?.flags?.replayBackend, 'maestro');
 });
 
+test('structured replay.test command forwards Maestro backend for suite discovery', async () => {
+  const setup = createTransport(async () => ({ ok: true, data: {} }));
+  const client = createAgentDeviceClient(setup.config, { transport: setup.transport });
+
+  await runCommand(client, 'test', {
+    paths: ['./e2e/maestro'],
+    backend: 'maestro',
+    platform: 'android',
+  });
+
+  assert.equal(setup.calls.length, 1);
+  assert.equal(setup.calls[0]?.command, 'test');
+  assert.deepEqual(setup.calls[0]?.positionals, ['./e2e/maestro']);
+  assert.equal(setup.calls[0]?.flags?.replayBackend, 'maestro');
+  assert.equal(setup.calls[0]?.flags?.platform, 'android');
+});
+
+test('structured replay commands keep deprecated Maestro boolean alias', async () => {
+  const setup = createTransport(async () => ({ ok: true, data: {} }));
+  const client = createAgentDeviceClient(setup.config, { transport: setup.transport });
+
+  await runCommand(client, 'replay', {
+    path: './flows/login.yaml',
+    maestro: true,
+  });
+  await runCommand(client, 'test', {
+    paths: ['./e2e/maestro'],
+    maestro: true,
+    platform: 'android',
+  });
+
+  assert.equal(setup.calls.length, 2);
+  assert.equal(setup.calls[0]?.command, 'replay');
+  assert.equal(setup.calls[0]?.flags?.replayBackend, 'maestro');
+  assert.equal(setup.calls[1]?.command, 'test');
+  assert.equal(setup.calls[1]?.flags?.replayBackend, 'maestro');
+  assert.equal(setup.calls[1]?.flags?.platform, 'android');
+});
+
 test('client.command.wait prepares selector options and rejects invalid selectors', async () => {
   const setup = createTransport(async () => ({
     ok: true,

src/cli-test.ts

@@ -59,14 +59,7 @@ function renderVerboseTestResult(result: ReplaySuiteTestResult): void {
     return;
   }
 
-  const prefix =
-    result.status === 'passed'
-      ? isFlakyReplayTestResult(result)
-        ? 'FLAKY'
-        : 'PASS'
-      : result.status === 'skipped'
-        ? 'SKIP'
-        : 'INFO';
+  const prefix = replayResultPrefix(result);
   const attemptSuffix =
     'attempts' in result && result.attempts > 1 ? ` after ${result.attempts} attempts` : '';
   const durationSuffix = result.durationMs > 0 ? ` (${result.durationMs}ms)` : '';
@@ -83,14 +76,28 @@ function renderFailedTestResult(
   const durationSuffix = result.durationMs > 0 ? ` (${result.durationMs}ms)` : '';
   process.stdout.write(`FAIL ${result.file}${attemptSuffix}${durationSuffix}\n`);
   process.stdout.write(`  ${result.error?.message ?? 'Unknown test failure'}\n`);
-  if (result.error?.hint) process.stdout.write(`  hint: ${result.error.hint}\n`);
-  if (result.artifactsDir) process.stdout.write(`  artifacts: ${result.artifactsDir}\n`);
-  if (result.error?.logPath) process.stdout.write(`  log: ${result.error.logPath}\n`);
-  if (result.error?.diagnosticId) {
-    process.stdout.write(`  diagnostic: ${result.error.diagnosticId}\n`);
+  for (const line of replayFailureConsoleLines(result)) {
+    process.stdout.write(`  ${line}\n`);
   }
 }
 
+function replayResultPrefix(result: ReplaySuiteTestResult): string {
+  if (result.status === 'passed') return result.attempts > 1 ? 'FLAKY' : 'PASS';
+  if (result.status === 'skipped') return 'SKIP';
+  return 'INFO';
+}
+
+function replayFailureConsoleLines(
+  result: Extract<ReplaySuiteTestResult, { status: 'failed' }>,
+): string[] {
+  return [
+    result.error?.hint ? `hint: ${result.error.hint}` : '',
+    result.artifactsDir ? `artifacts: ${result.artifactsDir}` : '',
+    result.error?.logPath ? `log: ${result.error.logPath}` : '',
+    result.error?.diagnosticId ? `diagnostic: ${result.error.diagnosticId}` : '',
+  ].filter(Boolean);
+}
+
 function renderFlakyTestResult(result: Extract<ReplaySuiteTestResult, { status: 'passed' }>): void {
   const durationSuffix = result.durationMs > 0 ? ` (${result.durationMs}ms)` : '';
   process.stdout.write(`FLAKY ${result.file} after ${result.attempts} attempts${durationSuffix}\n`);
@@ -166,26 +173,68 @@ function renderJUnitTestCase(test: ReplaySuiteTestResult): string[] {
 
 function buildFailureDetails(test: Extract<ReplaySuiteTestResult, { status: 'failed' }>): string {
   const lines = [test.error.message];
-  if (test.error.hint) lines.push(`hint: ${test.error.hint}`);
-  if (test.error.diagnosticId) lines.push(`diagnosticId: ${test.error.diagnosticId}`);
-  if (test.error.logPath) lines.push(`logPath: ${test.error.logPath}`);
+  appendReplayErrorMetadata(lines, test.error, { includeDetails: false });
   if (test.artifactsDir) lines.push(`artifactsDir: ${test.artifactsDir}`);
-  const details = test.error.details ? JSON.stringify(test.error.details, null, 2) : undefined;
-  if (details) lines.push(`details: ${details}`);
+  appendReplayErrorDetails(lines, test.error, 2);
   return lines.join('\n');
 }
 
 function buildSystemOut(test: ReplaySuiteTestResult): string {
   const lines = [`status: ${test.status}`, `durationMs: ${test.durationMs}`];
-  if ('attempts' in test) lines.push(`attempts: ${test.attempts}`);
-  if ('session' in test) lines.push(`session: ${test.session}`);
-  if ('replayed' in test) lines.push(`replayed: ${test.replayed}`);
-  if ('healed' in test) lines.push(`healed: ${test.healed}`);
-  if ('artifactsDir' in test && test.artifactsDir) lines.push(`artifactsDir: ${test.artifactsDir}`);
-  if (test.status === 'passed' && test.attempts > 1) lines.push('flaky: true');
+  appendReplaySystemOutMetadata(lines, test);
   return lines.join('\n');
 }
 
+function appendReplaySystemOutMetadata(lines: string[], test: ReplaySuiteTestResult): void {
+  appendOptionalLine(lines, 'attempts' in test ? `attempts: ${test.attempts}` : undefined);
+  appendOptionalLine(lines, 'session' in test ? `session: ${test.session}` : undefined);
+  appendOptionalLine(lines, 'replayed' in test ? `replayed: ${test.replayed}` : undefined);
+  appendOptionalLine(lines, 'healed' in test ? `healed: ${test.healed}` : undefined);
+  appendOptionalLine(
+    lines,
+    'artifactsDir' in test && test.artifactsDir ? `artifactsDir: ${test.artifactsDir}` : undefined,
+  );
+  if (test.status === 'failed') {
+    appendReplayFailureSystemOut(lines, test);
+  }
+  appendOptionalLine(lines, isFlakyReplayTestResult(test) ? 'flaky: true' : undefined);
+}
+
+function appendReplayFailureSystemOut(
+  lines: string[],
+  test: Extract<ReplaySuiteTestResult, { status: 'failed' }>,
+): void {
+  lines.push(`errorCode: ${test.error.code}`);
+  appendReplayErrorMetadata(lines, test.error, { includeMessage: true });
+}
+
+function appendReplayErrorMetadata(
+  lines: string[],
+  error: Extract<ReplaySuiteTestResult, { status: 'failed' }>['error'],
+  options: { includeMessage?: boolean; includeDetails?: boolean; detailsIndent?: number } = {},
+): void {
+  if (options.includeMessage) lines.push(`errorMessage: ${error.message}`);
+  if (error.hint) lines.push(`hint: ${error.hint}`);
+  if (error.diagnosticId) lines.push(`diagnosticId: ${error.diagnosticId}`);
+  if (error.logPath) lines.push(`logPath: ${error.logPath}`);
+  if (options.includeDetails !== false) {
+    appendReplayErrorDetails(lines, error, options.detailsIndent);
+  }
+}
+
+function appendReplayErrorDetails(
+  lines: string[],
+  error: Extract<ReplaySuiteTestResult, { status: 'failed' }>['error'],
+  detailsIndent?: number,
+): void {
+  const details = error.details ? JSON.stringify(error.details, null, detailsIndent) : undefined;
+  if (details) lines.push(`details: ${details}`);
+}
+
+function appendOptionalLine(lines: string[], line: string | undefined): void {
+  if (line) lines.push(line);
+}
+
 function formatJUnitSeconds(durationMs: number): string {
   return (Math.max(0, durationMs) / 1000).toFixed(3);
 }

src/commands/client-command-contracts.ts

@@ -255,16 +255,19 @@ export const clientCommandDefinitions = [
       path: requiredField(stringField()),
       update: booleanField(),
       backend: stringField(),
+      maestro: booleanField(),
       env: stringArrayField(),
     },
     (client, input) => client.replay.run(input),
   ),
   defineFieldCommand(
     'test',
-    'Run one or more .ad scripts.',
+    'Run one or more replay scripts.',
     {
       paths: requiredField(stringArrayField()),
       update: booleanField(),
+      backend: stringField(),
+      maestro: booleanField(),
       env: stringArrayField(),
       failFast: booleanField(),
       timeoutMs: integerField(),