Skip to content

Commit ea58ba5

Browse files
thymikeethymikee
committed
feat: improve maestro test reporter
1 parent 0b499f2 commit ea58ba5

18 files changed

Lines changed: 827 additions & 251 deletions

‎src/__tests__/cli-network.test.ts‎

Lines changed: 58 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -106,11 +106,8 @@ test('test command prints suite summary and exits non-zero on failures', async (
106106
assert.equal(result.calls.length, 1);
107107
assert.equal(result.calls[0]?.meta?.requestProgress, 'replay-test');
108108
assert.doesNotMatch(result.stderr, /Running replay suite\.\.\./);
109-
assert.match(result.stdout, /PASS 01-pass\.ad \(0\.01s\)/);
110-
assert.match(
111-
result.stdout,
112-
/FAIL "Checkout failure" in 02-fail\.ad after 2 attempts \(total 0\.005s\)/,
113-
);
109+
assert.doesNotMatch(result.stdout, /✓ 01-pass\.ad \(0\.01s\)/);
110+
assert.doesNotMatch(result.stdout, /⨯ "Checkout failure" in 02-fail\.ad/);
114111
assert.match(result.stdout, /Replay failed at step 1 \(open Demo\): boom/);
115112
assert.match(result.stdout, /artifacts: \/tmp\/test-artifacts\/02-fail/);
116113
assert.doesNotMatch(result.stdout, /SKIP \/tmp\/03-skip\.ad/);
@@ -125,11 +122,12 @@ test('test command --verbose prints all test statuses', async () => {
125122
assert.equal(result.code, 1);
126123
assert.equal(result.calls[0]?.meta?.debug, false);
127124
assert.doesNotMatch(result.stderr, /Running replay suite\.\.\./);
128-
assert.match(result.stdout, /PASS 01-pass\.ad \(0\.01s\)/);
129-
assert.match(result.stdout, /SKIP 03-skip\.ad/);
125+
assert.doesNotMatch(result.stdout, /✓ 01-pass\.ad \(0\.01s\)/);
126+
assert.doesNotMatch(result.stdout, /SKIP 03-skip\.ad/);
127+
assert.match(result.stdout, /Test summary: 1 passed, 1 failed in 0\.025s/);
130128
});
131129

132-
test('test command --verbose prints step telemetry for passing tests without debug mode', async () => {
130+
test('test command --verbose omits step telemetry for passing tests without debug mode', async () => {
133131
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-cli-test-verbose-'));
134132
const artifactsDir = path.join(tmpDir, 'auth-flow');
135133
const attemptDir = path.join(artifactsDir, 'attempt-1');
@@ -203,16 +201,16 @@ test('test command --verbose prints step telemetry for passing tests without deb
203201

204202
assert.equal(result.code, null);
205203
assert.equal(result.calls[0]?.meta?.debug, false);
206-
assert.match(result.stdout, /PASS "Authentication flow" \(0\.5s\)/);
207-
assert.match(result.stdout, /steps:/);
208-
assert.match(result.stdout, /tapOn "text=\\"Log in\\"" \(line 3, 0\.25s\)/);
209-
assert.match(result.stdout, /assertVisible "text=\\"Home\\"" \(line 4, 0\.075s\)/);
204+
assert.doesNotMatch(result.stdout, /✓ "Authentication flow" in auth-flow\.yml \(0\.5s\)/);
205+
assert.doesNotMatch(result.stdout, /steps:/);
206+
assert.doesNotMatch(result.stdout, /tapOn "text=\\"Log in\\""/);
207+
assert.doesNotMatch(result.stdout, /assertVisible "text=\\"Home\\""/);
210208
} finally {
211209
await fs.rm(tmpDir, { recursive: true, force: true });
212210
}
213211
});
214212

215-
test('test command --verbose keeps nested retry and open step telemetry distinct', async () => {
213+
test('test command --verbose omits nested passing step telemetry', async () => {
216214
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-cli-test-verbose-retry-'));
217215
const artifactsDir = path.join(tmpDir, 'material-top-tabs');
218216
const attemptDir = path.join(artifactsDir, 'attempt-1');
@@ -300,15 +298,15 @@ test('test command --verbose keeps nested retry and open step telemetry distinct
300298
}));
301299

302300
assert.equal(result.code, null);
303-
assert.match(
301+
assert.doesNotMatch(
304302
result.stdout,
305303
/open "org\.reactnavigation\.playground" "rne:\/\/material-top-tabs-basic" \(line 4, 0\.727s\)/,
306304
);
307-
assert.match(
305+
assert.doesNotMatch(
308306
result.stdout,
309307
/assertVisible "label=\\"Chat\\" \|\| text=\\"Chat\\" \|\| id=\\"Chat\\"" "60000" \(line 4, 2\.58s\)/,
310308
);
311-
assert.match(result.stdout, /retry "3" \(line 4, 3\.31s\)/);
309+
assert.doesNotMatch(result.stdout, /retry "3" \(line 4, 3\.31s\)/);
312310
assert.doesNotMatch(
313311
result.stdout,
314312
/open "org\.reactnavigation\.playground" "rne:\/\/material-top-tabs-basic" \(line 4, 3\.31s\)/,
@@ -354,30 +352,45 @@ test('test command reports flaky passed-on-retry cases in the default summary',
354352
assert.equal(result.code, null);
355353
assert.doesNotMatch(result.stderr, /Running replay suite\.\.\./);
356354
assert.doesNotMatch(result.stdout, /FLAKY/);
357-
assert.match(
355+
assert.doesNotMatch(
358356
result.stdout,
359-
/PASS "Authentication flow" after 2 attempts \(passed attempt 17\.5s, total 112\.2s\)/,
357+
/^✓ "Authentication flow" in auth-flow\.yml \(passed attempt 17\.5s, total 112\.2s\)$/m,
360358
);
361359
assert.match(result.stdout, /Test summary: 1 passed, 0 failed, 1 flaky in 0\.025s/);
362360
assert.match(result.stdout, /Flaky tests:/);
363361
assert.match(
364362
result.stdout,
365-
/PASS "Authentication flow" after 2 attempts \(passed attempt 17\.5s, total 112\.2s\)/,
363+
/✓ "Authentication flow" in auth-flow\.yml after 2 attempts \(passed attempt 17\.5s, total 112\.2s\)/,
366364
);
367365
assert.match(
368366
result.stdout,
369367
/attempt 1 failed \(94\.7s\): Replay failed at step 3 \(tapOn "Log in"\): selector not found/,
370368
);
371369
});
372370

373-
test('test command prints failed attempt step telemetry when timing trace exists', async () => {
371+
test('test command --debug prints failed attempt step window when timing trace exists', async () => {
374372
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-cli-test-steps-'));
375373
const artifactsDir = path.join(tmpDir, 'checkout-flow');
376374
const attemptDir = path.join(artifactsDir, 'attempt-2');
377375
await fs.mkdir(attemptDir, { recursive: true });
378376
await fs.writeFile(
379377
path.join(attemptDir, 'replay-timing.ndjson'),
380378
[
379+
{
380+
type: 'replay_action_start',
381+
step: 0,
382+
line: 2,
383+
command: 'close',
384+
positionals: ['Demo'],
385+
},
386+
{
387+
type: 'replay_action_stop',
388+
step: 0,
389+
line: 2,
390+
command: 'close',
391+
ok: true,
392+
durationMs: 50,
393+
},
381394
{
382395
type: 'replay_action_start',
383396
step: 1,
@@ -406,6 +419,21 @@ test('test command prints failed attempt step telemetry when timing trace exists
406419
step: 2,
407420
line: 4,
408421
command: '__maestroTapOn',
422+
ok: true,
423+
durationMs: 200,
424+
},
425+
{
426+
type: 'replay_action_start',
427+
step: 3,
428+
line: 5,
429+
command: '__maestroAssertVisible',
430+
positionals: ['text="Receipt"', '3000'],
431+
},
432+
{
433+
type: 'replay_action_stop',
434+
step: 3,
435+
line: 5,
436+
command: '__maestroAssertVisible',
409437
ok: false,
410438
durationMs: 1500,
411439
errorCode: 'ASSERTION_FAILED',
@@ -426,10 +454,10 @@ test('test command prints failed attempt step telemetry when timing trace exists
426454
artifactsDir,
427455
error: {
428456
code: 'ASSERTION_FAILED',
429-
message: 'Replay failed at step 2 (click "Pay"): selector not found',
457+
message: 'Replay failed at step 3 (assertVisible "Receipt"): selector not found',
430458
},
431459
};
432-
const result = await runCliCapture(['test', './suite'], async () => ({
460+
const result = await runCliCapture(['test', './suite', '--debug'], async () => ({
433461
ok: true,
434462
data: {
435463
total: 1,
@@ -445,11 +473,18 @@ test('test command prints failed attempt step telemetry when timing trace exists
445473
}));
446474

447475
assert.equal(result.code, 1);
476+
assert.equal(result.calls[0]?.meta?.debug, true);
477+
assert.match(
478+
result.stdout,
479+
/Replay failed at step 3 \(assertVisible "Receipt"\): selector not found/,
480+
);
448481
assert.match(result.stdout, /steps \(attempt 2\):/);
482+
assert.doesNotMatch(result.stdout, /close "Demo" \(line 2, 0\.050s\)/);
449483
assert.match(result.stdout, /open "Demo" \(line 3, 0\.125s, timing \{"launchMs":100\}\)/);
484+
assert.match(result.stdout, /tapOn "text=\\"Pay\\"" \(line 4, 0\.2s\)/);
450485
assert.match(
451486
result.stdout,
452-
/\[FAIL\] tapOn "text=\\"Pay\\"" \(line 4, 1\.50s, ASSERTION_FAILED\)/,
487+
/\[FAIL\] assertVisible "text=\\"Receipt\\"" "3000" \(line 5, 1\.50s, ASSERTION_FAILED\)/,
453488
);
454489
} finally {
455490
await fs.rm(tmpDir, { recursive: true, force: true });

‎src/__tests__/cli-test-progress.test.ts‎

Lines changed: 108 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,19 @@ import assert from 'node:assert/strict';
33
import { formatReplayTestProgressEvent } from '../cli-test-progress.ts';
44
import type { RequestProgressEvent } from '../daemon/request-progress.ts';
55

6-
test('formatReplayTestProgressEvent renders replay suite start context', () => {
6+
function withStreamTty<T>(stream: NodeJS.WriteStream, isTTY: boolean, run: () => T): T {
7+
const descriptor = Object.getOwnPropertyDescriptor(stream, 'isTTY');
8+
const mutableStream = stream as unknown as Record<string, unknown>;
9+
try {
10+
Object.defineProperty(stream, 'isTTY', { configurable: true, value: isTTY });
11+
return run();
12+
} finally {
13+
if (descriptor) Object.defineProperty(stream, 'isTTY', descriptor);
14+
else delete mutableStream.isTTY;
15+
}
16+
}
17+
18+
test('formatReplayTestProgressEvent suppresses replay suite start context', () => {
719
const line = formatReplayTestProgressEvent({
820
type: 'replay-test-suite',
921
status: 'start',
@@ -15,17 +27,10 @@ test('formatReplayTestProgressEvent renders replay suite start context', () => {
1527
shardCount: 2,
1628
});
1729

18-
assert.equal(
19-
line,
20-
[
21-
'Running replay suite: 4 files',
22-
' sharding: split across 2 devices',
23-
' artifacts: /tmp/replay-suite',
24-
].join('\n'),
25-
);
30+
assert.equal(line, undefined);
2631
});
2732

28-
test('formatReplayTestProgressEvent renders replay test start context with shard metadata', () => {
33+
test('formatReplayTestProgressEvent suppresses replay test start context', () => {
2934
const line = formatReplayTestProgressEvent({
3035
type: 'replay-test',
3136
file: '/tmp/auth-flow.yml',
@@ -40,14 +45,7 @@ test('formatReplayTestProgressEvent renders replay test start context with shard
4045
deviceId: 'E140A942-965C-4A92-AC63-F3B23756BE02',
4146
});
4247

43-
assert.equal(
44-
line,
45-
[
46-
'[2/5] START "Authentication flow" in auth-flow.yml [shard 2/2 E140A942-965C-4A92-AC63-F3B23756BE02]',
47-
' session: maestro-test:test:suite:2:attempt-1',
48-
' artifacts: /tmp/replay-suite/auth-flow',
49-
].join('\n'),
50-
);
48+
assert.equal(line, undefined);
5149
});
5250

5351
test('formatReplayTestProgressEvent ignores unknown progress event types', () => {
@@ -72,7 +70,7 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases',
7270
maxAttempts: 2,
7371
durationMs: 12_345,
7472
},
75-
expected: /^\[1\/3] PASS 01-login\.ad after 2 attempts \(total 12\.3s\)$/,
73+
expected: /^✓ 01-login\.ad \(12\.3s\)$/,
7674
},
7775
{
7876
event: {
@@ -87,7 +85,7 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases',
8785
retrying: true,
8886
message: 'first attempt failed',
8987
},
90-
expected: /^\[2\/3] RETRY 02-checkout\.ad attempt 1\/2 \(1\.23s\)\n first attempt failed$/,
88+
expected: /^$/,
9189
},
9290
{
9391
event: {
@@ -104,7 +102,7 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases',
104102
artifactsDir: '/tmp/replay-suite/payment',
105103
},
106104
expected:
107-
/^\[3\/3] FAIL 03-payment\.ad after 2 attempts \(total 9\.88s\)\n assertVisible failed\n session: maestro-test:test:suite:3:attempt-2\n artifacts: \/tmp\/replay-suite\/payment$/,
105+
/^⨯ 03-payment\.ad \(9\.88s\)\n failed at: assertVisible failed\n session: maestro-test:test:suite:3:attempt-2\n artifacts: \/tmp\/replay-suite\/payment$/,
108106
},
109107
{
110108
event: {
@@ -115,11 +113,99 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases',
115113
total: 5,
116114
message: 'missing platform metadata for --platform ios',
117115
},
118-
expected: /^\[4\/5] SKIP 04-skip\.ad\n missing platform metadata for --platform ios$/,
116+
expected: /^- 04-skip\.ad\n missing platform metadata for --platform ios$/,
119117
},
120118
];
121119

122120
for (const { event, expected } of cases) {
123121
assert.match(formatReplayTestProgressEvent(event) ?? '', expected);
124122
}
125123
});
124+
125+
test('formatReplayTestProgressEvent colors stderr progress rows when stdout is piped', () => {
126+
const originalForceColor = process.env.FORCE_COLOR;
127+
const originalNoColor = process.env.NO_COLOR;
128+
delete process.env.FORCE_COLOR;
129+
delete process.env.NO_COLOR;
130+
try {
131+
const line = withStreamTty(process.stdout, false, () =>
132+
withStreamTty(process.stderr, true, () =>
133+
formatReplayTestProgressEvent({
134+
type: 'replay-test',
135+
file: '/tmp/01-pass.ad',
136+
status: 'pass',
137+
index: 1,
138+
total: 1,
139+
attempt: 1,
140+
durationMs: 10,
141+
}),
142+
),
143+
);
144+
145+
assert.equal(line, '\u001B[32m✓\u001B[39m 01-pass.ad (0.01s)');
146+
} finally {
147+
if (typeof originalForceColor === 'string') process.env.FORCE_COLOR = originalForceColor;
148+
else delete process.env.FORCE_COLOR;
149+
if (typeof originalNoColor === 'string') process.env.NO_COLOR = originalNoColor;
150+
else delete process.env.NO_COLOR;
151+
}
152+
});
153+
154+
test('formatReplayTestProgressEvent colors completed result markers when color is enabled', () => {
155+
const originalForceColor = process.env.FORCE_COLOR;
156+
const originalNoColor = process.env.NO_COLOR;
157+
process.env.FORCE_COLOR = '1';
158+
delete process.env.NO_COLOR;
159+
try {
160+
formatReplayTestProgressEvent({
161+
type: 'replay-test-suite',
162+
status: 'start',
163+
total: 3,
164+
runnable: 3,
165+
skipped: 0,
166+
artifactsDir: '/tmp/replay-suite',
167+
});
168+
assert.equal(
169+
formatReplayTestProgressEvent({
170+
type: 'replay-test',
171+
file: '/tmp/01-pass.ad',
172+
status: 'pass',
173+
index: 1,
174+
total: 3,
175+
attempt: 1,
176+
durationMs: 10,
177+
}),
178+
'\u001B[32m✓\u001B[39m 01-pass.ad (0.01s)',
179+
);
180+
assert.equal(
181+
formatReplayTestProgressEvent({
182+
type: 'replay-test',
183+
file: '/tmp/02-flaky.yml',
184+
title: 'Retry flow',
185+
status: 'pass',
186+
index: 2,
187+
total: 3,
188+
attempt: 2,
189+
durationMs: 30,
190+
}),
191+
'\u001B[33m✓\u001B[39m "Retry flow" in 02-flaky.yml (0.03s)',
192+
);
193+
const failedLine = formatReplayTestProgressEvent({
194+
type: 'replay-test',
195+
file: '/tmp/03-fail.ad',
196+
title: 'Checkout failure',
197+
status: 'fail',
198+
index: 3,
199+
total: 3,
200+
attempt: 1,
201+
durationMs: 5,
202+
message: 'boom',
203+
});
204+
assert.ok(failedLine?.startsWith('\u001B[31m⨯\u001B[39m "Checkout failure" in 03-fail.ad'));
205+
} finally {
206+
if (typeof originalForceColor === 'string') process.env.FORCE_COLOR = originalForceColor;
207+
else delete process.env.FORCE_COLOR;
208+
if (typeof originalNoColor === 'string') process.env.NO_COLOR = originalNoColor;
209+
else delete process.env.NO_COLOR;
210+
}
211+
});

0 commit comments

Comments
 (0)