Skip to content

Commit 05eeed5

Browse files
thymikeethymikee
committed
feat: improve maestro test reporter
1 parent 0b499f2 commit 05eeed5

18 files changed

Lines changed: 829 additions & 181 deletions

‎src/__tests__/cli-network.test.ts‎

Lines changed: 58 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -106,11 +106,8 @@ test('test command prints suite summary and exits non-zero on failures', async (
106106
assert.equal(result.calls.length, 1);
107107
assert.equal(result.calls[0]?.meta?.requestProgress, 'replay-test');
108108
assert.doesNotMatch(result.stderr, /Running replay suite\.\.\./);
109-
assert.match(result.stdout, /PASS 01-pass\.ad \(0\.01s\)/);
110-
assert.match(
111-
result.stdout,
112-
/FAIL "Checkout failure" in 02-fail\.ad after 2 attempts \(total 0\.005s\)/,
113-
);
109+
assert.doesNotMatch(result.stdout, /✓ 01-pass\.ad \(0\.01s\)/);
110+
assert.doesNotMatch(result.stdout, /⨯ "Checkout failure" in 02-fail\.ad/);
114111
assert.match(result.stdout, /Replay failed at step 1 \(open Demo\): boom/);
115112
assert.match(result.stdout, /artifacts: \/tmp\/test-artifacts\/02-fail/);
116113
assert.doesNotMatch(result.stdout, /SKIP \/tmp\/03-skip\.ad/);
@@ -125,11 +122,12 @@ test('test command --verbose prints all test statuses', async () => {
125122
assert.equal(result.code, 1);
126123
assert.equal(result.calls[0]?.meta?.debug, false);
127124
assert.doesNotMatch(result.stderr, /Running replay suite\.\.\./);
128-
assert.match(result.stdout, /PASS 01-pass\.ad \(0\.01s\)/);
129-
assert.match(result.stdout, /SKIP 03-skip\.ad/);
125+
assert.doesNotMatch(result.stdout, /✓ 01-pass\.ad \(0\.01s\)/);
126+
assert.doesNotMatch(result.stdout, /SKIP 03-skip\.ad/);
127+
assert.match(result.stdout, /Test summary: 1 passed, 1 failed in 0\.025s/);
130128
});
131129

132-
test('test command --verbose prints step telemetry for passing tests without debug mode', async () => {
130+
test('test command --verbose omits step telemetry for passing tests without debug mode', async () => {
133131
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-cli-test-verbose-'));
134132
const artifactsDir = path.join(tmpDir, 'auth-flow');
135133
const attemptDir = path.join(artifactsDir, 'attempt-1');
@@ -203,16 +201,16 @@ test('test command --verbose prints step telemetry for passing tests without deb
203201

204202
assert.equal(result.code, null);
205203
assert.equal(result.calls[0]?.meta?.debug, false);
206-
assert.match(result.stdout, /PASS "Authentication flow" \(0\.5s\)/);
207-
assert.match(result.stdout, /steps:/);
208-
assert.match(result.stdout, /tapOn "text=\\"Log in\\"" \(line 3, 0\.25s\)/);
209-
assert.match(result.stdout, /assertVisible "text=\\"Home\\"" \(line 4, 0\.075s\)/);
204+
assert.doesNotMatch(result.stdout, /✓ "Authentication flow" in auth-flow\.yml \(0\.5s\)/);
205+
assert.doesNotMatch(result.stdout, /steps:/);
206+
assert.doesNotMatch(result.stdout, /tapOn "text=\\"Log in\\""/);
207+
assert.doesNotMatch(result.stdout, /assertVisible "text=\\"Home\\""/);
210208
} finally {
211209
await fs.rm(tmpDir, { recursive: true, force: true });
212210
}
213211
});
214212

215-
test('test command --verbose keeps nested retry and open step telemetry distinct', async () => {
213+
test('test command --verbose omits nested passing step telemetry', async () => {
216214
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-cli-test-verbose-retry-'));
217215
const artifactsDir = path.join(tmpDir, 'material-top-tabs');
218216
const attemptDir = path.join(artifactsDir, 'attempt-1');
@@ -300,15 +298,15 @@ test('test command --verbose keeps nested retry and open step telemetry distinct
300298
}));
301299

302300
assert.equal(result.code, null);
303-
assert.match(
301+
assert.doesNotMatch(
304302
result.stdout,
305303
/open "org\.reactnavigation\.playground" "rne:\/\/material-top-tabs-basic" \(line 4, 0\.727s\)/,
306304
);
307-
assert.match(
305+
assert.doesNotMatch(
308306
result.stdout,
309307
/assertVisible "label=\\"Chat\\" \|\| text=\\"Chat\\" \|\| id=\\"Chat\\"" "60000" \(line 4, 2\.58s\)/,
310308
);
311-
assert.match(result.stdout, /retry "3" \(line 4, 3\.31s\)/);
309+
assert.doesNotMatch(result.stdout, /retry "3" \(line 4, 3\.31s\)/);
312310
assert.doesNotMatch(
313311
result.stdout,
314312
/open "org\.reactnavigation\.playground" "rne:\/\/material-top-tabs-basic" \(line 4, 3\.31s\)/,
@@ -354,30 +352,45 @@ test('test command reports flaky passed-on-retry cases in the default summary',
354352
assert.equal(result.code, null);
355353
assert.doesNotMatch(result.stderr, /Running replay suite\.\.\./);
356354
assert.doesNotMatch(result.stdout, /FLAKY/);
357-
assert.match(
355+
assert.doesNotMatch(
358356
result.stdout,
359-
/PASS "Authentication flow" after 2 attempts \(passed attempt 17\.5s, total 112\.2s\)/,
357+
/^✓ "Authentication flow" in auth-flow\.yml \(passed attempt 17\.5s, total 112\.2s\)$/m,
360358
);
361359
assert.match(result.stdout, /Test summary: 1 passed, 0 failed, 1 flaky in 0\.025s/);
362360
assert.match(result.stdout, /Flaky tests:/);
363361
assert.match(
364362
result.stdout,
365-
/PASS "Authentication flow" after 2 attempts \(passed attempt 17\.5s, total 112\.2s\)/,
363+
/✓ "Authentication flow" in auth-flow\.yml after 2 attempts \(passed attempt 17\.5s, total 112\.2s\)/,
366364
);
367365
assert.match(
368366
result.stdout,
369367
/attempt 1 failed \(94\.7s\): Replay failed at step 3 \(tapOn "Log in"\): selector not found/,
370368
);
371369
});
372370

373-
test('test command prints failed attempt step telemetry when timing trace exists', async () => {
371+
test('test command --debug prints failed attempt step window when timing trace exists', async () => {
374372
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-cli-test-steps-'));
375373
const artifactsDir = path.join(tmpDir, 'checkout-flow');
376374
const attemptDir = path.join(artifactsDir, 'attempt-2');
377375
await fs.mkdir(attemptDir, { recursive: true });
378376
await fs.writeFile(
379377
path.join(attemptDir, 'replay-timing.ndjson'),
380378
[
379+
{
380+
type: 'replay_action_start',
381+
step: 0,
382+
line: 2,
383+
command: 'close',
384+
positionals: ['Demo'],
385+
},
386+
{
387+
type: 'replay_action_stop',
388+
step: 0,
389+
line: 2,
390+
command: 'close',
391+
ok: true,
392+
durationMs: 50,
393+
},
381394
{
382395
type: 'replay_action_start',
383396
step: 1,
@@ -406,6 +419,21 @@ test('test command prints failed attempt step telemetry when timing trace exists
406419
step: 2,
407420
line: 4,
408421
command: '__maestroTapOn',
422+
ok: true,
423+
durationMs: 200,
424+
},
425+
{
426+
type: 'replay_action_start',
427+
step: 3,
428+
line: 5,
429+
command: '__maestroAssertVisible',
430+
positionals: ['text="Receipt"', '3000'],
431+
},
432+
{
433+
type: 'replay_action_stop',
434+
step: 3,
435+
line: 5,
436+
command: '__maestroAssertVisible',
409437
ok: false,
410438
durationMs: 1500,
411439
errorCode: 'ASSERTION_FAILED',
@@ -426,10 +454,10 @@ test('test command prints failed attempt step telemetry when timing trace exists
426454
artifactsDir,
427455
error: {
428456
code: 'ASSERTION_FAILED',
429-
message: 'Replay failed at step 2 (click "Pay"): selector not found',
457+
message: 'Replay failed at step 3 (assertVisible "Receipt"): selector not found',
430458
},
431459
};
432-
const result = await runCliCapture(['test', './suite'], async () => ({
460+
const result = await runCliCapture(['test', './suite', '--debug'], async () => ({
433461
ok: true,
434462
data: {
435463
total: 1,
@@ -445,11 +473,18 @@ test('test command prints failed attempt step telemetry when timing trace exists
445473
}));
446474

447475
assert.equal(result.code, 1);
476+
assert.equal(result.calls[0]?.meta?.debug, true);
477+
assert.match(
478+
result.stdout,
479+
/Replay failed at step 3 \(assertVisible "Receipt"\): selector not found/,
480+
);
448481
assert.match(result.stdout, /steps \(attempt 2\):/);
482+
assert.doesNotMatch(result.stdout, /close "Demo" \(line 2, 0\.050s\)/);
449483
assert.match(result.stdout, /open "Demo" \(line 3, 0\.125s, timing \{"launchMs":100\}\)/);
484+
assert.match(result.stdout, /tapOn "text=\\"Pay\\"" \(line 4, 0\.2s\)/);
450485
assert.match(
451486
result.stdout,
452-
/\[FAIL\] tapOn "text=\\"Pay\\"" \(line 4, 1\.50s, ASSERTION_FAILED\)/,
487+
/\[FAIL\] assertVisible "text=\\"Receipt\\"" "3000" \(line 5, 1\.50s, ASSERTION_FAILED\)/,
453488
);
454489
} finally {
455490
await fs.rm(tmpDir, { recursive: true, force: true });

‎src/__tests__/cli-test-progress.test.ts‎

Lines changed: 67 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import assert from 'node:assert/strict';
33
import { formatReplayTestProgressEvent } from '../cli-test-progress.ts';
44
import type { RequestProgressEvent } from '../daemon/request-progress.ts';
55

6-
test('formatReplayTestProgressEvent renders replay suite start context', () => {
6+
test('formatReplayTestProgressEvent suppresses replay suite start context', () => {
77
const line = formatReplayTestProgressEvent({
88
type: 'replay-test-suite',
99
status: 'start',
@@ -15,17 +15,10 @@ test('formatReplayTestProgressEvent renders replay suite start context', () => {
1515
shardCount: 2,
1616
});
1717

18-
assert.equal(
19-
line,
20-
[
21-
'Running replay suite: 4 files',
22-
' sharding: split across 2 devices',
23-
' artifacts: /tmp/replay-suite',
24-
].join('\n'),
25-
);
18+
assert.equal(line, undefined);
2619
});
2720

28-
test('formatReplayTestProgressEvent renders replay test start context with shard metadata', () => {
21+
test('formatReplayTestProgressEvent suppresses replay test start context', () => {
2922
const line = formatReplayTestProgressEvent({
3023
type: 'replay-test',
3124
file: '/tmp/auth-flow.yml',
@@ -40,14 +33,7 @@ test('formatReplayTestProgressEvent renders replay test start context with shard
4033
deviceId: 'E140A942-965C-4A92-AC63-F3B23756BE02',
4134
});
4235

43-
assert.equal(
44-
line,
45-
[
46-
'[2/5] START "Authentication flow" in auth-flow.yml [shard 2/2 E140A942-965C-4A92-AC63-F3B23756BE02]',
47-
' session: maestro-test:test:suite:2:attempt-1',
48-
' artifacts: /tmp/replay-suite/auth-flow',
49-
].join('\n'),
50-
);
36+
assert.equal(line, undefined);
5137
});
5238

5339
test('formatReplayTestProgressEvent ignores unknown progress event types', () => {
@@ -72,7 +58,7 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases',
7258
maxAttempts: 2,
7359
durationMs: 12_345,
7460
},
75-
expected: /^\[1\/3] PASS 01-login\.ad after 2 attempts \(total 12\.3s\)$/,
61+
expected: /^✓ 01-login\.ad \(12\.3s\)$/,
7662
},
7763
{
7864
event: {
@@ -87,7 +73,7 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases',
8773
retrying: true,
8874
message: 'first attempt failed',
8975
},
90-
expected: /^\[2\/3] RETRY 02-checkout\.ad attempt 1\/2 \(1\.23s\)\n first attempt failed$/,
76+
expected: /^$/,
9177
},
9278
{
9379
event: {
@@ -104,7 +90,7 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases',
10490
artifactsDir: '/tmp/replay-suite/payment',
10591
},
10692
expected:
107-
/^\[3\/3] FAIL 03-payment\.ad after 2 attempts \(total 9\.88s\)\n assertVisible failed\n session: maestro-test:test:suite:3:attempt-2\n artifacts: \/tmp\/replay-suite\/payment$/,
93+
/^⨯ 03-payment\.ad \(9\.88s\)\n failed at: assertVisible failed\n session: maestro-test:test:suite:3:attempt-2\n artifacts: \/tmp\/replay-suite\/payment$/,
10894
},
10995
{
11096
event: {
@@ -115,11 +101,70 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases',
115101
total: 5,
116102
message: 'missing platform metadata for --platform ios',
117103
},
118-
expected: /^\[4\/5] SKIP 04-skip\.ad\n missing platform metadata for --platform ios$/,
104+
expected: /^- 04-skip\.ad\n missing platform metadata for --platform ios$/,
119105
},
120106
];
121107

122108
for (const { event, expected } of cases) {
123109
assert.match(formatReplayTestProgressEvent(event) ?? '', expected);
124110
}
125111
});
112+
113+
test('formatReplayTestProgressEvent colors completed result markers when color is enabled', () => {
114+
const originalForceColor = process.env.FORCE_COLOR;
115+
const originalNoColor = process.env.NO_COLOR;
116+
process.env.FORCE_COLOR = '1';
117+
delete process.env.NO_COLOR;
118+
try {
119+
formatReplayTestProgressEvent({
120+
type: 'replay-test-suite',
121+
status: 'start',
122+
total: 3,
123+
runnable: 3,
124+
skipped: 0,
125+
artifactsDir: '/tmp/replay-suite',
126+
});
127+
assert.equal(
128+
formatReplayTestProgressEvent({
129+
type: 'replay-test',
130+
file: '/tmp/01-pass.ad',
131+
status: 'pass',
132+
index: 1,
133+
total: 3,
134+
attempt: 1,
135+
durationMs: 10,
136+
}),
137+
'\u001B[32m✓\u001B[39m 01-pass.ad (0.01s)',
138+
);
139+
assert.equal(
140+
formatReplayTestProgressEvent({
141+
type: 'replay-test',
142+
file: '/tmp/02-flaky.yml',
143+
title: 'Retry flow',
144+
status: 'pass',
145+
index: 2,
146+
total: 3,
147+
attempt: 2,
148+
durationMs: 30,
149+
}),
150+
'\u001B[33m✓\u001B[39m "Retry flow" in 02-flaky.yml (0.03s)',
151+
);
152+
const failedLine = formatReplayTestProgressEvent({
153+
type: 'replay-test',
154+
file: '/tmp/03-fail.ad',
155+
title: 'Checkout failure',
156+
status: 'fail',
157+
index: 3,
158+
total: 3,
159+
attempt: 1,
160+
durationMs: 5,
161+
message: 'boom',
162+
});
163+
assert.ok(failedLine?.startsWith('\u001B[31m⨯\u001B[39m "Checkout failure" in 03-fail.ad'));
164+
} finally {
165+
if (typeof originalForceColor === 'string') process.env.FORCE_COLOR = originalForceColor;
166+
else delete process.env.FORCE_COLOR;
167+
if (typeof originalNoColor === 'string') process.env.NO_COLOR = originalNoColor;
168+
else delete process.env.NO_COLOR;
169+
}
170+
});

0 commit comments

Comments
 (0)