Skip to content

Commit 87aa82e

Browse files
thymikeethymikee
committed
fix: improve maestro test output
1 parent 096785b commit 87aa82e

12 files changed

Lines changed: 547 additions & 38 deletions

File tree

‎src/__tests__/cli-network.test.ts‎

Lines changed: 203 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import { runCliCapture } from './cli-capture.ts';
88
function makeFailedReplayResult() {
99
return {
1010
file: '/tmp/02-fail.ad',
11+
title: 'Checkout failure',
1112
session: 'default:test:suite:2',
1213
status: 'failed',
1314
durationMs: 5,
@@ -106,11 +107,14 @@ test('test command prints suite summary and exits non-zero on failures', async (
106107
assert.equal(result.calls[0]?.meta?.requestProgress, 'replay-test');
107108
assert.match(result.stderr, /Running replay suite\.\.\./);
108109
assert.doesNotMatch(result.stdout, /PASS \/tmp\/01-pass\.ad/);
109-
assert.match(result.stdout, /FAIL \/tmp\/02-fail\.ad after 2 attempts \(5ms\)/);
110+
assert.match(
111+
result.stdout,
112+
/FAIL "Checkout failure" in 02-fail\.ad after 2 attempts \(total 0\.005s\)/,
113+
);
110114
assert.match(result.stdout, /Replay failed at step 1 \(open Demo\): boom/);
111115
assert.match(result.stdout, /artifacts: \/tmp\/test-artifacts\/02-fail/);
112116
assert.doesNotMatch(result.stdout, /SKIP \/tmp\/03-skip\.ad/);
113-
assert.match(result.stdout, /Test summary: 1 passed, 1 failed in 25ms/);
117+
assert.match(result.stdout, /Test summary: 1 passed, 1 failed in 0\.025s/);
114118
});
115119

116120
test('test command --verbose prints all test statuses', async () => {
@@ -119,9 +123,93 @@ test('test command --verbose prints all test statuses', async () => {
119123
);
120124

121125
assert.equal(result.code, 1);
126+
assert.equal(result.calls[0]?.meta?.debug, false);
122127
assert.match(result.stderr, /Running replay suite\.\.\./);
123-
assert.match(result.stdout, /PASS \/tmp\/01-pass\.ad \(10ms\)/);
124-
assert.match(result.stdout, /SKIP \/tmp\/03-skip\.ad/);
128+
assert.match(result.stdout, /PASS 01-pass\.ad \(0\.01s\)/);
129+
assert.match(result.stdout, /SKIP 03-skip\.ad/);
130+
});
131+
132+
test('test command --verbose prints step telemetry for passing tests without debug mode', async () => {
133+
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-cli-test-verbose-'));
134+
const artifactsDir = path.join(tmpDir, 'auth-flow');
135+
const attemptDir = path.join(artifactsDir, 'attempt-1');
136+
await fs.mkdir(attemptDir, { recursive: true });
137+
await fs.writeFile(
138+
path.join(attemptDir, 'replay-timing.ndjson'),
139+
[
140+
{
141+
type: 'replay_action_start',
142+
step: 1,
143+
line: 3,
144+
command: '__maestroTapOn',
145+
positionals: ['text="Log in"'],
146+
},
147+
{
148+
type: 'replay_action_stop',
149+
step: 1,
150+
line: 3,
151+
command: '__maestroTapOn',
152+
ok: true,
153+
durationMs: 250,
154+
},
155+
{
156+
type: 'replay_action_start',
157+
step: 2,
158+
line: 4,
159+
command: '__maestroAssertVisible',
160+
positionals: ['text="Home"'],
161+
},
162+
{
163+
type: 'replay_action_stop',
164+
step: 2,
165+
line: 4,
166+
command: '__maestroAssertVisible',
167+
ok: true,
168+
durationMs: 75,
169+
},
170+
]
171+
.map((entry) => JSON.stringify(entry))
172+
.join('\n'),
173+
);
174+
175+
try {
176+
const result = await runCliCapture(['test', './suite', '--verbose'], async () => ({
177+
ok: true,
178+
data: {
179+
total: 1,
180+
executed: 1,
181+
passed: 1,
182+
failed: 0,
183+
skipped: 0,
184+
notRun: 0,
185+
durationMs: 500,
186+
failures: [],
187+
tests: [
188+
{
189+
file: '/tmp/auth-flow.yml',
190+
title: 'Authentication flow',
191+
session: 'default:test:suite:1',
192+
status: 'passed',
193+
durationMs: 500,
194+
finalAttemptDurationMs: 500,
195+
attempts: 1,
196+
artifactsDir,
197+
replayed: 2,
198+
healed: 0,
199+
},
200+
],
201+
},
202+
}));
203+
204+
assert.equal(result.code, null);
205+
assert.equal(result.calls[0]?.meta?.debug, false);
206+
assert.match(result.stdout, /PASS "Authentication flow" \(0\.5s\)/);
207+
assert.match(result.stdout, /steps \(attempt 1\):/);
208+
assert.match(result.stdout, /\[ok\] tapOn "text=\\"Log in\\"" \(line 3, 0\.25s\)/);
209+
assert.match(result.stdout, /\[ok\] assertVisible "text=\\"Home\\"" \(line 4, 0\.075s\)/);
210+
} finally {
211+
await fs.rm(tmpDir, { recursive: true, force: true });
212+
}
125213
});
126214

127215
test('test command reports flaky passed-on-retry cases in the default summary', async () => {
@@ -138,20 +226,127 @@ test('test command reports flaky passed-on-retry cases in the default summary',
138226
failures: [],
139227
tests: [
140228
{
141-
file: '/tmp/01-flaky.ad',
229+
file: '/tmp/auth-flow.yml',
230+
title: 'Authentication flow',
142231
session: 'default:test:suite:1',
143232
status: 'passed',
144-
durationMs: 10,
233+
durationMs: 112151,
234+
finalAttemptDurationMs: 17492,
145235
attempts: 2,
236+
attemptFailures: [
237+
{
238+
attempt: 1,
239+
message: 'Replay failed at step 3 (tapOn "Log in"): selector not found',
240+
durationMs: 94659,
241+
},
242+
],
146243
},
147244
],
148245
},
149246
}));
150247

151248
assert.equal(result.code, null);
152249
assert.match(result.stderr, /Running replay suite\.\.\./);
153-
assert.match(result.stdout, /FLAKY \/tmp\/01-flaky\.ad after 2 attempts \(10ms\)/);
154-
assert.match(result.stdout, /Test summary: 1 passed, 0 failed, 1 flaky in 25ms/);
250+
assert.doesNotMatch(result.stdout, /FLAKY/);
251+
assert.match(result.stdout, /Test summary: 1 passed, 0 failed, 1 flaky in 0\.025s/);
252+
assert.match(result.stdout, /Flaky tests:/);
253+
assert.match(
254+
result.stdout,
255+
/PASS "Authentication flow" after 2 attempts \(passed attempt 17\.5s, total 112\.2s\)/,
256+
);
257+
assert.match(
258+
result.stdout,
259+
/attempt 1 failed \(94\.7s\): Replay failed at step 3 \(tapOn "Log in"\): selector not found/,
260+
);
261+
});
262+
263+
test('test command prints failed attempt step telemetry when timing trace exists', async () => {
264+
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-cli-test-steps-'));
265+
const artifactsDir = path.join(tmpDir, 'checkout-flow');
266+
const attemptDir = path.join(artifactsDir, 'attempt-2');
267+
await fs.mkdir(attemptDir, { recursive: true });
268+
await fs.writeFile(
269+
path.join(attemptDir, 'replay-timing.ndjson'),
270+
[
271+
{
272+
type: 'replay_action_start',
273+
step: 1,
274+
line: 3,
275+
command: 'open',
276+
positionals: ['Demo'],
277+
},
278+
{
279+
type: 'replay_action_stop',
280+
step: 1,
281+
line: 3,
282+
command: 'open',
283+
ok: true,
284+
durationMs: 125,
285+
resultTiming: { launchMs: 100 },
286+
},
287+
{
288+
type: 'replay_action_start',
289+
step: 2,
290+
line: 4,
291+
command: '__maestroTapOn',
292+
positionals: ['text="Pay"'],
293+
},
294+
{
295+
type: 'replay_action_stop',
296+
step: 2,
297+
line: 4,
298+
command: '__maestroTapOn',
299+
ok: false,
300+
durationMs: 1500,
301+
errorCode: 'ASSERTION_FAILED',
302+
},
303+
]
304+
.map((entry) => JSON.stringify(entry))
305+
.join('\n'),
306+
);
307+
308+
try {
309+
const failedReplayResult = {
310+
file: '/tmp/checkout-flow.yml',
311+
title: 'Checkout flow',
312+
session: 'default:test:suite:1',
313+
status: 'failed',
314+
durationMs: 2000,
315+
attempts: 2,
316+
artifactsDir,
317+
error: {
318+
code: 'ASSERTION_FAILED',
319+
message: 'Replay failed at step 2 (click "Pay"): selector not found',
320+
},
321+
};
322+
const result = await runCliCapture(['test', './suite'], async () => ({
323+
ok: true,
324+
data: {
325+
total: 1,
326+
executed: 1,
327+
passed: 0,
328+
failed: 1,
329+
skipped: 0,
330+
notRun: 0,
331+
durationMs: 2000,
332+
failures: [failedReplayResult],
333+
tests: [failedReplayResult],
334+
},
335+
}));
336+
337+
assert.equal(result.code, 1);
338+
assert.match(result.stdout, /steps \(attempt 2\):/);
339+
assert.match(
340+
result.stdout,
341+
/\[ok\] open "Demo" \(line 3, 0\.125s, timing \{"launchMs":100\}\)/,
342+
);
343+
assert.match(
344+
result.stdout,
345+
/\[FAIL\] tapOn "text=\\"Pay\\"" \(line 4, 1\.50s, ASSERTION_FAILED\)/,
346+
);
347+
} finally {
348+
await fs.rm(tmpDir, { recursive: true, force: true });
349+
}
155350
});
156351

157352
test('test --maestro forwards Maestro backend and platform for directory suites', async () => {

0 commit comments

Comments
 (0)