Skip to content

Commit a7972ab

Browse files
bashmishbashmish
authored
test: improve tests stability (#3136)
* test: improve reload test stability * test: rely on timers less and give Windows more time * test: reduce each test waiting to fit into overall suite timeout
1 parent fb42aae commit a7972ab

5 files changed

Lines changed: 103 additions & 96 deletions

File tree

‎integration/test-runner/tests/location-change/browser-tests/fail-location-href.test.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,6 @@ setTimeout(() => {
33
}, 100);
44

55
it('x', async function test() {
6-
this.timeout(20000);
7-
await new Promise(r => setTimeout(r, 10000));
6+
this.timeout(10000);
7+
await new Promise(r => setTimeout(r, 5000));
88
});

‎integration/test-runner/tests/location-change/browser-tests/fail-location-reload.test.js‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,13 @@
1-
setTimeout(() => {
2-
window.location.reload();
3-
}, 100);
1+
if (!sessionStorage.getItem('reloaded')) {
2+
sessionStorage.setItem('reloaded', 'true');
3+
setTimeout(() => {
4+
window.location.reload();
5+
}, 100);
6+
} else {
7+
sessionStorage.removeItem('reloaded');
8+
}
49

510
it('x', async function test() {
6-
this.timeout(20000);
7-
await new Promise(r => setTimeout(r, 10000));
11+
this.timeout(10000);
12+
await new Promise(r => setTimeout(r, 5000));
813
});

‎integration/test-runner/tests/location-change/browser-tests/fail-location-replace.test.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,6 @@ setTimeout(() => {
33
}, 100);
44

55
it('x', async function test() {
6-
this.timeout(20000);
7-
await new Promise(r => setTimeout(r, 10000));
6+
this.timeout(10000);
7+
await new Promise(r => setTimeout(r, 5000));
88
});

‎packages/test-runner-core/test/src/runner/TestScheduler.test.ts‎

Lines changed: 83 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,22 @@ import type { TestSession } from '../../../dist/test-session/TestSession.js';
1010
import { TestSessionManager } from '../../../dist/test-session/TestSessionManager.js';
1111
import { SESSION_STATUS } from '../../../dist/test-session/TestSessionStatus.js';
1212

13+
async function waitUntilEqual<T>(
14+
getActual: () => T,
15+
expected: T,
16+
timeoutMs = 100,
17+
intervalMs = 5,
18+
): Promise<void> {
19+
const start = Date.now();
20+
while (getActual() !== expected) {
21+
if (Date.now() - start > timeoutMs) {
22+
assert.equal(getActual(), expected);
23+
return;
24+
}
25+
await timeout(intervalMs);
26+
}
27+
}
28+
1329
describe('TestScheduler', () => {
1430
let mockConfig: TestRunnerCoreConfig;
1531

@@ -76,8 +92,7 @@ describe('TestScheduler', () => {
7692
const [scheduler, sessions, [session1], browser] = createTestFixture('1');
7793
scheduler.schedule(1, [session1]);
7894

79-
const finalSession1 = sessions.get(session1.id)!;
80-
assert.equal(finalSession1.status, SESSION_STATUS.INITIALIZING);
95+
await waitUntilEqual(() => sessions.get(session1.id)!.status, SESSION_STATUS.INITIALIZING);
8196
assert.equal(browser.startSession.mock.callCount(), 1);
8297
});
8398

@@ -87,12 +102,11 @@ describe('TestScheduler', () => {
87102
browser.stopSession.mock.mockImplementation(() => timeout(1).then(() => ({ testCoverage })));
88103
scheduler.schedule(1, [session1]);
89104

90-
await timeout(5);
91-
sessions.updateStatus({ ...session1, passed: true }, SESSION_STATUS.TEST_FINISHED);
92-
await timeout(20);
105+
await waitUntilEqual(() => sessions.get(session1.id)!.status, SESSION_STATUS.INITIALIZING);
93106

107+
sessions.updateStatus({ ...session1, passed: true }, SESSION_STATUS.TEST_FINISHED);
108+
await waitUntilEqual(() => sessions.get(session1.id)!.status, SESSION_STATUS.FINISHED);
94109
const finalSession1 = sessions.get(session1.id)!;
95-
assert.equal(finalSession1.status, SESSION_STATUS.FINISHED);
96110
assert.equal(finalSession1.passed, true);
97111
assert.equal(finalSession1.testCoverage, testCoverage);
98112
});
@@ -101,29 +115,25 @@ describe('TestScheduler', () => {
101115
const [scheduler, sessions, sessionsToSchedule] = createTestFixture('1', '2', '3');
102116
scheduler.schedule(1, sessionsToSchedule);
103117

104-
assert.equal(sessions.get('1')!.status, SESSION_STATUS.INITIALIZING);
105-
assert.equal(sessions.get('2')!.status, SESSION_STATUS.INITIALIZING);
106-
assert.equal(sessions.get('3')!.status, SESSION_STATUS.SCHEDULED);
107-
108118
// wait for browser to start, session 3 should still not be started
109-
await timeout(2);
110-
assert.equal(sessions.get('3')!.status, SESSION_STATUS.SCHEDULED);
119+
await waitUntilEqual(() => sessions.get('1')!.status, SESSION_STATUS.INITIALIZING);
120+
await waitUntilEqual(() => sessions.get('2')!.status, SESSION_STATUS.INITIALIZING);
121+
await waitUntilEqual(() => sessions.get('3')!.status, SESSION_STATUS.SCHEDULED);
111122

112123
// mark tests as finished
113124
sessions.updateStatus({ ...sessions.get('1')!, passed: true }, SESSION_STATUS.TEST_FINISHED);
114125
sessions.updateStatus({ ...sessions.get('2')!, passed: true }, SESSION_STATUS.TEST_FINISHED);
115126

116-
// wait for browser to end
117-
await timeout(20);
127+
// wait for sessions to finish
128+
await waitUntilEqual(() => sessions.get('1')!.status, SESSION_STATUS.FINISHED);
129+
await waitUntilEqual(() => sessions.get('2')!.status, SESSION_STATUS.FINISHED);
118130

119-
// sessions 1 and 2 should be finished
120-
assert.equal(sessions.get('1')!.status, SESSION_STATUS.FINISHED);
131+
// sessions 1 and 2 should pass
121132
assert.equal(sessions.get('1')!.passed, true);
122-
assert.equal(sessions.get('2')!.status, SESSION_STATUS.FINISHED);
123133
assert.equal(sessions.get('2')!.passed, true);
124134

125135
// session 3 should be started
126-
assert.equal(sessions.get('3')!.status, SESSION_STATUS.INITIALIZING);
136+
await waitUntilEqual(() => sessions.get('3')!.status, SESSION_STATUS.INITIALIZING);
127137
});
128138

129139
it('scheduling new tests while executing keeps batching', async () => {
@@ -132,53 +142,44 @@ describe('TestScheduler', () => {
132142

133143
// schedule 2 sessions
134144
scheduler.schedule(1, sessionsToSchedule.slice(0, 2));
135-
136-
assert.equal(sessions.get('1')!.status, SESSION_STATUS.INITIALIZING);
137-
assert.equal(sessions.get('2')!.status, SESSION_STATUS.INITIALIZING);
145+
await waitUntilEqual(() => sessions.get('1')!.status, SESSION_STATUS.INITIALIZING);
146+
await waitUntilEqual(() => sessions.get('2')!.status, SESSION_STATUS.INITIALIZING);
138147

139148
// schedule 3 more sessions after browser starts
140-
await timeout(20);
141149
scheduler.schedule(1, sessionsToSchedule.slice(2, 5));
142-
assert.equal(sessions.get('1')!.status, SESSION_STATUS.INITIALIZING);
143-
assert.equal(sessions.get('2')!.status, SESSION_STATUS.INITIALIZING);
144-
assert.equal(sessions.get('3')!.status, SESSION_STATUS.SCHEDULED);
145-
assert.equal(sessions.get('4')!.status, SESSION_STATUS.SCHEDULED);
146-
assert.equal(sessions.get('5')!.status, SESSION_STATUS.SCHEDULED);
150+
await waitUntilEqual(() => sessions.get('1')!.status, SESSION_STATUS.INITIALIZING);
151+
await waitUntilEqual(() => sessions.get('2')!.status, SESSION_STATUS.INITIALIZING);
152+
await waitUntilEqual(() => sessions.get('3')!.status, SESSION_STATUS.SCHEDULED);
153+
await waitUntilEqual(() => sessions.get('4')!.status, SESSION_STATUS.SCHEDULED);
154+
await waitUntilEqual(() => sessions.get('5')!.status, SESSION_STATUS.SCHEDULED);
147155

148156
// mark first test as finished
149157
sessions.updateStatus({ ...sessions.get('1')!, passed: true }, SESSION_STATUS.TEST_FINISHED);
150158

151-
// wait for browser to end
152-
await timeout(20);
153-
154-
// session 1 is finished, session 2 is still waiting, session 3 is now starting and the rest is still scheduled
155-
assert.equal(sessions.get('1')!.status, SESSION_STATUS.FINISHED);
159+
// session 1 passed, session 2 is still waiting, session 3 is now starting and the rest is still scheduled
160+
await waitUntilEqual(() => sessions.get('1')!.status, SESSION_STATUS.FINISHED);
156161
assert.equal(sessions.get('1')!.passed, true);
157-
assert.equal(sessions.get('2')!.status, SESSION_STATUS.INITIALIZING);
158-
assert.equal(sessions.get('3')!.status, SESSION_STATUS.INITIALIZING);
159-
assert.equal(sessions.get('4')!.status, SESSION_STATUS.SCHEDULED);
160-
assert.equal(sessions.get('5')!.status, SESSION_STATUS.SCHEDULED);
162+
await waitUntilEqual(() => sessions.get('2')!.status, SESSION_STATUS.INITIALIZING);
163+
await waitUntilEqual(() => sessions.get('3')!.status, SESSION_STATUS.INITIALIZING);
164+
await waitUntilEqual(() => sessions.get('4')!.status, SESSION_STATUS.SCHEDULED);
165+
await waitUntilEqual(() => sessions.get('5')!.status, SESSION_STATUS.SCHEDULED);
161166

162167
// mark 2 and 3 as finished
163-
await timeout(5);
164168
sessions.updateStatus({ ...sessions.get('2')!, passed: true }, SESSION_STATUS.TEST_FINISHED);
165169
sessions.updateStatus({ ...sessions.get('3')!, passed: true }, SESSION_STATUS.TEST_FINISHED);
166170

167171
// 2 and 3 finish when browser closes
168-
await timeout(20);
169-
assert.equal(sessions.get('2')!.status, SESSION_STATUS.FINISHED);
170-
assert.equal(sessions.get('3')!.status, SESSION_STATUS.FINISHED);
172+
await waitUntilEqual(() => sessions.get('2')!.status, SESSION_STATUS.FINISHED);
173+
await waitUntilEqual(() => sessions.get('3')!.status, SESSION_STATUS.FINISHED);
171174
});
172175

173176
it('error while starting browser marks session as failed', async () => {
174177
const [scheduler, sessions, [session1], browser] = createTestFixture('1');
175178
browser.startSession.mock.mockImplementation(() => Promise.reject(new Error('mock error')));
176179
scheduler.schedule(1, [session1]);
177180

178-
await timeout(20);
179-
181+
await waitUntilEqual(() => sessions.get(session1.id)!.status, SESSION_STATUS.FINISHED);
180182
const finalSession1 = sessions.get(session1.id)!;
181-
assert.equal(finalSession1.status, SESSION_STATUS.FINISHED);
182183
assert.equal(finalSession1.passed, false);
183184
assert.equal(finalSession1.errors.length, 1);
184185
assert.equal(finalSession1.errors[0].message, 'mock error');
@@ -188,22 +189,20 @@ describe('TestScheduler', () => {
188189
const errorStub = mock.method(mockConfig.logger, 'error');
189190
const [scheduler, sessions, [session1], browser] = createTestFixture('1');
190191
browser.startSession.mock.mockImplementation(() =>
191-
timeout(5).then(() => {
192+
timeout(1).then(() => {
192193
throw new Error('mock error');
193194
}),
194195
);
195196
scheduler.schedule(1, [session1]);
196197

197-
await timeout(2);
198-
sessions.updateStatus({ ...session1, passed: true }, SESSION_STATUS.TEST_FINISHED);
199-
await timeout(5);
198+
await waitUntilEqual(() => sessions.get(session1.id)!.status, SESSION_STATUS.INITIALIZING);
200199

200+
sessions.updateStatus({ ...session1, passed: true }, SESSION_STATUS.TEST_FINISHED);
201+
await waitUntilEqual(() => sessions.get(session1.id)!.status, SESSION_STATUS.FINISHED);
201202
const finalSession1 = sessions.get(session1.id)!;
202-
assert.equal(finalSession1.status, SESSION_STATUS.FINISHED);
203203
assert.equal(finalSession1.passed, true);
204204

205-
await timeout(20);
206-
205+
await waitUntilEqual(() => errorStub.mock.callCount(), 1);
207206
assert.equal(errorStub.mock.callCount(), 1);
208207
assert.ok(errorStub.mock.calls[0].arguments[0] instanceof Error);
209208
assert.equal((errorStub.mock.calls[0].arguments[0] as Error).message, 'mock error');
@@ -214,12 +213,11 @@ describe('TestScheduler', () => {
214213
browser.stopSession.mock.mockImplementation(() => Promise.reject(new Error('mock error')));
215214
scheduler.schedule(1, [session1]);
216215

217-
await timeout(5);
218-
sessions.updateStatus({ ...session1, passed: true }, SESSION_STATUS.TEST_FINISHED);
219-
await timeout(20);
216+
await waitUntilEqual(() => sessions.get(session1.id)!.status, SESSION_STATUS.INITIALIZING);
220217

218+
sessions.updateStatus({ ...session1, passed: true }, SESSION_STATUS.TEST_FINISHED);
219+
await waitUntilEqual(() => sessions.get(session1.id)!.status, SESSION_STATUS.FINISHED);
221220
const finalSession1 = sessions.get(session1.id)!;
222-
assert.equal(finalSession1.status, SESSION_STATUS.FINISHED);
223221
assert.equal(finalSession1.passed, false);
224222
assert.equal(finalSession1.errors.length, 1);
225223
assert.equal(finalSession1.errors[0].message, 'mock error');
@@ -228,13 +226,11 @@ describe('TestScheduler', () => {
228226
it('timeout starting the browser marks the session as failed', async () => {
229227
mockConfig.browserStartTimeout = 2;
230228
const [scheduler, sessions, [session1], browser] = createTestFixture('1');
231-
browser.startSession.mock.mockImplementation(() => timeout(40));
229+
browser.startSession.mock.mockImplementation(() => timeout(50));
232230
scheduler.schedule(1, [session1]);
233231

234-
await timeout(20);
235-
232+
await waitUntilEqual(() => sessions.get(session1.id)!.status, SESSION_STATUS.FINISHED);
236233
const finalSession1 = sessions.get(session1.id)!;
237-
assert.equal(finalSession1.status, SESSION_STATUS.FINISHED);
238234
assert.equal(finalSession1.passed, false);
239235
assert.equal(finalSession1.errors.length, 1);
240236
assert.equal(
@@ -248,8 +244,7 @@ describe('TestScheduler', () => {
248244
const [scheduler, sessions, [session1]] = createTestFixture('1');
249245
scheduler.schedule(1, [session1]);
250246

251-
await timeout(20);
252-
247+
await waitUntilEqual(() => sessions.get(session1.id)!.status, SESSION_STATUS.FINISHED);
253248
const finalSession1 = sessions.get(session1.id)!;
254249
assert.equal(finalSession1.status, SESSION_STATUS.FINISHED);
255250
assert.equal(finalSession1.passed, false);
@@ -265,12 +260,11 @@ describe('TestScheduler', () => {
265260
const [scheduler, sessions, [session1]] = createTestFixture('1');
266261
scheduler.schedule(1, [session1]);
267262

268-
await timeout(5);
269-
sessions.updateStatus({ ...session1, passed: true }, SESSION_STATUS.TEST_STARTED);
270-
await timeout(20);
263+
await waitUntilEqual(() => sessions.get(session1.id)!.status, SESSION_STATUS.INITIALIZING);
271264

265+
sessions.updateStatus({ ...session1, passed: true }, SESSION_STATUS.TEST_STARTED);
266+
await waitUntilEqual(() => sessions.get(session1.id)!.status, SESSION_STATUS.FINISHED);
272267
const finalSession1 = sessions.get(session1.id)!;
273-
assert.equal(finalSession1.status, SESSION_STATUS.FINISHED);
274268
assert.equal(finalSession1.passed, false);
275269
assert.equal(finalSession1.errors.length, 1);
276270
assert.equal(
@@ -310,37 +304,37 @@ describe('TestScheduler', () => {
310304
]);
311305
scheduler.schedule(1, sessions);
312306

307+
await waitUntilEqual(() => sessionManager.get('1')!.status, SESSION_STATUS.INITIALIZING);
308+
await waitUntilEqual(() => sessionManager.get('2')!.status, SESSION_STATUS.INITIALIZING);
309+
await waitUntilEqual(() => sessionManager.get('3')!.status, SESSION_STATUS.SCHEDULED);
313310
const session1 = sessionManager.get('1')!;
314311
const session2 = sessionManager.get('2')!;
315312
const session3 = sessionManager.get('3')!;
316313
assert.equal(session1.browser, browsers[0]);
317314
assert.equal(session2.browser, browsers[0]);
318315
assert.equal(session3.browser, browsers[0]);
319-
assert.equal(session1.status, SESSION_STATUS.INITIALIZING);
320-
assert.equal(session2.status, SESSION_STATUS.INITIALIZING);
321-
assert.equal(session3.status, SESSION_STATUS.SCHEDULED);
322316
assert.equal(browsers[0].startSession.mock.callCount(), 2);
323317

318+
await waitUntilEqual(() => sessionManager.get('4')!.status, SESSION_STATUS.INITIALIZING);
319+
await waitUntilEqual(() => sessionManager.get('5')!.status, SESSION_STATUS.INITIALIZING);
320+
await waitUntilEqual(() => sessionManager.get('6')!.status, SESSION_STATUS.SCHEDULED);
324321
const session4 = sessionManager.get('4')!;
325322
const session5 = sessionManager.get('5')!;
326323
const session6 = sessionManager.get('6')!;
327324
assert.equal(session4.browser, browsers[1]);
328325
assert.equal(session5.browser, browsers[1]);
329326
assert.equal(session6.browser, browsers[1]);
330-
assert.equal(session4.status, SESSION_STATUS.INITIALIZING);
331-
assert.equal(session5.status, SESSION_STATUS.INITIALIZING);
332-
assert.equal(session6.status, SESSION_STATUS.SCHEDULED);
333327
assert.equal(browsers[1].startSession.mock.callCount(), 2);
334328

329+
await waitUntilEqual(() => sessionManager.get('7')!.status, SESSION_STATUS.SCHEDULED);
330+
await waitUntilEqual(() => sessionManager.get('8')!.status, SESSION_STATUS.SCHEDULED);
331+
await waitUntilEqual(() => sessionManager.get('9')!.status, SESSION_STATUS.SCHEDULED);
335332
const session7 = sessionManager.get('7')!;
336333
const session8 = sessionManager.get('8')!;
337334
const session9 = sessionManager.get('9')!;
338335
assert.equal(session7.browser, browsers[2]);
339336
assert.equal(session8.browser, browsers[2]);
340337
assert.equal(session9.browser, browsers[2]);
341-
assert.equal(session7.status, SESSION_STATUS.SCHEDULED);
342-
assert.equal(session8.status, SESSION_STATUS.SCHEDULED);
343-
assert.equal(session9.status, SESSION_STATUS.SCHEDULED);
344338
assert.equal(browsers[2].startSession.mock.callCount(), 0);
345339
});
346340

@@ -352,13 +346,13 @@ describe('TestScheduler', () => {
352346
]);
353347
scheduler.schedule(1, sessions);
354348

355-
await timeout(5);
356-
sessionManager.updateStatus({ ...sessions[0], passed: true }, SESSION_STATUS.TEST_FINISHED);
357-
await timeout(20);
349+
await waitUntilEqual(() => sessionManager.get('1')!.status, SESSION_STATUS.INITIALIZING);
350+
await waitUntilEqual(() => sessionManager.get('2')!.status, SESSION_STATUS.INITIALIZING);
358351

352+
sessionManager.updateStatus({ ...sessions[0], passed: true }, SESSION_STATUS.TEST_FINISHED);
353+
await waitUntilEqual(() => sessionManager.get('3')!.status, SESSION_STATUS.INITIALIZING);
359354
const session3 = sessionManager.get('3')!;
360355
assert.equal(session3.browser, browsers[0]);
361-
assert.equal(session3.status, SESSION_STATUS.INITIALIZING);
362356
assert.equal(browsers[0].startSession.mock.callCount(), 3);
363357
});
364358

@@ -370,14 +364,14 @@ describe('TestScheduler', () => {
370364
]);
371365
scheduler.schedule(1, sessions);
372366

373-
await timeout(5);
367+
await waitUntilEqual(() => sessionManager.get('1')!.status, SESSION_STATUS.INITIALIZING);
368+
await waitUntilEqual(() => sessionManager.get('2')!.status, SESSION_STATUS.INITIALIZING);
369+
374370
sessionManager.updateStatus({ ...sessions[0], passed: true }, SESSION_STATUS.TEST_FINISHED);
375371
sessionManager.updateStatus({ ...sessions[1], passed: true }, SESSION_STATUS.TEST_FINISHED);
376-
await timeout(20);
377-
372+
await waitUntilEqual(() => sessionManager.get('7')!.status, SESSION_STATUS.SCHEDULED);
378373
const session7 = sessionManager.get('7')!;
379374
assert.equal(session7.browser, browsers[2]);
380-
assert.equal(session7.status, SESSION_STATUS.SCHEDULED);
381375
assert.equal(browsers[2].startSession.mock.callCount(), 0);
382376
});
383377

@@ -389,16 +383,19 @@ describe('TestScheduler', () => {
389383
]);
390384
scheduler.schedule(1, sessions);
391385

392-
await timeout(5);
386+
await waitUntilEqual(() => sessionManager.get('1')!.status, SESSION_STATUS.INITIALIZING);
387+
await waitUntilEqual(() => sessionManager.get('2')!.status, SESSION_STATUS.INITIALIZING);
388+
393389
sessionManager.updateStatus({ ...sessions[0], passed: true }, SESSION_STATUS.TEST_FINISHED);
394390
sessionManager.updateStatus({ ...sessions[1], passed: true }, SESSION_STATUS.TEST_FINISHED);
395-
await timeout(5);
396-
sessionManager.updateStatus({ ...sessions[2], passed: true }, SESSION_STATUS.TEST_FINISHED);
397-
await timeout(20);
391+
await waitUntilEqual(() => sessionManager.get('1')!.status, SESSION_STATUS.FINISHED);
392+
await waitUntilEqual(() => sessionManager.get('2')!.status, SESSION_STATUS.FINISHED);
393+
await waitUntilEqual(() => sessionManager.get('7')!.status, SESSION_STATUS.SCHEDULED);
398394

395+
sessionManager.updateStatus({ ...sessions[2], passed: true }, SESSION_STATUS.TEST_FINISHED);
396+
await waitUntilEqual(() => sessionManager.get('7')!.status, SESSION_STATUS.INITIALIZING);
399397
const session7 = sessionManager.get('7')!;
400398
assert.equal(session7.browser, browsers[2]);
401-
assert.equal(session7.status, SESSION_STATUS.INITIALIZING);
402399
assert.equal(browsers[2].startSession.mock.callCount(), 2);
403400
});
404401
});

‎packages/test-runner/demo/test/fail-location-reload.test.js‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
1-
window.location.reload();
1+
if (!sessionStorage.getItem('reloaded')) {
2+
sessionStorage.setItem('reloaded', 'true');
3+
window.location.reload();
4+
} else {
5+
sessionStorage.removeItem('reloaded');
6+
}
27

38
it('x', async () => {
49
await new Promise(r => setTimeout(r, 1000));

0 commit comments

Comments
 (0)