Skip to content

Commit 82af075

Browse files
sirtimidsirtimid
authored
test(e2e): Attempt to fix flaky E2E tests by re-evaluating locator usage (#534)
E2E tests have been intermittently failing on CI, specifically where the message-output locator doesn't return the expected value despite it being clearly visible in the trace screenshot. My suspicion is that the assigned const locator in beforeEach might become stale in some test runs. This PR updates the tests to retrieve the locator at each point it's needed, instead of caching it up front. Hopefully this helps stabilize the tests. Will monitor results and investigate further if needed.
1 parent eb8dc04 commit 82af075

1 file changed

Lines changed: 112 additions & 52 deletions

File tree

‎packages/extension/test/e2e/control-panel.test.ts‎

Lines changed: 112 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import defaultClusterConfig from '@metamask/kernel-browser-runtime/default-cluster' assert { type: 'json' };
22
import { test, expect } from '@playwright/test';
3-
import type { Page, BrowserContext, Locator } from '@playwright/test';
3+
import type { Page, BrowserContext } from '@playwright/test';
44

55
// Vitest/Playwright needs the import assertions
66
import minimalClusterConfig from '../../src/vats/minimal-cluster.json' assert { type: 'json' };
@@ -12,14 +12,12 @@ test.describe('Control Panel', () => {
1212
let extensionContext: BrowserContext;
1313
let popupPage: Page;
1414
let extensionId: string;
15-
let messageOutput: Locator;
1615

1716
test.beforeEach(async () => {
1817
const extension = await makeLoadExtension();
1918
extensionContext = extension.browserContext;
2019
popupPage = extension.popupPage;
2120
extensionId = extension.extensionId;
22-
messageOutput = popupPage.locator('[data-testid="message-output"]');
2321
await expect(popupPage.locator('[data-testid="vat-table"]')).toBeVisible();
2422
await expect(popupPage.locator('table tr')).toHaveCount(4); // Header + 3 rows
2523
});
@@ -33,11 +31,15 @@ test.describe('Control Panel', () => {
3331
*/
3432
async function clearState(): Promise<void> {
3533
await popupPage.locator('[data-testid="clear-logs-button"]').click();
36-
await expect(messageOutput).toContainText('');
34+
await expect(
35+
popupPage.locator('[data-testid="message-output"]'),
36+
).toContainText('');
3737
await popupPage.fill('input[placeholder="Vat Name"]', '');
3838
await popupPage.fill('input[placeholder="Bundle URL"]', '');
3939
await popupPage.click('button:text("Clear All State")');
40-
await expect(messageOutput).toContainText('State cleared');
40+
await expect(
41+
popupPage.locator('[data-testid="message-output"]'),
42+
).toContainText('State cleared');
4143
await expect(
4244
popupPage.locator('[data-testid="vat-table"]'),
4345
).not.toBeVisible();
@@ -55,7 +57,9 @@ test.describe('Control Panel', () => {
5557
'http://localhost:3000/sample-vat.bundle',
5658
);
5759
await popupPage.click('button:text("Launch Vat")');
58-
await expect(messageOutput).toContainText(`Launched vat "${name}"`);
60+
await expect(
61+
popupPage.locator('[data-testid="message-output"]'),
62+
).toContainText(`Launched vat "${name}"`);
5963
}
6064

6165
test('should load popup with kernel panel', async () => {
@@ -103,7 +107,9 @@ test.describe('Control Panel', () => {
103107
popupPage.locator('button:text("Restart")').first(),
104108
).toBeVisible();
105109
await popupPage.locator('button:text("Restart")').first().click();
106-
await expect(messageOutput).toContainText('Restarted vat "v1"');
110+
await expect(
111+
popupPage.locator('[data-testid="message-output"]'),
112+
).toContainText('Restarted vat "v1"');
107113
});
108114

109115
test('should terminate a vat', async () => {
@@ -112,7 +118,9 @@ test.describe('Control Panel', () => {
112118
popupPage.locator('td button:text("Terminate")').first(),
113119
).toBeVisible();
114120
await popupPage.locator('td button:text("Terminate")').first().click();
115-
await expect(messageOutput).toContainText('Terminated vat "v1"');
121+
await expect(
122+
popupPage.locator('[data-testid="message-output"]'),
123+
).toContainText('Terminated vat "v1"');
116124
await expect(popupPage.locator('table tr')).toHaveCount(3);
117125
});
118126

@@ -121,20 +129,28 @@ test.describe('Control Panel', () => {
121129
popupPage.locator('td button:text("Ping")').first(),
122130
).toBeVisible();
123131
await popupPage.locator('td button:text("Ping")').first().click();
124-
await expect(messageOutput).toContainText('"method": "pingVat",');
125-
await expect(messageOutput).toContainText('pong');
132+
await expect(
133+
popupPage.locator('[data-testid="message-output"]'),
134+
).toContainText('"method": "pingVat",');
135+
await expect(
136+
popupPage.locator('[data-testid="message-output"]'),
137+
).toContainText('pong');
126138
});
127139

128140
test('should terminate all vats', async () => {
129141
await expect(
130142
popupPage.locator('button:text("Terminate All Vats")'),
131143
).toBeVisible();
132144
await popupPage.click('button:text("Terminate All Vats")');
133-
await expect(messageOutput).toContainText('All vats terminated');
145+
await expect(
146+
popupPage.locator('[data-testid="message-output"]'),
147+
).toContainText('All vats terminated');
134148
await expect(popupPage.locator('table')).not.toBeVisible();
135149
// ensure all references were garbage collected
136150
await popupPage.locator('[data-testid="clear-logs-button"]').click();
137-
await expect(messageOutput).toContainText('');
151+
await expect(
152+
popupPage.locator('[data-testid="message-output"]'),
153+
).toContainText('');
138154
await popupPage.click('button:text("Database Inspector")');
139155
const expectedValues = JSON.stringify([
140156
{ key: 'queue.run.head', value: '6' },
@@ -148,16 +164,22 @@ test.describe('Control Panel', () => {
148164
{ key: 'nextRemoteId', value: '1' },
149165
{ key: 'initialized', value: 'true' },
150166
]);
151-
await expect(messageOutput).toContainText(expectedValues);
167+
await expect(
168+
popupPage.locator('[data-testid="message-output"]'),
169+
).toContainText(expectedValues);
152170
});
153171

154172
test('should clear kernel state', async () => {
155173
await popupPage.click('button:text("Clear All State")');
156-
await expect(messageOutput).toContainText('State cleared');
174+
await expect(
175+
popupPage.locator('[data-testid="message-output"]'),
176+
).toContainText('State cleared');
157177
await expect(popupPage.locator('table')).not.toBeVisible();
158178
// ensure kernel state was cleared
159179
await popupPage.locator('[data-testid="clear-logs-button"]').click();
160-
await expect(messageOutput).toContainText('');
180+
await expect(
181+
popupPage.locator('[data-testid="message-output"]'),
182+
).toContainText('');
161183
await popupPage.click('button:text("Database Inspector")');
162184
const expectedValues = JSON.stringify([
163185
{ key: 'queue.run.head', value: '1' },
@@ -170,8 +192,12 @@ test.describe('Control Panel', () => {
170192
{ key: 'nextVatId', value: '1' },
171193
{ key: 'nextRemoteId', value: '1' },
172194
]);
173-
await expect(messageOutput).toContainText(expectedValues);
174-
await expect(messageOutput).not.toContainText('"initialized":true');
195+
await expect(
196+
popupPage.locator('[data-testid="message-output"]'),
197+
).toContainText(expectedValues);
198+
await expect(
199+
popupPage.locator('[data-testid="message-output"]'),
200+
).not.toContainText('"initialized":true');
175201
await popupPage.click('button:text("Control Panel")');
176202
await launchVat('test-vat-new');
177203
await expect(popupPage.locator('table tr')).toHaveCount(2);
@@ -250,8 +276,12 @@ test.describe('Control Panel', () => {
250276
popupPage.locator('button:text("Reload Kernel")'),
251277
).toBeVisible();
252278
await popupPage.click('button:text("Reload Kernel")');
253-
await expect(messageOutput).toContainText('"method": "reload"');
254-
await expect(messageOutput).toContainText('Default sub-cluster reloaded', {
279+
await expect(
280+
popupPage.locator('[data-testid="message-output"]'),
281+
).toContainText('"method": "reload"');
282+
await expect(
283+
popupPage.locator('[data-testid="message-output"]'),
284+
).toContainText('Default sub-cluster reloaded', {
255285
timeout: 10000,
256286
});
257287
});
@@ -269,7 +299,9 @@ test.describe('Control Panel', () => {
269299
// Test invalid JSON handling
270300
await configTextarea.fill('{ invalid json }');
271301
await popupPage.click('button:text("Update Config")');
272-
await expect(messageOutput).toContainText('SyntaxError');
302+
await expect(
303+
popupPage.locator('[data-testid="message-output"]'),
304+
).toContainText('SyntaxError');
273305
// Verify original vats still exist
274306
const firstVatKey = Object.keys(
275307
defaultClusterConfig.vats,
@@ -323,9 +355,9 @@ test.describe('Control Panel', () => {
323355
popupPage.locator('button:text("Database Inspector")'),
324356
).toBeVisible();
325357
await popupPage.click('button:text("Database Inspector")');
326-
await expect(messageOutput).toContainText(
327-
'{"key":"vats.terminated","value":"[]"}',
328-
);
358+
await expect(
359+
popupPage.locator('[data-testid="message-output"]'),
360+
).toContainText('{"key":"vats.terminated","value":"[]"}');
329361
const v3Values = [
330362
'{"key":"e.nextPromiseId.v3","value":"2"}',
331363
'{"key":"e.nextObjectId.v3","value":"1"}',
@@ -342,64 +374,92 @@ test.describe('Control Panel', () => {
342374
'{"key":"kp3.state","value":"fulfilled"}',
343375
'{"key":"kp3.value","value"',
344376
];
345-
await expect(messageOutput).toContainText(
346-
'{"key":"kp3.refCount","value":"2"}',
347-
);
348-
await expect(messageOutput).toContainText('{"key":"vatConfig.v3","value"');
377+
await expect(
378+
popupPage.locator('[data-testid="message-output"]'),
379+
).toContainText('{"key":"kp3.refCount","value":"2"}');
380+
await expect(
381+
popupPage.locator('[data-testid="message-output"]'),
382+
).toContainText('{"key":"vatConfig.v3","value"');
349383
for (const value of v3Values) {
350-
await expect(messageOutput).toContainText(value);
384+
await expect(
385+
popupPage.locator('[data-testid="message-output"]'),
386+
).toContainText(value);
351387
}
352388
for (const value of v1ko3Values) {
353-
await expect(messageOutput).toContainText(value);
389+
await expect(
390+
popupPage.locator('[data-testid="message-output"]'),
391+
).toContainText(value);
354392
}
355393
await popupPage.click('button:text("Control Panel")');
356394
await popupPage.locator('td button:text("Terminate")').last().click();
357-
await expect(messageOutput).toContainText('Terminated vat "v3"');
395+
await expect(
396+
popupPage.locator('[data-testid="message-output"]'),
397+
).toContainText('Terminated vat "v3"');
358398
await popupPage.locator('[data-testid="clear-logs-button"]').click();
359-
await expect(messageOutput).toContainText('');
399+
await expect(
400+
popupPage.locator('[data-testid="message-output"]'),
401+
).toContainText('');
360402
await popupPage.click('button:text("Database Inspector")');
361-
await expect(messageOutput).toContainText(
362-
'{"key":"vats.terminated","value":"[\\"v3\\"]"}',
363-
);
364-
await expect(messageOutput).not.toContainText(
365-
'{"key":"vatConfig.v3","value"',
366-
);
403+
await expect(
404+
popupPage.locator('[data-testid="message-output"]'),
405+
).toContainText('{"key":"vats.terminated","value":"[\\"v3\\"]"}');
406+
await expect(
407+
popupPage.locator('[data-testid="message-output"]'),
408+
).not.toContainText('{"key":"vatConfig.v3","value"');
367409
for (const value of v3Values) {
368-
await expect(messageOutput).toContainText(value);
410+
await expect(
411+
popupPage.locator('[data-testid="message-output"]'),
412+
).toContainText(value);
369413
}
370414
await popupPage.click('button:text("Control Panel")');
371415

372416
await popupPage.click('button:text("Collect Garbage")');
373-
await expect(messageOutput).toContainText('Garbage collected');
417+
await expect(
418+
popupPage.locator('[data-testid="message-output"]'),
419+
).toContainText('Garbage collected');
374420
await popupPage.locator('[data-testid="clear-logs-button"]').click();
375-
await expect(messageOutput).toContainText('');
421+
await expect(
422+
popupPage.locator('[data-testid="message-output"]'),
423+
).toContainText('');
376424
await popupPage.click('button:text("Database Inspector")');
377425
// v3 is gone
378426
for (const value of v3Values) {
379-
await expect(messageOutput).not.toContainText(value);
427+
await expect(
428+
popupPage.locator('[data-testid="message-output"]'),
429+
).not.toContainText(value);
380430
}
381431
// ko3 reference still exists for v1
382432
for (const value of v1ko3Values) {
383-
await expect(messageOutput).toContainText(value);
433+
await expect(
434+
popupPage.locator('[data-testid="message-output"]'),
435+
).toContainText(value);
384436
}
385437
// kp3 reference dropped to 1
386-
await expect(messageOutput).toContainText(
387-
'{"key":"kp3.refCount","value":"1"}',
388-
);
438+
await expect(
439+
popupPage.locator('[data-testid="message-output"]'),
440+
).toContainText('{"key":"kp3.refCount","value":"1"}');
389441
await popupPage.click('button:text("Control Panel")');
390442
// delete v1
391443
await popupPage.locator('td button:text("Terminate")').first().click();
392-
await expect(messageOutput).toContainText('Terminated vat "v1"');
444+
await expect(
445+
popupPage.locator('[data-testid="message-output"]'),
446+
).toContainText('Terminated vat "v1"');
393447
await popupPage.click('button:text("Collect Garbage")');
394-
await expect(messageOutput).toContainText('Garbage collected');
448+
await expect(
449+
popupPage.locator('[data-testid="message-output"]'),
450+
).toContainText('Garbage collected');
395451
await popupPage.locator('[data-testid="clear-logs-button"]').click();
396-
await expect(messageOutput).toContainText('');
452+
await expect(
453+
popupPage.locator('[data-testid="message-output"]'),
454+
).toContainText('');
397455
await popupPage.click('button:text("Database Inspector")');
398-
await expect(messageOutput).toContainText(
399-
'{"key":"vats.terminated","value":"[]"}',
400-
);
456+
await expect(
457+
popupPage.locator('[data-testid="message-output"]'),
458+
).toContainText('{"key":"vats.terminated","value":"[]"}');
401459
for (const value of v1ko3Values) {
402-
await expect(messageOutput).not.toContainText(value);
460+
await expect(
461+
popupPage.locator('[data-testid="message-output"]'),
462+
).not.toContainText(value);
403463
}
404464
});
405465
});

0 commit comments

Comments
 (0)