Skip to content

Commit acc095d

Browse files
FedericoPonziFedericoPonzi
authored
Fix flaky tests on Windows (#525)
* Add retries for pdflatex installs on windows Signed-off-by: Federico Ponzi <me@fponzi.me> * test: stub parseSpec in checkModelCancel to avoid timeout error in Windows Signed-off-by: Federico Ponzi <me@fponzi.me> * Increase divergence test timeout per test Signed-off-by: Federico Ponzi <me@fponzi.me> --------- Signed-off-by: Federico Ponzi <me@fponzi.me>
1 parent e40cbbd commit acc095d

3 files changed

Lines changed: 43 additions & 37 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -46,17 +46,15 @@ jobs:
4646
if: matrix.os == 'windows-latest'
4747
shell: pwsh
4848
run: |
49-
$miktexBin = 'C:\Program Files\MiKTeX\miktex\bin\x64'
50-
if (-not (Get-Command pdflatex -ErrorAction SilentlyContinue)) {
51-
choco install miktex --yes
52-
Add-Content -Path $env:GITHUB_PATH -Value $miktexBin
53-
$env:Path = "$miktexBin;$env:Path"
54-
}
55-
if (Test-Path "$miktexBin\pdflatex.exe") {
56-
& "$miktexBin\pdflatex.exe" --version
57-
} else {
58-
pdflatex --version
49+
# choco install miktex hits flaky CTAN mirrors; retry a few times.
50+
$bin = 'C:\Program Files\MiKTeX\miktex\bin\x64'
51+
for ($i = 1; $i -le 5; $i++) {
52+
try { choco install miktex --yes --no-progress } catch {}
53+
if (Test-Path "$bin\pdflatex.exe") { break }
54+
Start-Sleep -Seconds ($i * 10)
5955
}
56+
Add-Content $env:GITHUB_PATH $bin
57+
& "$bin\pdflatex.exe" --version
6058
- name: Build
6159
run: |
6260
npm run vscode:prepublish

‎tests/suite/commands/checkModelCancel.test.ts‎

Lines changed: 24 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,61 +3,75 @@ import * as fs from 'fs';
33
import * as os from 'os';
44
import * as path from 'path';
55
import * as vscode from 'vscode';
6+
import { SanyData } from '../../../src/parsers/sany';
67

78
suite('CheckModel cancellation handling', () => {
89
const fsp = fs.promises;
910
const tla2toolsPath = require.resolve(path.resolve(__dirname, '../../../src/tla2tools'));
11+
const parseModulePath = require.resolve(path.resolve(__dirname, '../../../src/commands/parseModule'));
1012
const checkModelPath = require.resolve(path.resolve(__dirname, '../../../src/commands/checkModel'));
1113
const modelPath = require.resolve(path.resolve(__dirname, '../../../src/model/check'));
1214

1315
let originalTla2tools: typeof import('../../../src/tla2tools') | undefined;
16+
let originalParseModule: typeof import('../../../src/commands/parseModule') | undefined;
1417
let originalExecuteCommand: typeof vscode.commands.executeCommand | undefined;
1518

16-
const makeCacheEntry = (exports: unknown): NodeJS.Module => ({
17-
id: tla2toolsPath,
18-
filename: tla2toolsPath,
19+
const makeCacheEntry = (filename: string, exports: unknown): NodeJS.Module => ({
20+
id: filename,
21+
filename,
1922
loaded: true,
2023
exports,
2124
parent: null,
22-
path: tla2toolsPath,
25+
path: filename,
2326
paths: [],
2427
children: [],
2528
require,
2629
isPreloading: false,
2730
} as unknown as NodeJS.Module);
2831

2932
setup(() => {
30-
// Ensure a clean slate for module cache
3133
delete require.cache[checkModelPath];
3234
delete require.cache[tla2toolsPath];
35+
delete require.cache[parseModulePath];
3336
});
3437

3538
teardown(() => {
36-
// Restore executeCommand and cached modules
3739
if (originalExecuteCommand) {
3840
(vscode.commands as unknown as { executeCommand: typeof vscode.commands.executeCommand })
3941
.executeCommand = originalExecuteCommand;
4042
}
4143
if (originalTla2tools) {
42-
require.cache[tla2toolsPath] = makeCacheEntry(originalTla2tools);
44+
require.cache[tla2toolsPath] = makeCacheEntry(tla2toolsPath, originalTla2tools);
4345
} else {
4446
delete require.cache[tla2toolsPath];
4547
}
48+
if (originalParseModule) {
49+
require.cache[parseModulePath] = makeCacheEntry(parseModulePath, originalParseModule);
50+
} else {
51+
delete require.cache[parseModulePath];
52+
}
4653
delete require.cache[checkModelPath];
4754
});
4855

4956
test('does not leave TLC running context when launch is cancelled', async function() {
5057
this.timeout(5000);
5158
originalTla2tools = await import(tla2toolsPath);
59+
originalParseModule = await import(parseModulePath);
5260

53-
// Stub runTlc to simulate user cancelling the options prompt
5461
const stubbedTla2tools = {
5562
...originalTla2tools,
5663
runTlc: async () => undefined,
5764
} as typeof import('../../../src/tla2tools');
58-
require.cache[tla2toolsPath] = makeCacheEntry(stubbedTla2tools);
65+
require.cache[tla2toolsPath] = makeCacheEntry(tla2toolsPath, stubbedTla2tools);
66+
67+
// Stub parseSpec to avoid spawning Java SANY, which can exceed the
68+
// 5s timeout during cold JVM startup on Windows CI runners.
69+
const stubbedParseModule = {
70+
...originalParseModule,
71+
parseSpec: async () => new SanyData(),
72+
} as typeof import('../../../src/commands/parseModule');
73+
require.cache[parseModulePath] = makeCacheEntry(parseModulePath, stubbedParseModule);
5974

60-
// Capture context updates
6175
const contextValues: Record<string, unknown> = {};
6276
originalExecuteCommand = vscode.commands.executeCommand;
6377
const stubExecuteCommand: typeof vscode.commands.executeCommand =

‎tests/suite/commands/divergence.test.ts‎

Lines changed: 11 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import {
88
} from '../../../src/parsers/sany';
99

1010
const FIXTURES_DIR = path.join(__dirname, '..', '..', '..', '..', 'tests', 'fixtures');
11+
const TEST_TIMEOUT_MS = 5 * 60 * 1000;
1112

1213
suite('PlusCal Divergence Detection Integration Tests', () => {
1314
let tempDir: string;
@@ -23,7 +24,7 @@ suite('PlusCal Divergence Detection Integration Tests', () => {
2324
});
2425

2526
test('SANY detects no divergence on clean file with matching checksums', async function () {
26-
this.timeout(15000);
27+
this.timeout(TEST_TIMEOUT_MS);
2728
const filePath = copyFixture('DivergenceTest.tla');
2829
const output = await runSanyOnFile(filePath);
2930
assert.ok(output.includes('Semantic processing of module DivergenceTest'),
@@ -34,7 +35,7 @@ suite('PlusCal Divergence Detection Integration Tests', () => {
3435
});
3536

3637
test('SANY detects TLA+ translation divergence when translation is modified', async function () {
37-
this.timeout(15000);
38+
this.timeout(TEST_TIMEOUT_MS);
3839
const filePath = copyFixture('DivergenceTest.tla');
3940

4041
let content = fs.readFileSync(filePath, 'utf-8');
@@ -47,7 +48,7 @@ suite('PlusCal Divergence Detection Integration Tests', () => {
4748
});
4849

4950
test('SANY detects PlusCal divergence when algorithm is modified', async function () {
50-
this.timeout(15000);
51+
this.timeout(TEST_TIMEOUT_MS);
5152
const filePath = copyFixture('DivergenceTest.tla');
5253

5354
let content = fs.readFileSync(filePath, 'utf-8');
@@ -60,7 +61,7 @@ suite('PlusCal Divergence Detection Integration Tests', () => {
6061
});
6162

6263
test('SANY detects both divergences when algorithm and translation are modified', async function () {
63-
this.timeout(15000);
64+
this.timeout(TEST_TIMEOUT_MS);
6465
const filePath = copyFixture('DivergenceTest.tla');
6566

6667
let content = fs.readFileSync(filePath, 'utf-8');
@@ -74,7 +75,7 @@ suite('PlusCal Divergence Detection Integration Tests', () => {
7475
});
7576

7677
test('Pre-model-check: divergence detected on modified TLA+ translation', async function () {
77-
this.timeout(15000);
78+
this.timeout(TEST_TIMEOUT_MS);
7879
const filePath = copyFixture('DivergenceTest.tla');
7980

8081
let content = fs.readFileSync(filePath, 'utf-8');
@@ -90,7 +91,7 @@ suite('PlusCal Divergence Detection Integration Tests', () => {
9091
});
9192

9293
test('Pre-model-check: no divergence on clean file', async function () {
93-
this.timeout(15000);
94+
this.timeout(TEST_TIMEOUT_MS);
9495
const filePath = copyFixture('DivergenceTest.tla');
9596
const content = fs.readFileSync(filePath, 'utf-8');
9697

@@ -103,7 +104,7 @@ suite('PlusCal Divergence Detection Integration Tests', () => {
103104
});
104105

105106
test('Pre-model-check: skipped when no checksums', async function () {
106-
this.timeout(15000);
107+
this.timeout(TEST_TIMEOUT_MS);
107108
const filePath = copyFixture('DivergenceTest.tla');
108109

109110
// Remove checksums from the BEGIN TRANSLATION line
@@ -119,7 +120,7 @@ suite('PlusCal Divergence Detection Integration Tests', () => {
119120
});
120121

121122
test('SANY detects divergence in an imported module', async function () {
122-
this.timeout(15000);
123+
this.timeout(TEST_TIMEOUT_MS);
123124
// Copy the fixture as an imported module and break its TLA+ translation.
124125
const helperPath = path.join(tempDir, 'DivHelper.tla');
125126
let helperContent = fs.readFileSync(path.join(FIXTURES_DIR, 'DivergenceTest.tla'), 'utf-8');
@@ -159,19 +160,12 @@ suite('PlusCal Divergence Detection Integration Tests', () => {
159160
capturedOutput += chunk.toString();
160161
});
161162

162-
await new Promise<void>((resolve, reject) => {
163+
await new Promise<void>((resolve) => {
163164
if (procInfo.mergedOutput.readableEnded) {
164165
resolve();
165166
return;
166167
}
167-
const timer = setTimeout(
168-
() => reject(new Error('SANY did not finish in time')),
169-
10000
170-
);
171-
procInfo.mergedOutput.once('end', () => {
172-
clearTimeout(timer);
173-
resolve();
174-
});
168+
procInfo.mergedOutput.once('end', resolve);
175169
});
176170

177171
return capturedOutput;

0 commit comments

Comments
 (0)