Skip to content

Commit 22d5fd9

Browse files
sorccuclaude
sorccu
and
claude
authored
test(e2e): fix slow Windows e2e (oclif PowerShell spawn) and add per-OS CI gates (#1357)
* test(e2e): set SHELL so oclif skips per-command PowerShell shell detection @oclif/core 4.11.5+ resolves the shell during Config.load() via getShell(), which on Windows runs determineWindowsShell() — an execSync of `powershell.exe -Command "(Get-CimInstance Win32_Process ...)"` on every command. CPU profiling showed that spawn dominating each Windows e2e command (~2.6s under suite load), which is what made the Windows e2e suite time out. getShell() short-circuits when SHELL is set, so setting it in the e2e harness skips the spawn. A/B on Windows: account-plan spec 80s -> 43s, and CI Windows e2e timeouts went 268 -> 0. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * ci: split test workflow per-OS and add CI gate jobs Move the Windows legs onto blacksmith-4vcpu-windows-2025 and split the three mixed-OS matrix jobs into per-OS jobs so each OS can be gated independently. The checkly e2e job is sharded into 4 on Windows (the slow leg); Ubuntu runs the suite in one job. Add two aggregate gate jobs, "CI gate - ubuntu" and "CI gate - windows", that fail if any of their OS's jobs fail or cancel (skipped is allowed for the push-to-main case). These are the single status checks to require in branch protection: one point of gating for the sharded Windows e2e, and the ability to make only Ubuntu required without touching individual job names. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 4b338b4 commit 22d5fd9

2 files changed

Lines changed: 133 additions & 22 deletions

File tree

‎.github/workflows/test.yml‎

Lines changed: 127 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,13 @@ jobs:
3333
# Test that the packaging works as well
3434
- run: pnpm -r pack
3535
- run: pnpm run lint
36-
test:
37-
strategy:
38-
fail-fast: false
39-
matrix:
40-
os: [ubuntu-latest, windows-latest-x64]
41-
name: test - ${{ matrix.os }}
42-
runs-on: ${{ matrix.os }}
36+
37+
# ---------------------------------------------------------------------------
38+
# Unit tests
39+
# ---------------------------------------------------------------------------
40+
test-ubuntu:
41+
name: test - ubuntu
42+
runs-on: ubuntu-latest
4343
steps:
4444
- uses: actions/checkout@v3
4545
- uses: pnpm/action-setup@v5
@@ -55,20 +55,41 @@ jobs:
5555
- name: Save LLM rules as an artifact
5656
uses: actions/upload-artifact@v4
5757
with:
58-
name: llm-rules-test-${{ matrix.os }}
58+
name: llm-rules-test-ubuntu
5959
if-no-files-found: error
6060
path: packages/cli/dist/ai-context/*
6161
retention-days: 1
62-
test-e2e-checkly:
63-
# Only on PRs — these hit the live Checkly API. The push-to-main run exists
64-
# solely to seed the dependency cache, which lint + test already cover.
62+
test-windows:
63+
name: test - windows
64+
runs-on: blacksmith-4vcpu-windows-2025
65+
steps:
66+
- uses: actions/checkout@v3
67+
- uses: pnpm/action-setup@v5
68+
with:
69+
version: 10
70+
- uses: actions/setup-node@v4
71+
with:
72+
node-version: '20.x'
73+
cache: "pnpm"
74+
- run: pnpm install --frozen-lockfile
75+
- run: pnpm run prepack
76+
- run: pnpm run test
77+
- name: Save LLM rules as an artifact
78+
uses: actions/upload-artifact@v4
79+
with:
80+
name: llm-rules-test-windows
81+
if-no-files-found: error
82+
path: packages/cli/dist/ai-context/*
83+
retention-days: 1
84+
85+
# ---------------------------------------------------------------------------
86+
# checkly e2e — hits the live Checkly API (PRs only). Windows is sharded
87+
# because it is the slow leg; Ubuntu runs the whole suite in one job.
88+
# ---------------------------------------------------------------------------
89+
test-e2e-checkly-ubuntu:
90+
name: e2e - checkly - ubuntu
6591
if: github.event_name == 'pull_request'
66-
strategy:
67-
fail-fast: false
68-
matrix:
69-
os: [ubuntu-latest, windows-latest-x64]
70-
name: e2e - checkly - ${{ matrix.os }}
71-
runs-on: ${{ matrix.os }}
92+
runs-on: ubuntu-latest
7293
steps:
7394
- uses: actions/checkout@v3
7495
- uses: pnpm/action-setup@v5
@@ -87,15 +108,55 @@ jobs:
87108
CHECKLY_API_KEY: ${{ secrets.E2E_CHECKLY_API_KEY }}
88109
CHECKLY_EMPTY_ACCOUNT_ID: ${{ secrets.E2E_EMPTY_CHECKLY_ACCOUNT_ID }}
89110
CHECKLY_EMPTY_API_KEY: ${{ secrets.E2E_EMPTY_CHECKLY_API_KEY }}
90-
test-e2e-create-checkly:
91-
# Only on PRs — see test-e2e-checkly above.
111+
test-e2e-checkly-windows:
112+
name: e2e - checkly - windows (${{ matrix.shard }})
92113
if: github.event_name == 'pull_request'
93114
strategy:
94115
fail-fast: false
95116
matrix:
96-
os: [ubuntu-latest, windows-latest-x64]
97-
name: e2e - create-checkly - ${{ matrix.os }}
98-
runs-on: ${{ matrix.os }}
117+
shard: ['1/4', '2/4', '3/4', '4/4']
118+
runs-on: blacksmith-4vcpu-windows-2025
119+
steps:
120+
- uses: actions/checkout@v3
121+
- uses: pnpm/action-setup@v5
122+
with:
123+
version: 10
124+
- uses: actions/setup-node@v4
125+
with:
126+
node-version: '20.x'
127+
cache: "pnpm"
128+
- run: pnpm install --frozen-lockfile
129+
- run: pnpm run prepack
130+
- run: pnpm --filter checkly run test:e2e -- --shard=${{ matrix.shard }}
131+
env:
132+
CHECKLY_ACCOUNT_NAME: ${{ secrets.E2E_CHECKLY_ACCOUNT_NAME }}
133+
CHECKLY_ACCOUNT_ID: ${{ secrets.E2E_CHECKLY_ACCOUNT_ID }}
134+
CHECKLY_API_KEY: ${{ secrets.E2E_CHECKLY_API_KEY }}
135+
CHECKLY_EMPTY_ACCOUNT_ID: ${{ secrets.E2E_EMPTY_CHECKLY_ACCOUNT_ID }}
136+
CHECKLY_EMPTY_API_KEY: ${{ secrets.E2E_EMPTY_CHECKLY_API_KEY }}
137+
138+
# ---------------------------------------------------------------------------
139+
# create-checkly e2e (PRs only)
140+
# ---------------------------------------------------------------------------
141+
test-e2e-create-checkly-ubuntu:
142+
name: e2e - create-checkly - ubuntu
143+
if: github.event_name == 'pull_request'
144+
runs-on: ubuntu-latest
145+
steps:
146+
- uses: actions/checkout@v3
147+
- uses: pnpm/action-setup@v5
148+
with:
149+
version: 10
150+
- uses: actions/setup-node@v4
151+
with:
152+
node-version: '20.x'
153+
cache: "pnpm"
154+
- run: pnpm install --frozen-lockfile
155+
- run: pnpm --filter create-checkly run test:e2e
156+
test-e2e-create-checkly-windows:
157+
name: e2e - create-checkly - windows
158+
if: github.event_name == 'pull_request'
159+
runs-on: blacksmith-4vcpu-windows-2025
99160
steps:
100161
- uses: actions/checkout@v3
101162
- uses: pnpm/action-setup@v5
@@ -107,3 +168,47 @@ jobs:
107168
cache: "pnpm"
108169
- run: pnpm install --frozen-lockfile
109170
- run: pnpm --filter create-checkly run test:e2e
171+
172+
# ---------------------------------------------------------------------------
173+
# Gates — one aggregate status check per OS. These are the jobs to require in
174+
# branch protection: a single check for Windows (which fans out into 4 e2e
175+
# shards) and a single check for Ubuntu, so Windows can be made non-required
176+
# independently of Ubuntu without touching individual job names.
177+
# ---------------------------------------------------------------------------
178+
gate-ubuntu:
179+
name: CI gate - ubuntu
180+
if: always()
181+
needs:
182+
- lint
183+
- test-ubuntu
184+
- test-e2e-checkly-ubuntu
185+
- test-e2e-create-checkly-ubuntu
186+
runs-on: ubuntu-latest
187+
steps:
188+
- name: Verify all Ubuntu jobs succeeded
189+
run: |
190+
results="${{ join(needs.*.result, ', ') }}"
191+
echo "Ubuntu job results: $results"
192+
# 'skipped' is allowed (e2e jobs are skipped on push-to-main); only
193+
# 'failure'/'cancelled' should fail the gate.
194+
if [[ "$results" == *failure* || "$results" == *cancelled* ]]; then
195+
echo "::error::One or more Ubuntu jobs did not succeed"
196+
exit 1
197+
fi
198+
gate-windows:
199+
name: CI gate - windows
200+
if: always()
201+
needs:
202+
- test-windows
203+
- test-e2e-checkly-windows
204+
- test-e2e-create-checkly-windows
205+
runs-on: ubuntu-latest
206+
steps:
207+
- name: Verify all Windows jobs succeeded
208+
run: |
209+
results="${{ join(needs.*.result, ', ') }}"
210+
echo "Windows job results: $results"
211+
if [[ "$results" == *failure* || "$results" == *cancelled* ]]; then
212+
echo "::error::One or more Windows jobs did not succeed"
213+
exit 1
214+
fi

‎packages/cli/e2e/run-checkly.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,12 @@ export function checklyEnv (overrides?: {
2929
CHECKLY_CLI_VERSION: cliVersion,
3030
CHECKLY_E2E_PROMPTS_INJECTIONS: promptsInjection?.length ? JSON.stringify(promptsInjection) : undefined,
3131
CHECKLY_E2E_DISABLE_FANCY_OUTPUT: '1',
32+
// Set SHELL so @oclif/core's getShell() short-circuits instead of running
33+
// determineWindowsShell(), which spawns `powershell.exe -Command Get-CimInstance`
34+
// on every command. That spawn costs seconds per command on Windows and is the
35+
// dominant cost of the Windows e2e suite. On Linux SHELL is already set, so this
36+
// only changes behaviour on Windows where it would otherwise be unset.
37+
SHELL: process.env.SHELL ?? 'powershell.exe',
3238
...env,
3339
}
3440
}

0 commit comments

Comments
 (0)