Skip to content

Commit 254dc6d

Browse files
apstenku123apstenku123
authored
Merge pull request #3 from DatasunriseOU/codex/self-hosted-workflows
CI: move all workflows to repository runners
2 parents 9bf8afa + c121b80 commit 254dc6d

27 files changed

Lines changed: 2480 additions & 156 deletions

‎.github/workflows/ci-self-hosted.yml‎

Lines changed: 37 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -11,57 +11,64 @@ concurrency:
1111
group: cppmega-mlx-ci-${{ github.ref }}
1212
cancel-in-progress: true
1313

14+
permissions:
15+
contents: read
16+
1417
jobs:
1518
mac-contracts:
1619
name: macOS MLX data, model, and eval contracts
20+
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
1721
runs-on: [self-hosted, macOS, ARM64, cppmega-mlx-macos]
1822
timeout-minutes: 30
1923
steps:
20-
- uses: actions/checkout@v4
24+
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
2125
- name: Run focused MLX contract suite
2226
shell: bash
2327
run: |
2428
set -euo pipefail
2529
python_bin=/Volumes/external/sources/cppmega.mlx/.venv/bin/python
2630
test -x "${python_bin}"
27-
"${python_bin}" -m pytest -q \
28-
tests/test_ast_fim.py \
29-
tests/test_audit_sidecar_parquet.py \
30-
tests/test_cpp_jsonl_generation_compile_eval.py \
31-
tests/test_data_package_imports.py \
32-
tests/test_domain_graph_routes.py \
33-
tests/test_eval_domain_routed_codegen.py \
34-
tests/test_inference_generation.py \
35-
tests/test_megatron_indexed.py \
36-
tests/test_pack_enriched_rows.py \
37-
tests/test_streaming_conveyor_progress.py \
38-
tests/test_process_commits_fail_loud.py \
39-
tests/test_repair_packed_document_boundaries.py
40-
git diff --check
31+
"${python_bin}" scripts/run_self_hosted_ci.py lane \
32+
--lane macos-mlx \
33+
--python "${python_bin}" \
34+
--repo-root "${GITHUB_WORKSPACE}" \
35+
--receipt-dir "${RUNNER_TEMP}/cppmega-mlx-ci-macos-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \
36+
--timeout-seconds 1500
37+
- name: Upload macOS runner receipt
38+
if: always()
39+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
40+
with:
41+
name: self-hosted-ci-macos-${{ github.run_id }}-${{ github.run_attempt }}
42+
path: ${{ runner.temp }}/cppmega-mlx-ci-macos-${{ github.run_id }}-${{ github.run_attempt }}
43+
if-no-files-found: error
44+
retention-days: 14
4145

4246
linux-portable:
4347
name: Linux portable syntax and conveyor contracts
48+
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
4449
runs-on: [self-hosted, Linux, X64, cppmega-mlx]
4550
timeout-minutes: 20
4651
steps:
47-
- uses: actions/checkout@v4
48-
- uses: actions/setup-python@v5
52+
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
53+
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
4954
with:
5055
python-version: "3.13"
51-
- name: Install portable test dependencies
52-
run: python -m pip install --disable-pip-version-check pytest numpy pyarrow tokenizers zstandard
5356
- name: Run portable script contracts
5457
shell: bash
5558
run: |
5659
set -euo pipefail
57-
# cppmega.mlx imports Apple's MLX runtime at package import time. The
58-
# Linux runner exercises only portable scripts and deliberately skips
59-
# the macOS-only root conftest/package initialization.
60-
python -m pytest -q --noconftest \
61-
tests/test_audit_sidecar_parquet.py \
62-
tests/test_data_package_imports.py \
63-
tests/test_streaming_conveyor_progress.py \
64-
tests/test_process_commits_fail_loud.py \
65-
tests/test_repair_packed_document_boundaries.py
66-
python -m compileall -q scripts tools/clang_indexer
67-
git diff --check
60+
python scripts/run_self_hosted_ci.py lane \
61+
--lane linux-portable \
62+
--python python \
63+
--repo-root "${GITHUB_WORKSPACE}" \
64+
--receipt-dir "${RUNNER_TEMP}/cppmega-mlx-ci-linux-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \
65+
--timeout-seconds 900 \
66+
--bootstrap-portable
67+
- name: Upload Linux runner receipt
68+
if: always()
69+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
70+
with:
71+
name: self-hosted-ci-linux-${{ github.run_id }}-${{ github.run_attempt }}
72+
path: ${{ runner.temp }}/cppmega-mlx-ci-linux-${{ github.run_id }}-${{ github.run_attempt }}
73+
if-no-files-found: error
74+
retention-days: 14

‎.github/workflows/cuda-lane.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,10 @@ jobs:
1212
runs-on: [self-hosted, cuda]
1313
timeout-minutes: 30
1414
steps:
15-
- uses: actions/checkout@v4
15+
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
1616

1717
- name: Set up Python
18-
uses: actions/setup-python@v5
18+
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
1919
with:
2020
python-version: "3.13"
2121

‎.github/workflows/e2e-matrix.yml‎

Lines changed: 104 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,16 @@ on:
1010
- "tests/fixtures/build_e2e_matrix.py"
1111
- "tests/fixtures/tokenizers/**"
1212
- "tests/fixtures/parquet/**"
13+
- "scripts/install_self_hosted_e2e_python.sh"
1314
- ".github/workflows/e2e-matrix.yml"
1415
pull_request:
1516
branches: [main]
1617
paths:
1718
- "cppmega_v4/**"
1819
- "vbgui/**"
1920
- "tests/fixtures/build_e2e_matrix.py"
21+
- "scripts/install_self_hosted_e2e_python.sh"
22+
- ".github/workflows/e2e-matrix.yml"
2023
schedule:
2124
# Nightly full run — includes the mini-train matrix on macOS.
2225
- cron: "0 4 * * *"
@@ -26,34 +29,50 @@ concurrency:
2629
group: e2e-matrix-${{ github.ref }}
2730
cancel-in-progress: true
2831

32+
env:
33+
CPPMEGA_MLX_LM_CHECKOUT: /Volumes/external/sources/mlx-lm
34+
CPPMEGA_MLX_LM_COMMIT: 8618587943181787d33bac4468d3088e80202b3f
35+
2936
jobs:
30-
# On every push/PR: smoke matrix (912 cells, ubuntu, no real MLX
31-
# training — backend just verifies). Quick gate, ~10 min.
37+
# On every push/PR: smoke matrix (912 cells). The backend imports MLX, so
38+
# this runs on the repository's persistent Apple Silicon runner.
3239
preset-matrix:
33-
name: Preset matrix (smoke, ubuntu)
34-
runs-on: ubuntu-latest
35-
timeout-minutes: 25
40+
name: Preset matrix (smoke, self-hosted macOS)
41+
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
42+
runs-on: [self-hosted, macOS, ARM64, cppmega-mlx-macos]
43+
# The matrix is one spec file. --fully-parallel is required or Playwright
44+
# assigns all 912 cells to shard 1 and leaves the other shards empty.
45+
timeout-minutes: 20
3646
strategy:
3747
fail-fast: false
3848
matrix:
3949
shard: [1, 2, 3, 4]
4050
steps:
41-
- uses: actions/checkout@v4
42-
- uses: actions/setup-node@v4
51+
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
52+
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
4353
with:
4454
node-version: "22"
45-
cache: "npm"
46-
cache-dependency-path: vbgui/package-lock.json
47-
- uses: actions/setup-python@v5
48-
with:
49-
python-version: "3.13"
50-
- name: Python deps
55+
- name: Create isolated MLX Python
5156
run: |
52-
python -m pip install --upgrade pip
53-
pip install -e ".[gui,parquet,widget]"
54-
pip install jsonschema pyarrow tokenizers anywidget
57+
base_python=/Volumes/external/sources/cppmega.mlx/.venv/bin/python
58+
test -x "$base_python"
59+
job_venv="$(mktemp -d "$RUNNER_TEMP/cppmega-mlx-e2e-${GITHUB_RUN_ID}-${GITHUB_JOB}.XXXXXX")"
60+
"$base_python" -m venv "$job_venv"
61+
echo "VBGUI_E2E_PYTHON=$job_venv/bin/python" >> "$GITHUB_ENV"
62+
echo "VBGUI_E2E_VENV=$job_venv" >> "$GITHUB_ENV"
63+
echo "PYTHONPATH=" >> "$GITHUB_ENV"
64+
echo "PYTHONNOUSERSITE=1" >> "$GITHUB_ENV"
65+
- name: Allocate isolated E2E ports
66+
run: |
67+
port_base=$((22000 + (GITHUB_RUN_ID % 1000) * 20))
68+
port_offset=${{ matrix.shard }}
69+
echo "VBGUI_E2E_BACKEND_PORT=$((port_base + port_offset))" >> "$GITHUB_ENV"
70+
echo "VBGUI_E2E_FRONTEND_PORT=$((port_base + 10 + port_offset))" >> "$GITHUB_ENV"
71+
- name: Python deps
72+
run: scripts/install_self_hosted_e2e_python.sh
5573
- name: Generate fixtures
56-
run: python tests/fixtures/build_e2e_matrix.py
74+
run: |
75+
"$VBGUI_E2E_PYTHON" tests/fixtures/build_e2e_matrix.py
5776
- name: Vbgui install + typecheck
5877
working-directory: vbgui
5978
env:
@@ -63,7 +82,7 @@ jobs:
6382
npm run typecheck
6483
- name: Playwright browsers
6584
working-directory: vbgui
66-
run: npx playwright install --with-deps chromium
85+
run: npx playwright install chromium
6786
- name: Run preset matrix shard ${{ matrix.shard }}/4
6887
working-directory: vbgui
6988
env:
@@ -72,42 +91,59 @@ jobs:
7291
run: |
7392
npx playwright test --config=e2e/playwright.config.ts \
7493
e2e/scenarios/02_preset_matrix.spec.ts \
94+
--fully-parallel \
95+
--global-timeout=720000 \
7596
--shard=${{ matrix.shard }}/4
7697
- name: Upload artefacts
7798
if: always()
78-
uses: actions/upload-artifact@v4
99+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
79100
with:
80101
name: preset-matrix-shard-${{ matrix.shard }}
81102
path: |
82103
vbgui/e2e/test-results/
83104
vbgui/e2e/screenshots/
84105
vbgui/e2e/logs/
85106
retention-days: 3
107+
- name: Remove isolated MLX Python
108+
if: always()
109+
run: rm -rf "$VBGUI_E2E_VENV"
86110

87111
# Specialised + canvas smoke scenarios — run on every push (lightweight).
88112
specialised:
89113
name: Specialised + canvas smoke
90-
runs-on: ubuntu-latest
114+
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
115+
runs-on: [self-hosted, macOS, ARM64, cppmega-mlx-macos]
91116
timeout-minutes: 15
92117
steps:
93-
- uses: actions/checkout@v4
94-
- uses: actions/setup-node@v4
95-
with: { node-version: "22", cache: "npm",
96-
cache-dependency-path: vbgui/package-lock.json }
97-
- uses: actions/setup-python@v5
98-
with: { python-version: "3.13" }
118+
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
119+
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
120+
with: { node-version: "22" }
121+
- name: Create isolated MLX Python
122+
run: |
123+
base_python=/Volumes/external/sources/cppmega.mlx/.venv/bin/python
124+
test -x "$base_python"
125+
job_venv="$(mktemp -d "$RUNNER_TEMP/cppmega-mlx-e2e-${GITHUB_RUN_ID}-${GITHUB_JOB}.XXXXXX")"
126+
"$base_python" -m venv "$job_venv"
127+
echo "VBGUI_E2E_PYTHON=$job_venv/bin/python" >> "$GITHUB_ENV"
128+
echo "VBGUI_E2E_VENV=$job_venv" >> "$GITHUB_ENV"
129+
echo "PYTHONPATH=" >> "$GITHUB_ENV"
130+
echo "PYTHONNOUSERSITE=1" >> "$GITHUB_ENV"
131+
- name: Allocate isolated E2E ports
132+
run: |
133+
port_base=$((22000 + (GITHUB_RUN_ID % 1000) * 20))
134+
port_offset=8
135+
echo "VBGUI_E2E_BACKEND_PORT=$((port_base + port_offset))" >> "$GITHUB_ENV"
136+
echo "VBGUI_E2E_FRONTEND_PORT=$((port_base + 10 + port_offset))" >> "$GITHUB_ENV"
137+
- run: scripts/install_self_hosted_e2e_python.sh
99138
- run: |
100-
python -m pip install --upgrade pip
101-
pip install -e ".[gui,parquet,widget]"
102-
pip install jsonschema pyarrow tokenizers anywidget
103-
- run: python tests/fixtures/build_e2e_matrix.py
139+
"$VBGUI_E2E_PYTHON" tests/fixtures/build_e2e_matrix.py
104140
- working-directory: vbgui
105141
env: { NODE_ENV: development }
106142
run: |
107143
npm ci
108144
npm run typecheck
109145
- working-directory: vbgui
110-
run: npx playwright install --with-deps chromium
146+
run: npx playwright install chromium
111147
- working-directory: vbgui
112148
env: { NODE_ENV: development, CI: "1" }
113149
run: |
@@ -118,75 +154,92 @@ jobs:
118154
e2e/scenarios/06_sharding_proposals.spec.ts \
119155
e2e/scenarios/07_gotchas.spec.ts
120156
- if: always()
121-
uses: actions/upload-artifact@v4
157+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
122158
with:
123159
name: specialised-results
124160
path: |
125161
vbgui/e2e/test-results/
126162
vbgui/e2e/screenshots/
127163
vbgui/e2e/logs/
128164
retention-days: 3
165+
- name: Remove isolated MLX Python
166+
if: always()
167+
run: rm -rf "$VBGUI_E2E_VENV"
129168

130169
# Mini-train matrix (192 cells, real mlx) — macOS only, nightly.
131170
mini-train-matrix:
132171
name: Mini-train matrix (macOS, nightly)
133172
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
134-
runs-on: macos-latest
173+
runs-on: [self-hosted, macOS, ARM64, cppmega-mlx-macos]
135174
timeout-minutes: 60
136175
steps:
137-
- uses: actions/checkout@v4
138-
- uses: actions/setup-node@v4
139-
with: { node-version: "22", cache: "npm",
140-
cache-dependency-path: vbgui/package-lock.json }
141-
- uses: actions/setup-python@v5
142-
with: { python-version: "3.13" }
176+
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
177+
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
178+
with: { node-version: "22" }
179+
- name: Create isolated MLX Python
180+
run: |
181+
base_python=/Volumes/external/sources/cppmega.mlx/.venv/bin/python
182+
test -x "$base_python"
183+
job_venv="$(mktemp -d "$RUNNER_TEMP/cppmega-mlx-e2e-${GITHUB_RUN_ID}-${GITHUB_JOB}.XXXXXX")"
184+
"$base_python" -m venv "$job_venv"
185+
echo "VBGUI_E2E_PYTHON=$job_venv/bin/python" >> "$GITHUB_ENV"
186+
echo "VBGUI_E2E_VENV=$job_venv" >> "$GITHUB_ENV"
187+
echo "PYTHONPATH=" >> "$GITHUB_ENV"
188+
echo "PYTHONNOUSERSITE=1" >> "$GITHUB_ENV"
189+
- name: Allocate isolated E2E ports
190+
run: |
191+
port_base=$((22000 + (GITHUB_RUN_ID % 1000) * 20))
192+
port_offset=9
193+
echo "VBGUI_E2E_BACKEND_PORT=$((port_base + port_offset))" >> "$GITHUB_ENV"
194+
echo "VBGUI_E2E_FRONTEND_PORT=$((port_base + 10 + port_offset))" >> "$GITHUB_ENV"
195+
- run: scripts/install_self_hosted_e2e_python.sh
143196
- run: |
144-
python -m pip install --upgrade pip
145-
pip install -e ".[gui,parquet,widget]"
146-
pip install jsonschema pyarrow tokenizers anywidget mlx mlx-lm
147-
- run: python tests/fixtures/build_e2e_matrix.py
197+
"$VBGUI_E2E_PYTHON" tests/fixtures/build_e2e_matrix.py
148198
- working-directory: vbgui
149199
env: { NODE_ENV: development }
150200
run: |
151201
npm ci
152202
npm run typecheck
153203
- working-directory: vbgui
154-
run: npx playwright install --with-deps chromium
204+
run: npx playwright install chromium
155205
- working-directory: vbgui
156206
env: { NODE_ENV: development, CI: "1" }
157207
run: |
158208
npx playwright test --config=e2e/playwright.config.ts \
159209
e2e/scenarios/03_train_matrix.spec.ts
160210
- if: always()
161-
uses: actions/upload-artifact@v4
211+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
162212
with:
163213
name: mini-train-results
164214
path: |
165215
vbgui/e2e/test-results/
166216
vbgui/e2e/screenshots/
167217
vbgui/e2e/logs/
168218
retention-days: 7
219+
- name: Remove isolated MLX Python
220+
if: always()
221+
run: rm -rf "$VBGUI_E2E_VENV"
169222

170223
# Matrix report — aggregates shard artefacts into one Markdown
171224
# summary, posted as a build summary + uploaded as standalone artefact.
172225
matrix-report:
173226
name: Matrix report
174227
needs: [preset-matrix, specialised]
175-
runs-on: ubuntu-latest
176-
if: always()
228+
runs-on: [self-hosted, Linux, X64, cppmega-mlx]
229+
if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}
177230
steps:
178-
- uses: actions/checkout@v4
179-
- uses: actions/download-artifact@v4
231+
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
232+
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
180233
with: { path: artifacts }
181-
- uses: actions/setup-python@v5
234+
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
182235
with: { python-version: "3.13" }
183236
- name: Build matrix report
184237
run: |
185238
python tools/build_e2e_matrix_report.py \
186239
--artifacts artifacts \
187240
--output e2e_matrix_report.md
188241
cat e2e_matrix_report.md >> "$GITHUB_STEP_SUMMARY"
189-
- uses: actions/upload-artifact@v4
242+
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
190243
with:
191244
name: e2e_matrix_report
192245
path: e2e_matrix_report.md

0 commit comments

Comments
 (0)