E2E Coverage Matrix #450
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Coverage Matrix | |
| on: | |
| push: | |
| branches: [main] | |
| paths: | |
| - "cppmega_v4/**" | |
| - "cppmega_mlx/**" | |
| - "vbgui/**" | |
| - "tests/fixtures/build_e2e_matrix.py" | |
| - "tests/fixtures/tokenizers/**" | |
| - "tests/fixtures/parquet/**" | |
| - ".github/workflows/e2e-matrix.yml" | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| - "cppmega_v4/**" | |
| - "vbgui/**" | |
| - "tests/fixtures/build_e2e_matrix.py" | |
| schedule: | |
| # Nightly full run — includes the mini-train matrix on macOS. | |
| - cron: "0 4 * * *" | |
| workflow_dispatch: | |
| concurrency: | |
| group: e2e-matrix-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| # On every push/PR: smoke matrix (912 cells). The backend imports MLX, so | |
| # this runs on the repository's persistent Apple Silicon runner. | |
| preset-matrix: | |
| name: Preset matrix (smoke, self-hosted macOS) | |
| if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} | |
| runs-on: [self-hosted, macOS, ARM64, cppmega-mlx-macos] | |
| timeout-minutes: 25 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: [1, 2, 3, 4] | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 | |
| with: | |
| node-version: "22" | |
| cache: "npm" | |
| cache-dependency-path: vbgui/package-lock.json | |
| - name: Create isolated MLX Python | |
| run: | | |
| base_python=/Volumes/external/sources/cppmega.mlx/.venv/bin/python | |
| test -x "$base_python" | |
| job_venv="$(mktemp -d "$RUNNER_TEMP/cppmega-mlx-e2e-${GITHUB_RUN_ID}-${GITHUB_JOB}.XXXXXX")" | |
| "$base_python" -m venv "$job_venv" | |
| echo "VBGUI_E2E_PYTHON=$job_venv/bin/python" >> "$GITHUB_ENV" | |
| echo "VBGUI_E2E_VENV=$job_venv" >> "$GITHUB_ENV" | |
| echo "PYTHONPATH=" >> "$GITHUB_ENV" | |
| echo "PYTHONNOUSERSITE=1" >> "$GITHUB_ENV" | |
| - name: Python deps | |
| run: | | |
| "$VBGUI_E2E_PYTHON" -m pip install --disable-pip-version-check \ | |
| -e ".[gui,parquet,widget]" | |
| "$VBGUI_E2E_PYTHON" - <<'PY' | |
| from cppmega_mlx.training.native_optim import status | |
| native_status = status() | |
| if not native_status.get("available", False): | |
| raise RuntimeError(native_status.get("reason", "native optimizer extension unavailable")) | |
| PY | |
| - name: Generate fixtures | |
| run: | | |
| "$VBGUI_E2E_PYTHON" tests/fixtures/build_e2e_matrix.py | |
| - name: Vbgui install + typecheck | |
| working-directory: vbgui | |
| env: | |
| NODE_ENV: development | |
| run: | | |
| npm ci | |
| npm run typecheck | |
| - name: Playwright browsers | |
| working-directory: vbgui | |
| run: npx playwright install chromium | |
| - name: Run preset matrix shard ${{ matrix.shard }}/4 | |
| working-directory: vbgui | |
| env: | |
| NODE_ENV: development | |
| CI: "1" | |
| run: | | |
| npx playwright test --config=e2e/playwright.config.ts \ | |
| e2e/scenarios/02_preset_matrix.spec.ts \ | |
| --shard=${{ matrix.shard }}/4 | |
| - name: Upload artefacts | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: preset-matrix-shard-${{ matrix.shard }} | |
| path: | | |
| vbgui/e2e/test-results/ | |
| vbgui/e2e/screenshots/ | |
| vbgui/e2e/logs/ | |
| retention-days: 3 | |
| - name: Remove isolated MLX Python | |
| if: always() | |
| run: rm -rf "$VBGUI_E2E_VENV" | |
| # Specialised + canvas smoke scenarios — run on every push (lightweight). | |
| specialised: | |
| name: Specialised + canvas smoke | |
| if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} | |
| runs-on: [self-hosted, macOS, ARM64, cppmega-mlx-macos] | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 | |
| with: { node-version: "22", cache: "npm", | |
| cache-dependency-path: vbgui/package-lock.json } | |
| - name: Create isolated MLX Python | |
| run: | | |
| base_python=/Volumes/external/sources/cppmega.mlx/.venv/bin/python | |
| test -x "$base_python" | |
| job_venv="$(mktemp -d "$RUNNER_TEMP/cppmega-mlx-e2e-${GITHUB_RUN_ID}-${GITHUB_JOB}.XXXXXX")" | |
| "$base_python" -m venv "$job_venv" | |
| echo "VBGUI_E2E_PYTHON=$job_venv/bin/python" >> "$GITHUB_ENV" | |
| echo "VBGUI_E2E_VENV=$job_venv" >> "$GITHUB_ENV" | |
| echo "PYTHONPATH=" >> "$GITHUB_ENV" | |
| echo "PYTHONNOUSERSITE=1" >> "$GITHUB_ENV" | |
| - run: | | |
| "$VBGUI_E2E_PYTHON" -m pip install --disable-pip-version-check \ | |
| -e ".[gui,parquet,widget]" | |
| "$VBGUI_E2E_PYTHON" - <<'PY' | |
| from cppmega_mlx.training.native_optim import status | |
| native_status = status() | |
| if not native_status.get("available", False): | |
| raise RuntimeError(native_status.get("reason", "native optimizer extension unavailable")) | |
| PY | |
| - run: | | |
| "$VBGUI_E2E_PYTHON" tests/fixtures/build_e2e_matrix.py | |
| - working-directory: vbgui | |
| env: { NODE_ENV: development } | |
| run: | | |
| npm ci | |
| npm run typecheck | |
| - working-directory: vbgui | |
| run: npx playwright install chromium | |
| - working-directory: vbgui | |
| env: { NODE_ENV: development, CI: "1" } | |
| run: | | |
| npx playwright test --config=e2e/playwright.config.ts \ | |
| e2e/scenarios/01_canvas_smoke.spec.ts \ | |
| e2e/scenarios/04_tokenizer_playground.spec.ts \ | |
| e2e/scenarios/05_data_inspector.spec.ts \ | |
| e2e/scenarios/06_sharding_proposals.spec.ts \ | |
| e2e/scenarios/07_gotchas.spec.ts | |
| - if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: specialised-results | |
| path: | | |
| vbgui/e2e/test-results/ | |
| vbgui/e2e/screenshots/ | |
| vbgui/e2e/logs/ | |
| retention-days: 3 | |
| - name: Remove isolated MLX Python | |
| if: always() | |
| run: rm -rf "$VBGUI_E2E_VENV" | |
| # Mini-train matrix (192 cells, real mlx) — macOS only, nightly. | |
| mini-train-matrix: | |
| name: Mini-train matrix (macOS, nightly) | |
| if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' | |
| runs-on: [self-hosted, macOS, ARM64, cppmega-mlx-macos] | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 | |
| with: { node-version: "22", cache: "npm", | |
| cache-dependency-path: vbgui/package-lock.json } | |
| - name: Create isolated MLX Python | |
| run: | | |
| base_python=/Volumes/external/sources/cppmega.mlx/.venv/bin/python | |
| test -x "$base_python" | |
| job_venv="$(mktemp -d "$RUNNER_TEMP/cppmega-mlx-e2e-${GITHUB_RUN_ID}-${GITHUB_JOB}.XXXXXX")" | |
| "$base_python" -m venv "$job_venv" | |
| echo "VBGUI_E2E_PYTHON=$job_venv/bin/python" >> "$GITHUB_ENV" | |
| echo "VBGUI_E2E_VENV=$job_venv" >> "$GITHUB_ENV" | |
| echo "PYTHONPATH=" >> "$GITHUB_ENV" | |
| echo "PYTHONNOUSERSITE=1" >> "$GITHUB_ENV" | |
| - run: | | |
| "$VBGUI_E2E_PYTHON" -m pip install --disable-pip-version-check \ | |
| -e ".[gui,parquet,widget]" | |
| "$VBGUI_E2E_PYTHON" - <<'PY' | |
| from cppmega_mlx.training.native_optim import status | |
| native_status = status() | |
| if not native_status.get("available", False): | |
| raise RuntimeError(native_status.get("reason", "native optimizer extension unavailable")) | |
| PY | |
| - run: | | |
| "$VBGUI_E2E_PYTHON" tests/fixtures/build_e2e_matrix.py | |
| - working-directory: vbgui | |
| env: { NODE_ENV: development } | |
| run: | | |
| npm ci | |
| npm run typecheck | |
| - working-directory: vbgui | |
| run: npx playwright install chromium | |
| - working-directory: vbgui | |
| env: { NODE_ENV: development, CI: "1" } | |
| run: | | |
| npx playwright test --config=e2e/playwright.config.ts \ | |
| e2e/scenarios/03_train_matrix.spec.ts | |
| - if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: mini-train-results | |
| path: | | |
| vbgui/e2e/test-results/ | |
| vbgui/e2e/screenshots/ | |
| vbgui/e2e/logs/ | |
| retention-days: 7 | |
| - name: Remove isolated MLX Python | |
| if: always() | |
| run: rm -rf "$VBGUI_E2E_VENV" | |
| # Matrix report — aggregates shard artefacts into one Markdown | |
| # summary, posted as a build summary + uploaded as standalone artefact. | |
| matrix-report: | |
| name: Matrix report | |
| needs: [preset-matrix, specialised] | |
| runs-on: [self-hosted, Linux, X64, cppmega-mlx] | |
| if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 | |
| with: { path: artifacts } | |
| - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 | |
| with: { python-version: "3.13" } | |
| - name: Build matrix report | |
| run: | | |
| python tools/build_e2e_matrix_report.py \ | |
| --artifacts artifacts \ | |
| --output e2e_matrix_report.md | |
| cat e2e_matrix_report.md >> "$GITHUB_STEP_SUMMARY" | |
| - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: e2e_matrix_report | |
| path: e2e_matrix_report.md | |
| retention-days: 14 |