perf: run dockerd in parallel with runner startup #77
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD | |
| on: | |
| push: | |
| branches: [main] | |
| paths: | |
| - 'runner/**' | |
| - 'app.py' | |
| - 'tests/**' | |
| - 'pyproject.toml' | |
| - '.github/workflows/**' | |
| tags: ['v*'] | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| - 'runner/**' | |
| - 'app.py' | |
| - 'tests/**' | |
| - 'pyproject.toml' | |
| - '.github/workflows/**' | |
| jobs: | |
| # ── Stage 1: CI Gate (every PR + push) ────────────────────────────── | |
| ci: | |
| name: Lint & Test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e ".[dev]" | |
| - name: Lint | |
| run: ruff check runner/ app.py tests/ | |
| - name: Test | |
| run: pytest tests/ -v | |
| # ── Stage 2: Deploy (push to main / tags only) ────────────────────── | |
| deploy: | |
| name: Deploy to Modal | |
| needs: ci | |
| if: github.event_name == 'push' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| concurrency: | |
| group: modal-production-deploy | |
| cancel-in-progress: false | |
| env: | |
| MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} | |
| MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e ".[dev]" | |
| - name: Deploy | |
| run: modal deploy app.py | |
| - name: Wait for propagation | |
| run: | | |
| echo "Waiting 30s for Modal deployment to propagate..." | |
| sleep 30 | |
| - name: Health check | |
| run: | | |
| HEALTH_URL="${{ secrets.MODAL_WEBHOOK_URL }}/health" | |
| if [ -n "$HEALTH_URL" ] && [ "$HEALTH_URL" != "/health" ]; then | |
| for i in $(seq 1 15); do | |
| if curl -sf "$HEALTH_URL" > /dev/null 2>&1; then | |
| echo "Runner health check passed" | |
| exit 0 | |
| fi | |
| echo "Waiting for runner... ($i/15)" | |
| sleep 2 | |
| done | |
| echo "Health check failed — runner not responding" | |
| exit 1 | |
| else | |
| echo "MODAL_WEBHOOK_URL not configured, skipping health check" | |
| sleep 10 | |
| fi | |
| # ── Stage 3: Smoke Test (confirms runner is alive) ────────────────── | |
| smoke-test: | |
| name: Smoke Test | |
| needs: deploy | |
| runs-on: [self-hosted, modal, "job-${{ github.run_id }}-smoke"] | |
| timeout-minutes: 5 | |
| steps: | |
| - name: Verify runner is healthy | |
| run: | | |
| echo "Runner is healthy" | |
| uname -a | |
| echo "Smoke test passed at $(date)" | |
| # ── Stage 4: Fast Tests (Basic + Matrix) ──────────────────────────── | |
| test-basic: | |
| name: Integration - Basic | |
| needs: smoke-test | |
| uses: ./.github/workflows/test.yml | |
| test-matrix: | |
| name: Integration - Matrix | |
| needs: smoke-test | |
| uses: ./.github/workflows/matrix-example.yml | |
| # ── Stage 5: Docker Tests (after fast tests complete) ─────────────── | |
| test-docker: | |
| name: Integration - Docker | |
| needs: [smoke-test, test-basic, test-matrix] | |
| uses: ./.github/workflows/test-docker.yml | |
| # ── Stage 6: Concurrency Tests (after Docker) ─────────────────────── | |
| test-concurrency: | |
| name: Integration - Concurrency | |
| needs: [smoke-test, test-docker] | |
| uses: ./.github/workflows/test-concurrency.yml | |
| # ── Stage 7: Concurrency Groups (after Concurrency) ───────────────── | |
| test-concurrency-groups: | |
| name: Integration - Concurrency Groups | |
| needs: [smoke-test, test-concurrency] | |
| uses: ./.github/workflows/test-dummy-concurrency.yml | |
| with: | |
| max_parallel: '2' | |
| job_count: '5' |