ci: simplify conformance while running against both spec versions #432
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Conformance | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| workflow_dispatch: | |
| concurrency: | |
| group: conformance-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| CONFORMANCE_VERSION: "0.2.0-alpha.9" | |
| jobs: | |
| server: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Build conformance binaries | |
| run: cargo build -p mcp-conformance | |
| - name: Test conformance server | |
| run: cargo test -p mcp-conformance --bin conformance-server | |
| - name: Start conformance server | |
| run: | | |
| PORT=8001 ./target/debug/conformance-server & | |
| echo $! > server.pid | |
| for _ in $(seq 1 30); do | |
| if curl -s -o /dev/null http://127.0.0.1:8001/mcp; then | |
| exit 0 | |
| fi | |
| sleep 1 | |
| done | |
| echo "conformance server did not become ready" >&2 | |
| exit 1 | |
| - name: Run 2025-11-25 server suite | |
| run: | | |
| npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \ | |
| --url http://127.0.0.1:8001/mcp \ | |
| --suite all \ | |
| --spec-version 2025-11-25 \ | |
| -o conformance-results/2025-11-25 | |
| - name: Run 2026-07-28 server suite | |
| run: | | |
| npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \ | |
| --url http://127.0.0.1:8001/mcp \ | |
| --suite all \ | |
| --spec-version 2026-07-28 \ | |
| -o conformance-results/2026-07-28 | |
| - name: Run extension server scenarios (informational) | |
| run: | | |
| scenarios=( | |
| tasks-lifecycle | |
| tasks-capability-negotiation | |
| tasks-wire-fields | |
| tasks-request-state-removal | |
| tasks-mrtr-input | |
| tasks-request-headers | |
| tasks-dispatch-and-envelope | |
| tasks-status-notifications | |
| tasks-required-task-error | |
| tasks-mrtr-composition | |
| ) | |
| for scenario in "${scenarios[@]}"; do | |
| npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \ | |
| --url http://127.0.0.1:8001/mcp \ | |
| --scenario "$scenario" \ | |
| --expected-failures conformance/expected-failures-extensions.yaml \ | |
| -o conformance-extension-results | |
| done | |
| - name: Stop conformance server | |
| if: always() | |
| run: kill "$(cat server.pid)" 2>/dev/null || true | |
| - name: Upload results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: conformance-server-results | |
| path: | | |
| conformance-results | |
| conformance-extension-results | |
| client: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Build conformance binaries | |
| run: cargo build -p mcp-conformance | |
| - name: Run 2025-11-25 client suite | |
| run: | | |
| npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" client \ | |
| --command "$(pwd)/target/debug/conformance-client" \ | |
| --suite all \ | |
| --spec-version 2025-11-25 \ | |
| -o conformance-client-results/2025-11-25 | |
| - name: Run 2026-07-28 client suite | |
| run: | | |
| npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" client \ | |
| --command "$(pwd)/target/debug/conformance-client" \ | |
| --suite all \ | |
| --spec-version 2026-07-28 \ | |
| -o conformance-client-results/2026-07-28 | |
| - name: Run extension client suite (informational) | |
| run: | | |
| npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" client \ | |
| --command "$(pwd)/target/debug/conformance-client" \ | |
| --suite extensions \ | |
| --expected-failures conformance/expected-failures-extensions.yaml \ | |
| -o conformance-client-results/extensions | |
| - name: Upload results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: conformance-client-results | |
| path: conformance-client-results |