chore: run full conformance CI suite #251
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Conformance | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| workflow_dispatch: | |
| concurrency: | |
| group: conformance-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| # Pinned for reproducible runs; bump deliberately when the suite updates. | |
| CONFORMANCE_VERSION: "0.1.16" | |
| DRAFT_CONFORMANCE_VERSION: "0.2.0-alpha.9" | |
| jobs: | |
| server: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| # Build the whole package (server + client bins): the conformance crate is | |
| # excluded from the workspace default-members. | |
| - name: Build conformance binaries | |
| run: cargo build -p mcp-conformance | |
| - name: Test conformance server | |
| run: cargo test -p mcp-conformance --bin conformance-server | |
| - name: Start 2025-11-25 server | |
| run: | | |
| PORT=8001 ./target/debug/conformance-server & | |
| echo $! > server.pid | |
| for _ in $(seq 1 30); do | |
| if curl -s -o /dev/null http://127.0.0.1:8001/mcp; then | |
| exit 0 | |
| fi | |
| sleep 1 | |
| done | |
| echo "conformance server did not become ready" >&2 | |
| exit 1 | |
| - name: Run 2025-11-25 server suite | |
| run: | | |
| npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \ | |
| --url http://127.0.0.1:8001/mcp \ | |
| --spec-version 2025-11-25 \ | |
| -o conformance-results | |
| # These pass today but are excluded from the default "active" suite; | |
| # run them explicitly so regressions are still caught. | |
| - name: Run 2025-11-25 pending scenarios | |
| run: | | |
| for scenario in json-schema-2020-12 server-sse-polling; do | |
| npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \ | |
| --url http://127.0.0.1:8001/mcp \ | |
| --scenario "$scenario" \ | |
| -o conformance-results | |
| done | |
| - name: Start draft server | |
| run: | | |
| STATELESS=1 PORT=8002 ./target/debug/conformance-server & | |
| echo $! > draft-server.pid | |
| for _ in $(seq 1 30); do | |
| if curl -s -o /dev/null http://127.0.0.1:8002/mcp; then | |
| exit 0 | |
| fi | |
| sleep 1 | |
| done | |
| echo "draft conformance server did not become ready" >&2 | |
| exit 1 | |
| - name: Run SEP-2575 discovery contract | |
| run: | | |
| endpoint=http://127.0.0.1:8002/mcp | |
| common_headers=( | |
| -H "Content-Type: application/json" | |
| -H "Accept: application/json, text/event-stream" | |
| -H "Mcp-Method: server/discover" | |
| ) | |
| discover_response="$( | |
| curl --fail-with-body --silent --show-error \ | |
| "${common_headers[@]}" \ | |
| -H "MCP-Protocol-Version: 2026-07-28" \ | |
| --data '{ | |
| "jsonrpc": "2.0", | |
| "id": "discover", | |
| "method": "server/discover", | |
| "params": { | |
| "_meta": { | |
| "io.modelcontextprotocol/protocolVersion": "2026-07-28", | |
| "io.modelcontextprotocol/clientInfo": { | |
| "name": "conformance-workflow", | |
| "version": "1.0.0" | |
| }, | |
| "io.modelcontextprotocol/clientCapabilities": {} | |
| } | |
| } | |
| }' \ | |
| "$endpoint" | |
| )" | |
| jq -e ' | |
| .result.resultType == "complete" and | |
| (.result.supportedVersions | index("2026-07-28") != null) and | |
| (.result.capabilities | type == "object") and | |
| (.result.serverInfo.name | type == "string") and | |
| .result.ttlMs == 0 and | |
| .result.cacheScope == "private" | |
| ' <<<"$discover_response" | |
| status="$( | |
| curl --silent --show-error \ | |
| --output /tmp/unsupported-version.json \ | |
| --write-out "%{http_code}" \ | |
| "${common_headers[@]}" \ | |
| -H "MCP-Protocol-Version: 2099-01-01" \ | |
| --data '{ | |
| "jsonrpc": "2.0", | |
| "id": "unsupported", | |
| "method": "server/discover", | |
| "params": { | |
| "_meta": { | |
| "io.modelcontextprotocol/protocolVersion": "2099-01-01", | |
| "io.modelcontextprotocol/clientInfo": { | |
| "name": "conformance-workflow", | |
| "version": "1.0.0" | |
| }, | |
| "io.modelcontextprotocol/clientCapabilities": {} | |
| } | |
| } | |
| }' \ | |
| "$endpoint" | |
| )" | |
| test "$status" = "400" | |
| jq -e ' | |
| .id == "unsupported" and | |
| .error.code == -32022 and | |
| .error.data.requested == "2099-01-01" and | |
| (.error.data.supported | index("2026-07-28") != null) | |
| ' /tmp/unsupported-version.json | |
| - name: Run 2026-07-28 server suite | |
| run: | | |
| npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" server \ | |
| --url http://127.0.0.1:8002/mcp \ | |
| --suite all \ | |
| --spec-version 2026-07-28 \ | |
| --expected-failures conformance/expected-failures-2026-07-28.yaml \ | |
| -o conformance-results | |
| - name: Stop conformance servers | |
| if: always() | |
| run: | | |
| kill "$(cat draft-server.pid)" 2>/dev/null || true | |
| kill "$(cat server.pid)" 2>/dev/null || true | |
| - name: Upload results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: conformance-server-results | |
| path: conformance-results | |
| client: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Build conformance binaries | |
| run: cargo build -p mcp-conformance | |
| - name: Run 2025-11-25 client suite | |
| run: | | |
| npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" client \ | |
| --command "$(pwd)/target/debug/conformance-client" \ | |
| --suite all \ | |
| --spec-version 2025-11-25 \ | |
| -o conformance-client-results/full | |
| - name: Run 2026-07-28 client suite | |
| run: | | |
| npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" client \ | |
| --command "$(pwd)/target/debug/conformance-client" \ | |
| --suite all \ | |
| --spec-version 2026-07-28 \ | |
| --expected-failures conformance/expected-failures-2026-07-28.yaml \ | |
| -o conformance-client-results/draft | |
| - name: Upload results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: conformance-client-results | |
| path: conformance-client-results |