conformance.yml

name: Conformance

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  workflow_dispatch:

concurrency:
  group: conformance-${{ github.ref }}
  cancel-in-progress: true

env:
  # Pinned for reproducible runs; bump deliberately when the suite updates.
  CONFORMANCE_VERSION: "0.1.16"
  DRAFT_CONFORMANCE_VERSION: "0.2.0-alpha.9"

jobs:
  server:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@v7

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@stable

      - uses: Swatinem/rust-cache@v2

      # Build the whole package (server + client bins): the conformance crate is
      # excluded from the workspace default-members.
      - name: Build conformance binaries
        run: cargo build -p mcp-conformance

      - name: Test conformance server
        run: cargo test -p mcp-conformance --bin conformance-server

      - name: Start 2025-11-25 server
        run: |
          PORT=8001 ./target/debug/conformance-server &
          echo $! > server.pid
          for _ in $(seq 1 30); do
            if curl -s -o /dev/null http://127.0.0.1:8001/mcp; then
              exit 0
            fi
            sleep 1
          done
          echo "conformance server did not become ready" >&2
          exit 1

      - name: Run 2025-11-25 server suite
        run: |
          npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \
            --url http://127.0.0.1:8001/mcp \
            --spec-version 2025-11-25 \
            -o conformance-results

      # These pass today but are excluded from the default "active" suite;
      # run them explicitly so regressions are still caught.
      - name: Run 2025-11-25 pending scenarios
        run: |
          for scenario in json-schema-2020-12 server-sse-polling; do
            npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \
              --url http://127.0.0.1:8001/mcp \
              --scenario "$scenario" \
              -o conformance-results
          done

      - name: Start draft server
        run: |
          STATELESS=1 PORT=8002 ./target/debug/conformance-server &
          echo $! > draft-server.pid
          for _ in $(seq 1 30); do
            if curl -s -o /dev/null http://127.0.0.1:8002/mcp; then
              exit 0
            fi
            sleep 1
          done
          echo "draft conformance server did not become ready" >&2
          exit 1

      - name: Run SEP-2575 discovery contract
        run: |
          endpoint=http://127.0.0.1:8002/mcp
          common_headers=(
            -H "Content-Type: application/json"
            -H "Accept: application/json, text/event-stream"
            -H "Mcp-Method: server/discover"
          )

          discover_response="$(
            curl --fail-with-body --silent --show-error \
              "${common_headers[@]}" \
              -H "MCP-Protocol-Version: 2026-07-28" \
              --data '{
                "jsonrpc": "2.0",
                "id": "discover",
                "method": "server/discover",
                "params": {
                  "_meta": {
                    "io.modelcontextprotocol/protocolVersion": "2026-07-28",
                    "io.modelcontextprotocol/clientInfo": {
                      "name": "conformance-workflow",
                      "version": "1.0.0"
                    },
                    "io.modelcontextprotocol/clientCapabilities": {}
                  }
                }
              }' \
              "$endpoint"
          )"
          jq -e '
            .result.resultType == "complete" and
            (.result.supportedVersions | index("2026-07-28") != null) and
            (.result.capabilities | type == "object") and
            (.result.serverInfo.name | type == "string") and
            .result.ttlMs == 0 and
            .result.cacheScope == "private"
          ' <<<"$discover_response"

          status="$(
            curl --silent --show-error \
              --output /tmp/unsupported-version.json \
              --write-out "%{http_code}" \
              "${common_headers[@]}" \
              -H "MCP-Protocol-Version: 2099-01-01" \
              --data '{
                "jsonrpc": "2.0",
                "id": "unsupported",
                "method": "server/discover",
                "params": {
                  "_meta": {
                    "io.modelcontextprotocol/protocolVersion": "2099-01-01",
                    "io.modelcontextprotocol/clientInfo": {
                      "name": "conformance-workflow",
                      "version": "1.0.0"
                    },
                    "io.modelcontextprotocol/clientCapabilities": {}
                  }
                }
              }' \
              "$endpoint"
          )"
          test "$status" = "400"
          jq -e '
            .id == "unsupported" and
            .error.code == -32022 and
            .error.data.requested == "2099-01-01" and
            (.error.data.supported | index("2026-07-28") != null)
          ' /tmp/unsupported-version.json

      - name: Run 2026-07-28 server suite
        run: |
          npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" server \
            --url http://127.0.0.1:8002/mcp \
            --suite all \
            --spec-version 2026-07-28 \
            --expected-failures conformance/expected-failures-2026-07-28.yaml \
            -o conformance-results

      - name: Stop conformance servers
        if: always()
        run: |
          kill "$(cat draft-server.pid)" 2>/dev/null || true
          kill "$(cat server.pid)" 2>/dev/null || true

      - name: Upload results
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: conformance-server-results
          path: conformance-results

  client:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@v7

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@stable

      - uses: Swatinem/rust-cache@v2

      - name: Build conformance binaries
        run: cargo build -p mcp-conformance

      - name: Run 2025-11-25 client suite
        run: |
          npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" client \
            --command "$(pwd)/target/debug/conformance-client" \
            --suite all \
            --spec-version 2025-11-25 \
            -o conformance-client-results/full

      - name: Run 2026-07-28 client suite
        run: |
          npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" client \
            --command "$(pwd)/target/debug/conformance-client" \
            --suite all \
            --spec-version 2026-07-28 \
            --expected-failures conformance/expected-failures-2026-07-28.yaml \
            -o conformance-client-results/draft

      - name: Upload results
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: conformance-client-results
          path: conformance-client-results