conformance.yml

name: Conformance

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  workflow_dispatch:

concurrency:
  group: conformance-${{ github.ref }}
  cancel-in-progress: true

env:
  # Pinned for reproducible runs; bump deliberately when the suite updates.
  CONFORMANCE_VERSION: "0.1.16"
  DRAFT_CONFORMANCE_VERSION: "0.2.0-alpha.9"

jobs:
  server:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@v7

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@stable

      - uses: Swatinem/rust-cache@v2

      # Build the whole package (server + client bins): the conformance crate is
      # excluded from the workspace default-members.
      - name: Build conformance binaries
        run: cargo build -p mcp-conformance

      - name: Start conformance server
        run: |
          PORT=8001 ./target/debug/conformance-server &
          echo $! > server.pid
          for _ in $(seq 1 30); do
            if curl -s -o /dev/null http://127.0.0.1:8001/mcp; then
              exit 0
            fi
            sleep 1
          done
          echo "conformance server did not become ready" >&2
          exit 1

      - name: Run server conformance suite
        run: |
          npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \
            --url http://127.0.0.1:8001/mcp \
            --spec-version 2025-11-25 \
            -o conformance-results

      # These pass today but are excluded from the default "active" suite;
      # run them explicitly so regressions are still caught.
      - name: Run pending scenarios
        run: |
          for scenario in json-schema-2020-12 server-sse-polling; do
            npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \
              --url http://127.0.0.1:8001/mcp \
              --scenario "$scenario" \
              -o conformance-results
          done

      - name: Start draft conformance server
        run: |
          STATELESS=1 PORT=8002 ./target/debug/conformance-server &
          echo $! > draft-server.pid
          for _ in $(seq 1 30); do
            if curl -s -o /dev/null http://127.0.0.1:8002/mcp; then
              exit 0
            fi
            sleep 1
          done
          echo "draft conformance server did not become ready" >&2
          exit 1

      - name: Run draft SEP scenarios
        run: |
          for scenario in sep-2164-resource-not-found caching http-header-validation; do
            npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" server \
              --url http://127.0.0.1:8002/mcp \
              --scenario "$scenario" \
              --spec-version draft \
              -o conformance-results
          done

      # SEP-2322 MRTR scenarios (spec 2026-07-28). They speak the stateless
      # lifecycle (bare JSON-RPC POSTs, no initialize handshake), so they run
      # against the stateless draft server.
      - name: Run SEP-2322 MRTR scenarios
        run: |
          for scenario in \
            input-required-result-basic-elicitation \
            input-required-result-basic-sampling \
            input-required-result-basic-list-roots \
            input-required-result-request-state \
            input-required-result-multiple-input-requests \
            input-required-result-multi-round \
            input-required-result-missing-input-response \
            input-required-result-non-tool-request \
            input-required-result-result-type \
            input-required-result-unsupported-methods \
            input-required-result-tampered-state \
            input-required-result-capability-check \
            input-required-result-ignore-extra-params \
            input-required-result-validate-input \
          ; do
            npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" server \
              --url http://127.0.0.1:8002/mcp \
              --scenario "$scenario" \
              -o conformance-results
          done

      - name: Stop draft conformance server
        if: always()
        run: kill "$(cat draft-server.pid)" 2>/dev/null || true


      - name: Stop conformance server
        if: always()
        run: kill "$(cat server.pid)" 2>/dev/null || true

      - name: Upload results
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: conformance-server-results
          path: conformance-results

  client:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@v7

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@stable

      - uses: Swatinem/rust-cache@v2

      - name: Build conformance binaries
        run: cargo build -p mcp-conformance

      - name: Run full client conformance suite
        run: |
          npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" client \
            --command "$(pwd)/target/debug/conformance-client" \
            --suite all \
            --spec-version 2025-11-25 \
            -o conformance-client-results/full

      # SEP-2322 MRTR client scenario (spec 2026-07-28).
      - name: Run SEP-2322 MRTR client scenario
        run: |
          npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" client \
            --command "$(pwd)/target/debug/conformance-client" \
            --scenario sep-2322-client-request-state \
            -o conformance-client-results/mrtr

      - name: Upload results
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: conformance-client-results
          path: conformance-client-results