Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 18 additions & 54 deletions .github/workflows/conformance.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,7 @@ concurrency:
cancel-in-progress: true

env:
# Pinned for reproducible runs; bump deliberately when the suite updates.
CONFORMANCE_VERSION: "0.1.16"
# When updating DRAFT_CONFORMANCE_VERSION, diff
# `conformance list --spec-version 2026-07-28`
# and update #977
DRAFT_CONFORMANCE_VERSION: "0.2.0-alpha.9"
CONFORMANCE_VERSION: "0.2.0-alpha.9"

jobs:
server:
Expand All @@ -33,15 +28,13 @@ jobs:

- uses: Swatinem/rust-cache@v2

# Build the whole package (server + client bins): the conformance crate is
# excluded from the workspace default-members.
- name: Build conformance binaries
run: cargo build -p mcp-conformance

- name: Test conformance server
run: cargo test -p mcp-conformance --bin conformance-server

- name: Start 2025-11-25 server
- name: Start conformance server
run: |
PORT=8001 ./target/debug/conformance-server &
echo $! > server.pid
Expand All @@ -58,45 +51,19 @@ jobs:
run: |
npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \
--url http://127.0.0.1:8001/mcp \
--suite all \
--spec-version 2025-11-25 \
-o conformance-results

# These pass today but are excluded from the default "active" suite;
# run them explicitly so regressions are still caught.
- name: Run 2025-11-25 pending scenarios
run: |
for scenario in json-schema-2020-12 server-sse-polling; do
npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \
--url http://127.0.0.1:8001/mcp \
--scenario "$scenario" \
-o conformance-results
done
-o conformance-results/2025-11-25

- name: Start draft server
- name: Run 2026-07-28 server suite
run: |
STATELESS=1 PORT=8002 ./target/debug/conformance-server &
echo $! > draft-server.pid
for _ in $(seq 1 30); do
if curl -s -o /dev/null http://127.0.0.1:8002/mcp; then
exit 0
fi
sleep 1
done
echo "draft conformance server did not become ready" >&2
exit 1

- name: Run 2026-07-28 versioned-spec server suite
run: |
npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" server \
--url http://127.0.0.1:8002/mcp \
npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \
--url http://127.0.0.1:8001/mcp \
--suite all \
--spec-version 2026-07-28 \
-o conformance-results
-o conformance-results/2026-07-28

# Extension scenarios are excluded by the --spec-version filter and
# are informational for tiering. Run them explicitly so their gaps remain
# visible and newly passing scenarios make the strict baseline fail stale.
- name: Run Tasks extension server suite (informational)
- name: Run extension server scenarios (informational)
run: |
scenarios=(
tasks-lifecycle
Expand All @@ -110,20 +77,17 @@ jobs:
tasks-required-task-error
tasks-mrtr-composition
)

for scenario in "${scenarios[@]}"; do
npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" server \
--url http://127.0.0.1:8002/mcp \
npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \
--url http://127.0.0.1:8001/mcp \
--scenario "$scenario" \
--expected-failures conformance/expected-failures-extensions.yaml \
-o conformance-extension-results
done

- name: Stop conformance servers
- name: Stop conformance server
if: always()
run: |
kill "$(cat draft-server.pid)" 2>/dev/null || true
kill "$(cat server.pid)" 2>/dev/null || true
run: kill "$(cat server.pid)" 2>/dev/null || true

- name: Upload results
if: always()
Expand Down Expand Up @@ -156,19 +120,19 @@ jobs:
--command "$(pwd)/target/debug/conformance-client" \
--suite all \
--spec-version 2025-11-25 \
-o conformance-client-results/full
-o conformance-client-results/2025-11-25

- name: Run 2026-07-28 versioned-spec client suite
- name: Run 2026-07-28 client suite
run: |
npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" client \
npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" client \
--command "$(pwd)/target/debug/conformance-client" \
--suite all \
--spec-version 2026-07-28 \
-o conformance-client-results/draft
-o conformance-client-results/2026-07-28

- name: Run extension client suite (informational)
run: |
npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" client \
npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" client \
--command "$(pwd)/target/debug/conformance-client" \
--suite extensions \
--expected-failures conformance/expected-failures-extensions.yaml \
Expand Down
4 changes: 2 additions & 2 deletions conformance/expected-failures-extensions.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Known failures for informational extension scenarios in
# @modelcontextprotocol/conformance DRAFT_CONFORMANCE_VERSION.
# @modelcontextprotocol/conformance CONFORMANCE_VERSION.
#
# Extensions are not selected by a `--spec-version` run and do not count toward
# SDK tiering. CI runs them separately so a green versioned-spec suite does not
Expand All @@ -9,7 +9,7 @@
# - an unlisted failure fails the build
# - a listed scenario that starts passing fails the build as a stale entry
#
# When bumping DRAFT_CONFORMANCE_VERSION, review the available extension and
# When bumping CONFORMANCE_VERSION, review the available extension and
# pending scenarios and update this file deliberately.

# The SEP-2663 Tasks Extension server scenarios (tracked in #868) all pass and
Expand Down
3 changes: 1 addition & 2 deletions conformance/src/bin/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -943,8 +943,7 @@ fn preferred_protocol_versions() -> Vec<ProtocolVersion> {
preferred_versions
}

/// Runs draft stateless scenarios through the public discover lifecycle and
/// Streamable HTTP transport.
/// Runs scenarios through the discover lifecycle and Streamable HTTP transport.
async fn run_discover_client(server_url: &str) -> anyhow::Result<()> {
let preferred_versions = preferred_protocol_versions();
let transport = StreamableHttpClientTransport::from_uri(server_url);
Expand Down
5 changes: 1 addition & 4 deletions conformance/src/bin/server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1740,10 +1740,7 @@ async fn main() -> anyhow::Result<()> {
tracing::info!("Starting conformance server on {}", bind_addr);

let server = ConformanceServer::new();
let stateless = std::env::var_os("STATELESS").is_some();
let config = StreamableHttpServerConfig::default()
.with_legacy_session_mode(!stateless)
.with_json_response(stateless);
let config = StreamableHttpServerConfig::default();
let service = StreamableHttpService::new(
move || Ok(server.clone()),
LocalSessionManager::default().into(),
Expand Down