Skip to content

Commit dd11e5b

Browse files
authored
chore: run full conformance CI suite (#1010)
1 parent c87b642 commit dd11e5b

3 files changed

Lines changed: 99 additions & 67 deletions

File tree

.github/workflows/conformance.yml

Lines changed: 24 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,15 @@ concurrency:
1414
env:
1515
# Pinned for reproducible runs; bump deliberately when the suite updates.
1616
CONFORMANCE_VERSION: "0.1.16"
17+
# When updating DRAFT_CONFORMANCE_VERSION, diff
18+
# `conformance list --spec-version 2026-07-28`
19+
# and update #977
1720
DRAFT_CONFORMANCE_VERSION: "0.2.0-alpha.9"
1821

1922
jobs:
2023
server:
2124
runs-on: ubuntu-latest
25+
timeout-minutes: 10
2226
permissions:
2327
contents: read
2428
steps:
@@ -81,7 +85,9 @@ jobs:
8185
echo "draft conformance server did not become ready" >&2
8286
exit 1
8387
84-
# Run discovery separately until #985 enables the full draft suite.
88+
# useful for enforcing partial conformance for SEP-2575.
89+
# when `server-stateless` is removed from `expected-failures-2026-07-28.yaml`,
90+
# this step can be removed.
8591
- name: Run SEP-2575 discovery contract
8692
run: |
8793
endpoint=http://127.0.0.1:8002/mcp
@@ -91,6 +97,8 @@ jobs:
9197
-H "Mcp-Method: server/discover"
9298
)
9399
100+
# The stateless server streams responses as SSE, so unwrap the
101+
# `data:` payload before parsing as JSON.
94102
discover_response="$(
95103
curl --fail-with-body --silent --show-error \
96104
"${common_headers[@]}" \
@@ -110,7 +118,8 @@ jobs:
110118
}
111119
}
112120
}' \
113-
"$endpoint"
121+
"$endpoint" \
122+
| sed -n 's/^data: //p'
114123
)"
115124
jq -e '
116125
.result.resultType == "complete" and
@@ -152,35 +161,14 @@ jobs:
152161
(.error.data.supported | index("2026-07-28") != null)
153162
' /tmp/unsupported-version.json
154163
155-
# Keep this explicit list until the full draft suite is enabled by #985.
156-
- name: Run supported draft server scenarios
164+
- name: Run 2026-07-28 server suite
157165
run: |
158-
for scenario in \
159-
sep-2164-resource-not-found \
160-
caching \
161-
http-header-validation \
162-
http-custom-header-server-validation \
163-
input-required-result-basic-elicitation \
164-
input-required-result-basic-sampling \
165-
input-required-result-basic-list-roots \
166-
input-required-result-request-state \
167-
input-required-result-multiple-input-requests \
168-
input-required-result-multi-round \
169-
input-required-result-missing-input-response \
170-
input-required-result-non-tool-request \
171-
input-required-result-result-type \
172-
input-required-result-unsupported-methods \
173-
input-required-result-tampered-state \
174-
input-required-result-capability-check \
175-
input-required-result-ignore-extra-params \
176-
input-required-result-validate-input \
177-
; do
178-
npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" server \
179-
--url http://127.0.0.1:8002/mcp \
180-
--scenario "$scenario" \
181-
--spec-version draft \
182-
-o conformance-results
183-
done
166+
npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" server \
167+
--url http://127.0.0.1:8002/mcp \
168+
--suite all \
169+
--spec-version 2026-07-28 \
170+
--expected-failures conformance/expected-failures-2026-07-28.yaml \
171+
-o conformance-results
184172
185173
- name: Stop conformance servers
186174
if: always()
@@ -197,6 +185,7 @@ jobs:
197185

198186
client:
199187
runs-on: ubuntu-latest
188+
timeout-minutes: 10
200189
permissions:
201190
contents: read
202191
steps:
@@ -218,21 +207,14 @@ jobs:
218207
--spec-version 2025-11-25 \
219208
-o conformance-client-results/full
220209
221-
# SEP-2322 MRTR client scenario (spec 2026-07-28).
222-
- name: Run draft SEP-2322 client scenario
210+
- name: Run 2026-07-28 client suite
223211
run: |
224212
npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" client \
225213
--command "$(pwd)/target/debug/conformance-client" \
226-
--scenario sep-2322-client-request-state \
227-
-o conformance-client-results/mrtr
228-
229-
- name: Run draft SEP-2575 client scenario
230-
run: |
231-
npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" client \
232-
--command "$(pwd)/target/debug/conformance-client" \
233-
--scenario request-metadata \
234-
--spec-version draft \
235-
-o conformance-client-results/sep-2575
214+
--suite all \
215+
--spec-version 2026-07-28 \
216+
--expected-failures conformance/expected-failures-2026-07-28.yaml \
217+
-o conformance-client-results/draft
236218
237219
- name: Upload results
238220
if: always()
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Known failures for the pinned 2026-07-28 draft conformance suite
2+
# (@modelcontextprotocol/conformance DRAFT_CONFORMANCE_VERSION).
3+
#
4+
# The full suites run in CI with `--expected-failures` pointing at this file:
5+
# - a scenario failing that is NOT listed here fails the build
6+
# - a scenario listed here that starts passing also fails the build (stale entry),
7+
# so remove it from this list when the underlying issue is fixed.
8+
#
9+
# When bumping DRAFT_CONFORMANCE_VERSION, diff
10+
# `conformance list --spec-version 2026-07-28` and update #977.
11+
12+
server:
13+
# SEP-2575 stateless lifecycle gaps: discover capability declaration,
14+
# missing-capability rejection, HTTP 404 for removed methods, and
15+
# diagnostic tools (test_missing_capability, test_streaming_elicitation,
16+
# test_logging_tool) not yet implemented.
17+
# tracked in #1004
18+
- server-stateless
19+
# SEP-2106: composition/conditional/$anchor keywords are stripped from
20+
# published tool input schemas.
21+
# tracked in #1003
22+
- json-schema-2020-12
23+
24+
client:
25+
# Client does not yet send MCP-Protocol-Version header pre-initialize as
26+
# required by the 2026-07-28 stateless lifecycle.
27+
# tracked in #1002
28+
- tools_call
29+
# Auth feature gaps in the 2026-07-28 auth scenarios.
30+
# tracked in #1002
31+
- auth/scope-step-up
32+
# SEP-2352: SDK lacks issuer-stamped credential storage (#879), so the
33+
# sep-2352-reregister-on-as-change check fails.
34+
- auth/authorization-server-migration

conformance/src/bin/client.rs

Lines changed: 41 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1006,25 +1006,49 @@ async fn main() -> anyhow::Result<()> {
10061006

10071007
tracing::info!("Running scenario '{}' against {}", scenario, server_url);
10081008

1009-
match scenario.as_str() {
1009+
// Safety net: some harness servers intentionally misbehave (e.g. reply
1010+
// with an id-less error instead of answering a request), which would
1011+
// leave the client waiting forever. Exit on our own before the harness's
1012+
// 30s client timeout so it never has to kill us (which has been observed
1013+
// to wedge the harness process in CI).
1014+
let timeout_secs: u64 = std::env::var("MCP_CONFORMANCE_TIMEOUT_SECS")
1015+
.ok()
1016+
.and_then(|v| v.parse().ok())
1017+
.unwrap_or(25);
1018+
tokio::time::timeout(
1019+
std::time::Duration::from_secs(timeout_secs),
1020+
run_scenario(&scenario, &server_url, &ctx),
1021+
)
1022+
.await
1023+
.map_err(|_| anyhow::anyhow!("Scenario '{scenario}' timed out after {timeout_secs}s"))??;
1024+
1025+
Ok(())
1026+
}
1027+
1028+
async fn run_scenario(
1029+
scenario: &str,
1030+
server_url: &str,
1031+
ctx: &ConformanceContext,
1032+
) -> anyhow::Result<()> {
1033+
match scenario {
10101034
// Non-auth scenarios
1011-
"initialize" => run_basic_client(&server_url).await?,
1035+
"initialize" => run_basic_client(server_url).await?,
10121036
// SEP-2106: the scenario serves a tool whose schema carries a network
10131037
// `$ref`; the check passes when the client lists tools without
10141038
// dereferencing (fetching) that URL. A plain connect → list_tools →
10151039
// close is sufficient; the scenario's mock server does not implement
10161040
// the discover lifecycle, so `run_discover_client` hangs against it.
1017-
"json-schema-ref-no-deref" => run_basic_client(&server_url).await?,
1018-
"tools_call" => run_tools_call_client(&server_url, &ctx).await?,
1041+
"json-schema-ref-no-deref" => run_basic_client(server_url).await?,
1042+
"tools_call" => run_tools_call_client(server_url, ctx).await?,
10191043
"elicitation-sep1034-client-defaults" => {
1020-
run_elicitation_defaults_client(&server_url).await?
1044+
run_elicitation_defaults_client(server_url).await?
10211045
}
1022-
"sse-retry" => run_sse_retry_client(&server_url).await?,
1046+
"sse-retry" => run_sse_retry_client(server_url).await?,
10231047
"request-metadata" | "sep-2322-client-request-state" => {
1024-
run_discover_client(&server_url).await?
1048+
run_discover_client(server_url).await?
10251049
}
10261050
"http-standard-headers" | "http-custom-headers" | "http-invalid-tool-headers" => {
1027-
run_discover_tools_call_client(&server_url, &ctx).await?
1051+
run_discover_tools_call_client(server_url, ctx).await?
10281052
}
10291053

10301054
// Auth scenarios - standard OAuth flow
@@ -1056,34 +1080,26 @@ async fn main() -> anyhow::Result<()> {
10561080
| "auth/iss-wrong-issuer"
10571081
| "auth/iss-unexpected"
10581082
| "auth/iss-normalized"
1059-
| "auth/metadata-issuer-mismatch"
1060-
| "auth/metadata-issuer-mismatch"
1061-
// SEP-2352: PRM `authorization_servers` switches between calls; a
1062-
// compliant client re-registers at the new AS. Known partial failure:
1063-
// the SDK lacks issuer-stamped credential storage (#879), so the
1064-
// `sep-2352-reregister-on-as-change` check fails. Left on the standard
1065-
// flow rather than fixture-orchestrated re-registration so the
1066-
// conformance result reflects real SDK behavior.
1067-
| "auth/authorization-server-migration" => run_auth_client(&server_url, &ctx).await?,
1083+
| "auth/metadata-issuer-mismatch" => run_auth_client(server_url, ctx).await?,
10681084

10691085
// Auth - scope step-up
1070-
"auth/scope-step-up" => run_auth_scope_step_up_client(&server_url, &ctx).await?,
1086+
"auth/scope-step-up" => run_auth_scope_step_up_client(server_url, ctx).await?,
10711087

10721088
// Auth - scope retry limit
1073-
"auth/scope-retry-limit" => run_auth_scope_retry_limit_client(&server_url, &ctx).await?,
1089+
"auth/scope-retry-limit" => run_auth_scope_retry_limit_client(server_url, ctx).await?,
10741090

10751091
// Auth - authorization server migration (SEP-2352)
10761092
"auth/authorization-server-migration" => {
1077-
run_auth_server_migration_client(&server_url, &ctx).await?
1093+
run_auth_server_migration_client(server_url, ctx).await?
10781094
}
10791095

10801096
// Auth - pre-registration
1081-
"auth/pre-registration" => run_auth_preregistered_client(&server_url, &ctx).await?,
1097+
"auth/pre-registration" => run_auth_preregistered_client(server_url, ctx).await?,
10821098

10831099
// Auth - resource mismatch (should fail to auth → pass)
10841100
"auth/resource-mismatch" => {
10851101
// Try to auth; it should fail because PRM resource doesn't match
1086-
match run_auth_client(&server_url, &ctx).await {
1102+
match run_auth_client(server_url, ctx).await {
10871103
Ok(_) => {
10881104
tracing::warn!("Auth succeeded despite resource mismatch!");
10891105
}
@@ -1094,12 +1110,12 @@ async fn main() -> anyhow::Result<()> {
10941110
}
10951111

10961112
// Auth - client credentials
1097-
"auth/client-credentials-basic" => run_client_credentials_basic(&server_url, &ctx).await?,
1098-
"auth/client-credentials-jwt" => run_client_credentials_jwt(&server_url, &ctx).await?,
1113+
"auth/client-credentials-basic" => run_client_credentials_basic(server_url, ctx).await?,
1114+
"auth/client-credentials-jwt" => run_client_credentials_jwt(server_url, ctx).await?,
10991115

11001116
// Auth - cross-app access
11011117
"auth/cross-app-access-complete-flow" => {
1102-
run_cross_app_access_client(&server_url, &ctx).await?
1118+
run_cross_app_access_client(server_url, ctx).await?
11031119
}
11041120

11051121
unknown => anyhow::bail!("Unsupported conformance scenario: {unknown}"),

0 commit comments

Comments
 (0)