Skip to content

Commit 78514c2

Browse files
committed
feat: support "draft" as a first-class spec version target
- Add LATEST_SPEC_VERSION and DATED_SPEC_VERSIONS constants so the next spec release is a one-line change in types.ts. - Accept "draft" as a valid protocolVersion in the initialize check and mock-server response. - --spec-version draft now selects latest-dated scenarios plus draft-tagged ones, so SEP authors can run the full suite against an SDK tracking the in-progress spec without retagging core scenarios. - Forward --spec-version to the client process via MCP_CONFORMANCE_SPEC_VERSION so SDK examples can pick the matching protocolVersion. Closes #253
1 parent 90b2334 commit 78514c2

9 files changed

Lines changed: 160 additions & 43 deletions

File tree

README.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,11 +64,12 @@ npx @modelcontextprotocol/conformance client --command "<client-command>" --scen
6464
- `--command` - The command to run your MCP client (can include flags)
6565
- `--scenario` - The test scenario to run (e.g., "initialize")
6666
- `--suite` - Run a suite of tests in parallel (e.g., "auth")
67+
- `--spec-version <version>` - Filter scenarios by spec version (e.g., `2025-11-25`, `draft`). `draft` runs the latest dated release plus any draft-only scenarios
6768
- `--expected-failures <path>` - Path to YAML baseline file of known failures (see [Expected Failures](#expected-failures))
6869
- `--timeout` - Timeout in milliseconds (default: 30000)
6970
- `--verbose` - Show verbose output
7071

71-
The framework appends `<server-url>` as an argument to your command and sets the `MCP_CONFORMANCE_SCENARIO` environment variable to the scenario name. For scenarios that require additional context (e.g., client credentials), the `MCP_CONFORMANCE_CONTEXT` environment variable contains a JSON object with scenario-specific data.
72+
The framework appends `<server-url>` as an argument to your command and sets the `MCP_CONFORMANCE_SCENARIO` environment variable to the scenario name. For scenarios that require additional context (e.g., client credentials), the `MCP_CONFORMANCE_CONTEXT` environment variable contains a JSON object with scenario-specific data. When `--spec-version` is passed, the corresponding wire protocol version is forwarded as `MCP_CONFORMANCE_PROTOCOL_VERSION` (e.g., `--spec-version draft` sets it to the current draft identifier such as `DRAFT-2026-v1`); example clients can use this value directly as their `protocolVersion`. SDKs that hard-code their protocol version can ignore it.
7273

7374
### Server Testing
7475

src/checks/checks.test.ts

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { createClientInitializationCheck } from './client';
2+
import { DRAFT_PROTOCOL_VERSION } from '../types';
23

34
describe('createClientInitializationCheck', () => {
45
it('should return SUCCESS for a valid initialize request', () => {
@@ -68,6 +69,31 @@ describe('createClientInitializationCheck', () => {
6869
expect(check.errorMessage).toContain('Client version missing');
6970
});
7071

72+
it('should accept the current draft protocol version', () => {
73+
const request = {
74+
protocolVersion: DRAFT_PROTOCOL_VERSION,
75+
clientInfo: { name: 'TestClient', version: '1.0.0' }
76+
};
77+
78+
const check = createClientInitializationCheck(request);
79+
expect(check.status).toBe('SUCCESS');
80+
expect(check.errorMessage).toBeUndefined();
81+
});
82+
83+
it.each(['DRAFT-2025-v1', 'draft'])(
84+
'should reject stale or non-canonical draft version %s',
85+
(protocolVersion) => {
86+
const request = {
87+
protocolVersion,
88+
clientInfo: { name: 'TestClient', version: '1.0.0' }
89+
};
90+
91+
const check = createClientInitializationCheck(request);
92+
expect(check.status).toBe('FAILURE');
93+
expect(check.errorMessage).toContain('Version mismatch');
94+
}
95+
);
96+
7197
it('should support custom expected spec version', () => {
7298
const request = {
7399
protocolVersion: '2024-11-05',

src/checks/client.ts

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
1-
import { ConformanceCheck, CheckStatus } from '../types';
1+
import {
2+
ConformanceCheck,
3+
CheckStatus,
4+
LATEST_SPEC_VERSION,
5+
DRAFT_PROTOCOL_VERSION
6+
} from '../types';
27

38
export function createServerInfoCheck(serverInfo: {
49
name: string;
@@ -23,12 +28,16 @@ export function createServerInfoCheck(serverInfo: {
2328
};
2429
}
2530

26-
// Valid MCP protocol versions
27-
const VALID_PROTOCOL_VERSIONS = ['2025-06-18', '2025-11-25'];
31+
// Protocol versions the mock server will accept on initialize.
32+
const VALID_PROTOCOL_VERSIONS = [
33+
'2025-06-18',
34+
LATEST_SPEC_VERSION,
35+
DRAFT_PROTOCOL_VERSION
36+
];
2837

2938
export function createClientInitializationCheck(
3039
initializeRequest: any,
31-
expectedSpecVersion: string = '2025-11-25'
40+
expectedSpecVersion: string = LATEST_SPEC_VERSION
3241
): ConformanceCheck {
3342
const protocolVersionSent = initializeRequest?.protocolVersion;
3443

src/index.ts

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,8 @@ program
152152
options.command,
153153
scenarioName,
154154
timeout,
155-
outputDir
155+
outputDir,
156+
specVersionFilter
156157
);
157158
return {
158159
scenario: scenarioName,
@@ -259,7 +260,8 @@ program
259260
validated.command,
260261
validated.scenario,
261262
timeout,
262-
outputDir
263+
outputDir,
264+
specVersionFilter
263265
);
264266

265267
const { overallFailure } = printClientResults(

src/runner/client.ts

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
import { spawn } from 'child_process';
22
import { promises as fs } from 'fs';
33
import path from 'path';
4-
import { ConformanceCheck } from '../types';
4+
import {
5+
ConformanceCheck,
6+
SpecVersion,
7+
specVersionToProtocolVersion
8+
} from '../types';
59
import { getScenario } from '../scenarios';
610
import { createResultDir, formatPrettyChecks } from './utils';
711

@@ -17,7 +21,8 @@ async function executeClient(
1721
scenarioName: string,
1822
serverUrl: string,
1923
timeout: number = 30000,
20-
context?: Record<string, unknown>
24+
context?: Record<string, unknown>,
25+
specVersion?: SpecVersion
2126
): Promise<ClientExecutionResult> {
2227
const commandParts = command.split(' ');
2328
const executable = commandParts[0];
@@ -34,6 +39,12 @@ async function executeClient(
3439
// 3. Semantic separation: scenario identifies "which test", context provides "test data"
3540
const env = { ...process.env };
3641
env.MCP_CONFORMANCE_SCENARIO = scenarioName;
42+
const protocolVersion = specVersion
43+
? specVersionToProtocolVersion(specVersion)
44+
: undefined;
45+
if (protocolVersion) {
46+
env.MCP_CONFORMANCE_PROTOCOL_VERSION = protocolVersion;
47+
}
3748
if (context) {
3849
// Include scenario name in context for discriminated union parsing
3950
env.MCP_CONFORMANCE_CONTEXT = JSON.stringify({
@@ -92,7 +103,8 @@ export async function runConformanceTest(
92103
clientCommand: string,
93104
scenarioName: string,
94105
timeout: number = 30000,
95-
outputDir?: string
106+
outputDir?: string,
107+
specVersion?: SpecVersion
96108
): Promise<{
97109
checks: ConformanceCheck[];
98110
clientOutput: ClientExecutionResult;
@@ -123,7 +135,8 @@ export async function runConformanceTest(
123135
scenarioName,
124136
urls.serverUrl,
125137
timeout,
126-
urls.context
138+
urls.context,
139+
specVersion
127140
);
128141

129142
// Print stdout/stderr if client exited with nonzero code

src/scenarios/client/initialize.ts

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,9 @@ import {
33
Scenario,
44
ScenarioUrls,
55
ConformanceCheck,
6-
SpecVersion
6+
SpecVersion,
7+
LATEST_SPEC_VERSION,
8+
DRAFT_PROTOCOL_VERSION
79
} from '../../types';
810
import { clientChecks } from '../../checks/index';
911

@@ -117,11 +119,15 @@ export class InitializeScenario implements Scenario {
117119
this.checks.push(clientChecks.createServerInfoCheck(serverInfo));
118120

119121
// Echo back client's version if valid, otherwise use latest
120-
const VALID_VERSIONS = ['2025-06-18', '2025-11-25'];
122+
const VALID_VERSIONS = [
123+
'2025-06-18',
124+
LATEST_SPEC_VERSION,
125+
DRAFT_PROTOCOL_VERSION
126+
];
121127
const clientVersion = initializeRequest?.protocolVersion;
122128
const responseVersion = VALID_VERSIONS.includes(clientVersion)
123129
? clientVersion
124-
: '2025-11-25';
130+
: LATEST_SPEC_VERSION;
125131

126132
const response = {
127133
jsonrpc: '2.0',

src/scenarios/index.ts

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@ import {
22
Scenario,
33
ClientScenario,
44
ClientScenarioForAuthorizationServer,
5-
SpecVersion
5+
SpecVersion,
6+
DATED_SPEC_VERSIONS,
7+
LATEST_SPEC_VERSION
68
} from '../types';
79
import { InitializeScenario } from './client/initialize';
810
import { ToolsCallScenario } from './client/tools_call';
@@ -257,9 +259,7 @@ export { listMetadataScenarios };
257259

258260
// All valid spec versions, used by the CLI to validate --spec-version input.
259261
export const ALL_SPEC_VERSIONS: SpecVersion[] = [
260-
'2025-03-26',
261-
'2025-06-18',
262-
'2025-11-25',
262+
...DATED_SPEC_VERSIONS,
263263
'draft',
264264
'extension'
265265
];
@@ -273,23 +273,39 @@ export function resolveSpecVersion(value: string): SpecVersion {
273273
process.exit(1);
274274
}
275275

276+
// `draft` selects everything in the latest dated release plus scenarios tagged
277+
// draft-only, so SEP authors can run the full suite against an SDK tracking the
278+
// in-progress spec without retagging core scenarios.
279+
function matchesSpecVersion(
280+
scenario: { specVersions: SpecVersion[] },
281+
version: SpecVersion
282+
): boolean {
283+
if (version === 'draft') {
284+
return (
285+
scenario.specVersions.includes('draft') ||
286+
scenario.specVersions.includes(LATEST_SPEC_VERSION)
287+
);
288+
}
289+
return scenario.specVersions.includes(version);
290+
}
291+
276292
export function listScenariosForSpec(version: SpecVersion): string[] {
277293
return scenariosList
278-
.filter((s) => s.specVersions.includes(version))
294+
.filter((s) => matchesSpecVersion(s, version))
279295
.map((s) => s.name);
280296
}
281297

282298
export function listClientScenariosForSpec(version: SpecVersion): string[] {
283299
return allClientScenariosList
284-
.filter((s) => s.specVersions.includes(version))
300+
.filter((s) => matchesSpecVersion(s, version))
285301
.map((s) => s.name);
286302
}
287303

288304
export function listClientScenariosForAuthorizationServerForSpec(
289305
version: SpecVersion
290306
): string[] {
291307
return allClientScenariosListForAuthorizationServer
292-
.filter((s) => s.specVersions.includes(version))
308+
.filter((s) => matchesSpecVersion(s, version))
293309
.map((s) => s.name);
294310
}
295311

src/scenarios/spec-version.test.ts

Lines changed: 38 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,16 @@ import {
33
listScenarios,
44
listClientScenarios,
55
listScenariosForSpec,
6+
listDraftScenarios,
67
getScenarioSpecVersions,
78
ALL_SPEC_VERSIONS
89
} from './index';
10+
import {
11+
DATED_SPEC_VERSIONS,
12+
DRAFT_PROTOCOL_VERSION,
13+
LATEST_SPEC_VERSION,
14+
specVersionToProtocolVersion
15+
} from '../types';
916

1017
describe('specVersions helpers', () => {
1118
it('every Scenario has specVersions', () => {
@@ -69,26 +76,41 @@ describe('specVersions helpers', () => {
6976
}
7077
});
7178

72-
it('draft and extension scenarios are isolated', () => {
73-
const draft = listScenariosForSpec('draft');
74-
for (const name of draft) {
75-
expect(getScenarioSpecVersions(name)).toContain('draft');
79+
it('--spec-version draft is a superset of the latest dated release', () => {
80+
const latest = new Set(listScenariosForSpec(LATEST_SPEC_VERSION));
81+
const draft = new Set(listScenariosForSpec('draft'));
82+
for (const name of latest) {
83+
expect(draft.has(name)).toBe(true);
7684
}
77-
const ext = listScenariosForSpec('extension');
78-
for (const name of ext) {
79-
expect(getScenarioSpecVersions(name)).toContain('extension');
85+
for (const name of listDraftScenarios()) {
86+
expect(draft.has(name)).toBe(true);
87+
}
88+
});
89+
90+
it('draft-tagged scenarios are not also tagged with a dated version', () => {
91+
for (const name of listDraftScenarios()) {
92+
const versions = getScenarioSpecVersions(name)!;
93+
for (const dated of DATED_SPEC_VERSIONS) {
94+
expect(
95+
versions,
96+
`scenario "${name}" is tagged with both 'draft' and '${dated}'`
97+
).not.toContain(dated);
98+
}
8099
}
81100
});
82101

83-
it('draft scenarios are not in dated versions', () => {
84-
const draft = listScenariosForSpec('draft');
85-
const dated = new Set([
86-
...listScenariosForSpec('2025-03-26'),
87-
...listScenariosForSpec('2025-06-18'),
88-
...listScenariosForSpec('2025-11-25')
89-
]);
90-
for (const name of draft) {
91-
expect(dated.has(name)).toBe(false);
102+
it('specVersionToProtocolVersion maps tags to wire versions', () => {
103+
expect(specVersionToProtocolVersion('draft')).toBe(DRAFT_PROTOCOL_VERSION);
104+
expect(specVersionToProtocolVersion(LATEST_SPEC_VERSION)).toBe(
105+
LATEST_SPEC_VERSION
106+
);
107+
expect(specVersionToProtocolVersion('extension')).toBeUndefined();
108+
});
109+
110+
it('extension scenarios are isolated', () => {
111+
const ext = listScenariosForSpec('extension');
112+
for (const name of ext) {
113+
expect(getScenarioSpecVersions(name)).toContain('extension');
92114
}
93115
});
94116
});

src/types.ts

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,34 @@ export interface ConformanceCheck {
2323
logs?: string[];
2424
}
2525

26-
export type SpecVersion =
27-
| '2025-03-26'
28-
| '2025-06-18'
29-
| '2025-11-25'
30-
| 'draft'
31-
| 'extension';
26+
export const DATED_SPEC_VERSIONS = [
27+
'2025-03-26',
28+
'2025-06-18',
29+
'2025-11-25'
30+
] as const;
31+
32+
export type DatedSpecVersion = (typeof DATED_SPEC_VERSIONS)[number];
33+
34+
export const LATEST_SPEC_VERSION: DatedSpecVersion = '2025-11-25';
35+
36+
// Mirrors LATEST_PROTOCOL_VERSION in the spec repo's schema/draft/schema.ts.
37+
// Bump when that constant changes.
38+
export const DRAFT_PROTOCOL_VERSION = 'DRAFT-2026-v1';
39+
40+
export type SpecVersion = DatedSpecVersion | 'draft' | 'extension';
41+
42+
export function specVersionToProtocolVersion(
43+
version: SpecVersion
44+
): string | undefined {
45+
if (version === 'draft') return DRAFT_PROTOCOL_VERSION;
46+
// TODO(#253 follow-up): 'extension' isn't a spec version — it's a scenario
47+
// category that got lumped into SpecVersion so `--spec-version extension`
48+
// could reuse the filter plumbing. It has no corresponding wire
49+
// protocolVersion. Split it out of this type when moving to
50+
// introducedIn/removedIn tagging.
51+
if (version === 'extension') return undefined;
52+
return version;
53+
}
3254

3355
export interface ScenarioUrls {
3456
serverUrl: string;

0 commit comments

Comments
 (0)