Port the end-to-end test suite to v2 (#2179)

Author: felixweinberger

.changeset/add-e2e-test-suite.md

@@ -0,0 +1,5 @@
+---
+'@modelcontextprotocol/test-e2e': patch
+---
+
+Add the end-to-end behavior test suite as a workspace package: a requirements manifest covering protocol-visible SDK behavior across the in-memory, stdio, and Streamable HTTP transports, ported from the v1.x branch and extended with coverage for v2 features.

.github/workflows/main.yml

@@ -54,7 +54,33 @@ jobs:
 
             - run: pnpm install
 
-            - run: pnpm test:all
+            # The e2e suite has its own job below; everything else runs here.
+            - run: pnpm -r --filter '!@modelcontextprotocol/test-e2e' test
+
+    test-e2e:
+        runs-on: ubuntu-latest
+        strategy:
+            fail-fast: false
+            matrix:
+                node-version: [20, 22, 24]
+
+        steps:
+            - uses: actions/checkout@v6
+
+            - name: Install pnpm
+              uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
+              id: pnpm-install
+              with:
+                  run_install: false
+            - uses: actions/setup-node@v6
+              with:
+                  node-version: ${{ matrix.node-version }}
+                  cache: pnpm
+                  cache-dependency-path: pnpm-lock.yaml
+
+            - run: pnpm install
+
+            - run: pnpm --filter @modelcontextprotocol/test-e2e test
 
     test-runtimes:
         runs-on: ubuntu-latest

pnpm-lock.yaml

@@ -0,0 +1,74 @@
+# E2E test suite
+
+Conformance-style tests for the SDK's public surface. `requirements.ts` is a pure-data manifest: every behavior the SDK must satisfy, with its spec/source link. Test files in `scenarios/` cite the requirement id(s) they prove via `verifies()` (`helpers/verifies.ts`), which
+registers one cell per applicable (transport, spec version). `coverage.test.ts` statically checks that every non-deferred requirement is cited and that the manifest is internally consistent.
+
+## Writing a test
+
+Add a `verifies()` call with an anonymous async body to `scenarios/<area>.test.ts`:
+
+```ts
+verifies('tools:call:content:text', async ({ transport }) => {
+    const makeServer = () => {
+        const s = new McpServer({ name: 't', version: '0' });
+        s.registerTool('echo', { inputSchema: z.object({ text: z.string() }) }, ({ text }) => ({
+            content: [{ type: 'text', text }]
+        }));
+        return s;
+    };
+    const client = new Client({ name: 'c', version: '0' });
+
+    await using _ = await wire(transport, makeServer, client);
+
+    const r = await client.callTool({ name: 'echo', arguments: { text: 'hi' } });
+    expect(r.content).toEqual([{ type: 'text', text: 'hi' }]);
+});
+```
+
+Self-contained: build server inline (factory), build client inline, `wire()`, assert. No shared fixture files. Pass an array of ids when one body genuinely proves several requirements; pass `{ title: '...' }` as the third argument only when a requirement needs more than one body
+(the title is how knownFailures target a specific body).
+
+The corresponding manifest entry is pure data:
+
+```ts
+'tools:call:content:text': {
+    source: 'https://modelcontextprotocol.io/...',
+    behavior: 'tools/call returns content[] with type:text...'
+},
+```
+
+## knownFailures, deferred, and transport restrictions
+
+When a test asserts required behavior the SDK does not satisfy, keep the test exact and record it in the manifest:
+
+```ts
+knownFailures: [{ note: 'changed in v2: ...' /* optional: test: '<verifies title>', transport, specVersion */ }];
+```
+
+`verifies()` runs matching cells as `test.fails()` — they pass while the SDK misbehaves and fail once it is fixed (then remove the entry). When the behavior cannot be expressed against the public surface at all (e.g. an API removed in v2), mark the requirement
+`deferred: '<reason>'` instead — deferred ids must not be cited by any `verifies()` call.
+
+When a transport structurally cannot express the behavior (e.g. server→client roundtrip on stateless hosting), restrict the requirement itself rather than skipping tests:
+
+```ts
+transports: STATEFUL_TRANSPORTS, // or an explicit list
+note: 'stateless hosting has no server→client back-channel'
+```
+
+`addedInSpecVersion` / `removedInSpecVersion` bound the spec versions a requirement applies to; a behavior changed by a spec release gets a sibling entry linked via `supersedes`.
+
+## Running
+
+From the repo root (the suite is the `@modelcontextprotocol/test-e2e` workspace package):
+
+```bash
+pnpm --filter @modelcontextprotocol/test-e2e test                                    # all
+pnpm --filter @modelcontextprotocol/test-e2e exec vitest run scenarios/tools.test.ts # one area
+pnpm --filter @modelcontextprotocol/test-e2e exec vitest run -t 'tools:'             # one requirement-id prefix
+pnpm --filter @modelcontextprotocol/test-e2e exec vitest run coverage.test.ts        # manifest gates
+pnpm --filter @modelcontextprotocol/test-e2e typecheck
+pnpm --filter @modelcontextprotocol/test-e2e lint
+```
+
+Slugs prefixed `typescript:` are TypeScript-SDK-specific requirements (they describe this SDK's own API surface and intentionally have no shared cross-SDK meaning); unprefixed slugs share their id and behavior wording with the Python interaction suite where both cover the
+behavior.

test/e2e/CLAUDE.md

@@ -0,0 +1,97 @@
+/**
+ * Manifest gates for the e2e suite.
+ *
+ * The linkage is inverted: test files cite the requirement id(s) they prove via
+ * `verifies(...)` (helpers/verifies.ts) and requirements.ts is pure data. These
+ * tests statically scan test/e2e/scenarios/*.test.ts for the cited ids and check them
+ * against the manifest, plus the manifest's own internal consistency rules.
+ */
+
+import { readdirSync, readFileSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { expect, test } from 'vitest';
+
+import { REQUIREMENTS } from './requirements.js';
+
+const E2E_DIR = path.dirname(fileURLToPath(import.meta.url));
+
+interface VerifiesCall {
+    file: string;
+    /** Explicit `{ title: '...' }` passed to verifies(), if any (undefined for an untitled body). */
+    title: string | undefined;
+    ids: string[];
+}
+
+/** Statically scan test/e2e/scenarios/*.test.ts for `verifies(<ids>, ...)` calls. */
+function scanVerifiesCalls(): VerifiesCall[] {
+    const calls: VerifiesCall[] = [];
+    const scenariosDir = path.join(E2E_DIR, 'scenarios');
+    const files = readdirSync(scenariosDir)
+        .filter(f => f.endsWith('.test.ts'))
+        .toSorted();
+    for (const file of files) {
+        const text = readFileSync(path.join(scenariosDir, file), 'utf8');
+        // Each call spans from its header to the first column-0 close (`});` for an
+        // untitled hugged call, `);` for a call expanded by an opts third argument).
+        for (const m of text.matchAll(/verifies\(\s*('[^']*'|\[[^\]]*\])\s*,\s*async\s*\([\s\S]*?\n(?:\}\);|\);)/g)) {
+            const ids = [...(m[1] ?? '').matchAll(/'([^']*)'/g)].map(x => x[1]).filter(id => id !== undefined);
+            const title = m[0].match(/\{\s*title:\s*'([^']*)'\s*\}\s*\n?\);$/)?.[1];
+            calls.push({ file, title, ids });
+        }
+    }
+    return calls;
+}
+
+const CALLS = scanVerifiesCalls();
+const CITED = new Set(CALLS.flatMap(c => c.ids));
+
+test('every non-deferred requirement id is cited by at least one verifies() call', () => {
+    const missing = Object.entries(REQUIREMENTS)
+        .filter(([id, r]) => !r.deferred && !CITED.has(id))
+        .map(([id]) => id);
+    expect(missing).toEqual([]);
+});
+
+test('every cited requirement id exists in the manifest and is not deferred', () => {
+    const bad: string[] = [];
+    for (const c of CALLS) {
+        for (const id of c.ids) {
+            const req = REQUIREMENTS[id];
+            if (!req) bad.push(`${c.file}: a verifies() call cites unknown requirement '${id}'`);
+            else if (req.deferred) bad.push(`${c.file}: a verifies() call cites deferred requirement '${id}'`);
+        }
+    }
+    expect(bad).toEqual([]);
+});
+
+test('every knownFailure with a test string names an explicit verifies() title that cites the requirement', () => {
+    const bad: string[] = [];
+    for (const [id, r] of Object.entries(REQUIREMENTS)) {
+        for (const kf of r.knownFailures ?? []) {
+            if (kf.test === undefined) continue;
+            const cited = CALLS.some(c => c.title === kf.test && c.ids.includes(id));
+            if (!cited)
+                bad.push(
+                    `${id}: knownFailure references title '${kf.test}', which is not an explicit verifies() title citing this requirement`
+                );
+        }
+    }
+    expect(bad).toEqual([]);
+});
+
+test('every transport-restricted requirement explains why in note', () => {
+    const missing = Object.entries(REQUIREMENTS)
+        .filter(([, r]) => r.transports !== undefined && !r.note)
+        .map(([id]) => id);
+    expect(missing).toEqual([]);
+});
+
+test('every supersedes reference points at an existing requirement id', () => {
+    for (const [id, req] of Object.entries(REQUIREMENTS)) {
+        if (req.supersedes !== undefined) {
+            expect(REQUIREMENTS[req.supersedes], `${id} supersedes unknown id '${req.supersedes}'`).toBeDefined();
+        }
+    }
+});

test/e2e/coverage.test.ts

@@ -0,0 +1,15 @@
+// @ts-check
+
+import baseConfig from '@modelcontextprotocol/eslint-config';
+
+export default [
+    ...baseConfig,
+    {
+        rules: {
+            // `await using _ = await wire(...)` holds the connection open for the test body; the binding is intentionally unused
+            '@typescript-eslint/no-unused-vars': ['error', { argsIgnorePattern: '^_', varsIgnorePattern: '^_$' }],
+            // scenario files keep the kebab-case names they share with the v1.x suite
+            'unicorn/filename-case': ['error', { cases: { camelCase: true, kebabCase: true } }]
+        }
+    }
+];

test/e2e/eslint.config.mjs

@@ -0,0 +1,64 @@
+/**
+ * Runnable stdio MCP server fixture for the transport:stdio:* e2e tests.
+ *
+ * Spawned as a real child process by test/e2e/scenarios/stdio.ts. Registers a
+ * single `echo` tool, writes a readiness marker line to stderr once it is
+ * serving, and — when E2E_IGNORE_SIGTERM=1 — keeps running after stdin EOF and
+ * swallows SIGTERM so the client transport's shutdown escalation
+ * (stdin EOF → SIGTERM → SIGKILL) is observable.
+ */
+
+/* eslint-disable unicorn/no-process-exit -- standalone spawned executable; exit codes are the behavior under test */
+
+import { McpServer } from '@modelcontextprotocol/server';
+import { StdioServerTransport } from '@modelcontextprotocol/server/stdio';
+import { z } from 'zod/v4';
+
+const server = new McpServer({ name: 'stdio-echo-server', version: '1.0.0' });
+
+server.registerTool(
+    'echo',
+    {
+        description: 'Echoes the input text back as a text content block, including multi-line text.',
+        inputSchema: z.object({ text: z.string() })
+    },
+    ({ text }) => ({ content: [{ type: 'text', text }] })
+);
+
+// env-report tool: returns JSON array of environment variable names (sorted) that
+// reached the child process. This allows tests to verify the env safelist behavior.
+server.registerTool(
+    'env-report',
+    {
+        description: 'Returns sorted array of environment variable names present in this process.',
+        inputSchema: z.object({})
+    },
+    () => {
+        const envKeys = Object.keys(process.env).toSorted();
+        return { content: [{ type: 'text', text: JSON.stringify(envKeys) }] };
+    }
+);
+
+if (process.env.E2E_IGNORE_SIGTERM === '1') {
+    // Misbehaving-server mode: keep alive after stdin EOF via interval (load-bearing — without it the child exits on stdin EOF and SIGTERM never arrives) and ignore SIGTERM, so only SIGKILL can end the process.
+    setInterval(() => {}, 1000);
+    setTimeout(() => process.exit(1), 30_000);
+    process.on('SIGTERM', () => {
+        process.stderr.write('[stdio-server] sigterm ignored\n');
+    });
+}
+
+if (process.env.E2E_GARBAGE_STDOUT === '1') {
+    // Broken-server mode: write non-JSON garbage to stdout before the server connects, simulating a broken or misconfigured server that pollutes the JSON-RPC channel.
+    process.stdout.write('GARBAGE LINE 1: not json\n');
+    process.stdout.write('GARBAGE LINE 2: {malformed json\n');
+    process.stdout.write('GARBAGE LINE 3: also not valid jsonrpc\n');
+    // Valid JSON but not a valid JSON-RPC message: v2 silently skips non-JSON noise, but schema-invalid messages must still surface via onerror.
+    process.stdout.write('{"jsonrpc":"1.0","bogus":true}\n');
+    process.stdin.resume();
+    process.stdin.on('end', () => process.exit(0));
+    setTimeout(() => process.exit(1), 30_000);
+} else {
+    await server.connect(new StdioServerTransport());
+    process.stderr.write('[stdio-server] ready\n');
+}