fix(test-conformance): stop leaking the conformance server and hanging on stale ports (#2276)

Author: felixweinberger

.changeset/fix-conformance-server-leak.md

@@ -0,0 +1,5 @@
+---
+'@modelcontextprotocol/test-conformance': patch
+---
+
+Fix the server conformance script leaking the test server process: the cleanup trap killed the npx wrapper while the actual server kept listening on port 3000, making later runs silently test stale code or hang forever in the readiness loop. The script now spawns the server directly with `node --import tsx`, refuses to start while the port is taken, and bounds each readiness probe; both test servers report `EADDRINUSE` with an actionable message, and the plain `test:conformance:client` script works again (`--suite core`, required since conformance 0.2.0-alpha.1).

pnpm-lock.yaml

@@ -60,6 +60,8 @@ npx @modelcontextprotocol/conformance server \
   --scenario server-initialize
 ```
 
+Note: `pnpm run test:conformance:server` always starts (and tests) its own server, and refuses to run while anything is still listening on its port. Stop the manually started server first, or keep using the direct `--url` invocation above against it.
+
 ## Files
 
 - `src/everythingClient.ts` - Client that handles all client conformance scenarios

test/conformance/README.md

@@ -28,13 +28,13 @@
         "start": "npm run server",
         "server": "tsx watch --clear-screen=false scripts/cli.ts server",
         "client": "tsx scripts/cli.ts client",
-        "test:conformance:client": "conformance client --command 'npx tsx ./src/everythingClient.ts' --expected-failures ./expected-failures.yaml",
-        "test:conformance:client:all": "conformance client --command 'npx tsx ./src/everythingClient.ts' --suite all --expected-failures ./expected-failures.yaml",
-        "test:conformance:client:run": "npx tsx ./src/everythingClient.ts",
+        "test:conformance:client": "conformance client --command 'node --import tsx ./src/everythingClient.ts' --suite core --expected-failures ./expected-failures.yaml",
+        "test:conformance:client:all": "conformance client --command 'node --import tsx ./src/everythingClient.ts' --suite all --expected-failures ./expected-failures.yaml",
+        "test:conformance:client:run": "node --import tsx ./src/everythingClient.ts",
         "test:conformance:server": "scripts/run-server-conformance.sh --expected-failures ./expected-failures.yaml",
         "test:conformance:server:draft": "scripts/run-server-conformance.sh --suite draft --expected-failures ./expected-failures.yaml",
         "test:conformance:server:all": "scripts/run-server-conformance.sh --suite all --expected-failures ./expected-failures.yaml",
-        "test:conformance:server:run": "npx tsx ./src/everythingServer.ts",
+        "test:conformance:server:run": "node --import tsx ./src/everythingServer.ts",
         "test:conformance:all": "pnpm run test:conformance:client:all && pnpm run test:conformance:server:all"
     },
     "devDependencies": {
@@ -50,6 +50,7 @@
         "@modelcontextprotocol/test-helpers": "workspace:^",
         "cors": "catalog:runtimeServerOnly",
         "express": "catalog:runtimeServerOnly",
+        "tsx": "catalog:devTools",
         "zod": "catalog:runtimeShared"
     }
 }

test/conformance/package.json

@@ -11,9 +11,21 @@ SERVER_URL="http://localhost:${PORT}/mcp"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 cd "$SCRIPT_DIR/.."
 
-# Start the server in the background
+# Refuse to start if something is already listening on the port. The readiness
+# check below cannot tell our server apart from a stale one, so a leftover
+# listener would mean silently running conformance against old code — or
+# hanging forever if the listener never responds.
+if (: > "/dev/tcp/localhost/${PORT}") 2>/dev/null; then
+    echo "Error: port ${PORT} is already in use."
+    echo "Stop the stale process first (lsof -ti:${PORT} -sTCP:LISTEN | xargs kill) or set PORT to a free port."
+    exit 1
+fi
+
+# Start the server in the background. Use `node --import tsx` rather than
+# `npx tsx` so SERVER_PID is the server process itself — killing an npx/tsx
+# wrapper leaves the actual server running and squatting the port.
 echo "Starting conformance test server on port ${PORT}..."
-npx tsx ./src/everythingServer.ts &
+node --import tsx ./src/everythingServer.ts &
 SERVER_PID=$!
 
 # Function to cleanup on exit
@@ -24,11 +36,16 @@ cleanup() {
 }
 trap cleanup EXIT
 
-# Wait for server to be ready
+# Wait for server to be ready. --max-time keeps a hung listener from wedging
+# the loop forever, and a dead server process fails fast instead of retrying.
 echo "Waiting for server to be ready..."
 MAX_RETRIES=30
 RETRY_COUNT=0
-while ! curl -s "${SERVER_URL}" > /dev/null 2>&1; do
+while ! curl -s --max-time 2 "${SERVER_URL}" > /dev/null 2>&1; do
+    if ! kill -0 $SERVER_PID 2>/dev/null; then
+        echo "Server process exited unexpectedly"
+        exit 1
+    fi
     RETRY_COUNT=$((RETRY_COUNT + 1))
     if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then
         echo "Server failed to start after ${MAX_RETRIES} attempts"

test/conformance/scripts/run-server-conformance.sh

@@ -412,12 +412,19 @@ async function startServer() {
     });
 
     // Start server
-    app.listen(PORT, () => {
+    const httpServer = app.listen(PORT, () => {
         console.log(`MCP Auth Test Server running at http://localhost:${PORT}/mcp`);
         console.log(`  - PRM endpoint: http://localhost:${PORT}/.well-known/oauth-protected-resource`);
         console.log(`  - Auth server: ${AUTH_SERVER_URL}`);
         console.log(`  - Introspection: ${asMetadata.introspection_endpoint}`);
     });
+    httpServer.on('error', (error: NodeJS.ErrnoException) => {
+        if (error.code !== 'EADDRINUSE') {
+            throw error;
+        }
+        console.error(`Port ${PORT} is already in use — is a stale auth test server still running?`);
+        process.exit(1);
+    });
 }
 
 // Start the server

test/conformance/src/authTestServer.ts

@@ -1020,7 +1020,14 @@ app.delete('/mcp', async (req: Request, res: Response) => {
 
 // Start server
 const PORT = process.env.PORT || 3000;
-app.listen(PORT, () => {
+const httpServer = app.listen(PORT, () => {
     console.log(`MCP Conformance Test Server running on http://localhost:${PORT}`);
     console.log(`  - MCP endpoint: http://localhost:${PORT}/mcp`);
 });
+httpServer.on('error', (error: NodeJS.ErrnoException) => {
+    if (error.code !== 'EADDRINUSE') {
+        throw error;
+    }
+    console.error(`Port ${PORT} is already in use — is a stale conformance server still running?`);
+    process.exit(1);
+});