Add stdio transport support to client integration test runner (#77)

* Initial plan

* feat: add stdio transport support to MCP client integration tests

- Update client to support both stdio and http transport modes (MCP_MODE env var, default: stdio)
- In stdio mode, client spawns server via StdioClientTransport from MCP SDK
- Update run-integration-tests.sh to skip manual server start/stop in stdio mode
- Add test:integration:http and test:integration:stdio scripts to client/package.json
- Update CI workflow to test both transport modes via matrix strategy
- Update docs/testing.md to document stdio as default transport
- Update docs/getting-started.md and docs/public.md to recommend VSIX installation

Co-authored-by: data-douser <70299490+data-douser@users.noreply.github.com>

* feat: make stdio server path configurable via MCP_SERVER_PATH env var

Add MCP_SERVER_PATH env var support for configuring the server binary path
in stdio mode. Update CLI help text to document new environment variables.

Co-authored-by: data-douser <70299490+data-douser@users.noreply.github.com>

* Fixes for stdio-based client integration tests

* fix bugs in stdio MCP client

Bug 1 — CWD mismatch in stdio mode (both platforms):
StdioClientTransport inherited the client/ working directory, so
relative paths passed to CodeQL CLI (e.g. server/ql/javascript/...)
resolved to client/server/ql/... which does not exist. Add cwd: repoRoot
to the transport options to match the HTTP-mode behavior where
start-server.sh explicitly cd's to the repo root.

Bug 2 — False success on Windows in stdio mode:
StdioClientTransport.close() could cause an abrupt process exit on
Windows before printTestSummary() and process.exit(exitCode) were
reached, so the shell script saw exit code 0 and reported success
despite test failures. Move summary printing and process.exitCode
assignment above the disconnect() call in all four run methods so
results are always reported even if disconnect triggers an early exit.

* fix: address review feedback on transport selection and shell arg forwarding

- Normalize MCP_MODE to lowercase in client constructor so that
  MCP_MODE=STDIO works the same as MCP_MODE=stdio
- Select stdio transport by default; only use HTTP when MCP_MODE === "http"
  (previously any non-"stdio" value fell through to HTTP)
- Add shift 2 in run_tests_in_mode() so mode_name and enable_monitoring
  positional args are not forwarded to node as extra CLI arguments

Co-authored-by: data-douser <70299490+data-douser@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: data-douser <70299490+data-douser@users.noreply.github.com>
Co-authored-by: Nathan Randall <data-douser@github.com>

Author: Copilot

.github/workflows/client-integration-tests.yml

@@ -26,17 +26,19 @@ permissions:
 
 jobs:
   integration-tests:
-    name: Integration Tests (${{ matrix.os }})
+    name: Integration Tests (${{ matrix.os }}, ${{ matrix.mcp-mode }})
     runs-on: ${{ matrix.os }}
 
     strategy:
       fail-fast: false
       matrix:
+        mcp-mode: [http, stdio]
         os: [ubuntu-latest, windows-latest]
 
     env:
       HTTP_HOST: 'localhost'
       HTTP_PORT: '3000'
+      MCP_MODE: ${{ matrix.mcp-mode }}
       TIMEOUT_SECONDS: '30'
       URL_SCHEME: 'http'
 
@@ -110,7 +112,7 @@ jobs:
         run: npm run test:integration --workspace=client
 
       - name: MCP Integration Tests - Stop the background MCP server process
-        if: always()
+        if: always() && matrix.mcp-mode == 'http'
         shell: bash
         run: |
           if [ -f server.pid ]; then
@@ -141,8 +143,8 @@ jobs:
       - name: MCP Integration Tests - Summary
         shell: bash
         run: |
-          echo "## Integration Tests Summary (${{ matrix.os }})" >> $GITHUB_STEP_SUMMARY
-          echo "✅ MCP server integration tests passed on ${{ matrix.os }}" >> $GITHUB_STEP_SUMMARY
+          echo "## Integration Tests Summary (${{ matrix.os }}, ${{ matrix.mcp-mode }})" >> $GITHUB_STEP_SUMMARY
+          echo "✅ MCP server integration tests passed on ${{ matrix.os }} with ${{ matrix.mcp-mode }} transport" >> $GITHUB_STEP_SUMMARY
 
   codeql-path-tests:
     name: CODEQL_PATH Tests (${{ matrix.os }})

.gitignore

@@ -49,6 +49,9 @@ codeql-development-mcp-server.code-workspace
 # Prevent accidentally committing integration test output files in root directory
 # These should only be in client/integration-tests/primitives/tools/*/after/ directories
 /evaluator-log.json
+/client/query-results/
+/client/query-results.bqrs
+/client/query-results.sarif
 /query-results/
 /query-results.bqrs
 /query-results.sarif

client/package.json

@@ -55,8 +55,10 @@
     "test:coverage": "echo 'NOOP client test:coverage'",
     "test:integration": "scripts/run-integration-tests.sh --no-install-packs",
     "test:integration:default": "ENABLE_MONITORING_TOOLS=false scripts/run-integration-tests.sh --no-install-packs",
+    "test:integration:http": "MCP_MODE=http scripts/run-integration-tests.sh --no-install-packs",
     "test:integration:install-packs": "scripts/run-integration-tests.sh",
-    "test:integration:monitoring": "ENABLE_MONITORING_TOOLS=true scripts/run-integration-tests.sh",
+    "test:integration:monitoring": "ENABLE_MONITORING_TOOLS=true scripts/run-integration-tests.sh --no-install-packs",
+    "test:integration:stdio": "MCP_MODE=stdio scripts/run-integration-tests.sh --no-install-packs",
     "tidy": "npm run lint && npm run format"
   }
 }

client/scripts/run-integration-tests.sh

@@ -8,8 +8,9 @@
 #   2. Monitoring mode (monitoring tools enabled) - tests session_* tools
 #
 # Environment Variables:
-#   HTTP_HOST              - Server host (default: localhost)
-#   HTTP_PORT              - Server port (default: 3000)
+#   MCP_MODE               - MCP transport mode (default: stdio, also: http)
+#   HTTP_HOST              - Server host for HTTP mode (default: localhost)
+#   HTTP_PORT              - Server port for HTTP mode (default: 3000)
 #   TIMEOUT_SECONDS        - Request timeout (default: 30)
 #   ENABLE_MONITORING_TOOLS - Force a specific mode instead of running both:
 #                            "true"  = only run with monitoring tools enabled
@@ -20,6 +21,7 @@
 #   ./run-integration-tests.sh                    # Run in BOTH modes (recommended)
 #   ENABLE_MONITORING_TOOLS=false ./run-integration-tests.sh  # Only default mode
 #   ENABLE_MONITORING_TOOLS=true ./run-integration-tests.sh   # Only monitoring mode
+#   MCP_MODE=http ./run-integration-tests.sh      # Run using HTTP transport
 #   ./run-integration-tests.sh --tools session_end            # Filter to specific tools
 
 set -e
@@ -29,6 +31,7 @@ CLIENT_DIR="$(dirname "$SCRIPT_DIR")"
 SERVER_DIR="$(dirname "$CLIENT_DIR")/server"
 
 # Set default environment variables
+export MCP_MODE="${MCP_MODE:-stdio}"
 export HTTP_HOST="${HTTP_HOST:-localhost}"
 export HTTP_PORT="${HTTP_PORT:-3000}"
 export TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-30}"
@@ -65,7 +68,10 @@ for arg in "$@"; do
 done
 
 echo "🚀 Starting CodeQL MCP Integration Tests"
-echo "Server URL: $URL_SCHEME://$HTTP_HOST:$HTTP_PORT/mcp"
+echo "MCP Mode: $MCP_MODE"
+if [ "$MCP_MODE" = "http" ]; then
+    echo "Server URL: $URL_SCHEME://$HTTP_HOST:$HTTP_PORT/mcp"
+fi
 
 # Step 1: Build and bundle the server code
 echo "📦 Building CodeQL MCP server bundle..."
@@ -81,43 +87,56 @@ else
 fi
 
 cd "$CLIENT_DIR"
-export MCP_MODE=http
-export MCP_SERVER_URL="$URL_SCHEME://$HTTP_HOST:$HTTP_PORT/mcp"
+
+# For HTTP mode, set the server URL for the client
+if [ "$MCP_MODE" = "http" ]; then
+    export MCP_SERVER_URL="$URL_SCHEME://$HTTP_HOST:$HTTP_PORT/mcp"
+fi
 
 # Function to run tests in a specific mode
 run_tests_in_mode() {
     local mode_name="$1"
     local enable_monitoring="$2"
-    
+    shift 2
+
     echo ""
     echo "═══════════════════════════════════════════════════════════════"
     echo "🧪 Running integration tests: $mode_name"
     echo "═══════════════════════════════════════════════════════════════"
-    
+
     # Set the monitoring tools flag for this run
     export ENABLE_MONITORING_TOOLS="$enable_monitoring"
-    
-    # Start MCP server with current settings
-    echo "🚀 Starting MCP server (monitoring=$enable_monitoring)..."
-    "$SCRIPT_DIR/start-server.sh"
-    
-    # Wait for server startup
-    echo "⏳ Waiting for server startup..."
-    "$SCRIPT_DIR/wait-for-server.sh"
-    
+
+    if [ "$MCP_MODE" = "http" ]; then
+        # HTTP mode: start server in background, run tests, stop server
+        echo "🚀 Starting MCP server (monitoring=$enable_monitoring)..."
+        "$SCRIPT_DIR/start-server.sh"
+
+        # Wait for server startup
+        echo "⏳ Waiting for server startup..."
+        "$SCRIPT_DIR/wait-for-server.sh"
+    else
+        # stdio mode: client spawns server directly via StdioClientTransport
+        echo "📡 Using stdio transport (client spawns server directly)"
+    fi
+
     # Run the integration tests (skip pack installation since we already did it)
     echo "🧪 Running tests..."
     node src/ql-mcp-client.js integration-tests --no-install-packs "$@"
-    
-    # Stop the server before next mode
-    echo "🛑 Stopping server..."
-    "$SCRIPT_DIR/stop-server.sh"
+
+    if [ "$MCP_MODE" = "http" ]; then
+        # Stop the server before next mode
+        echo "🛑 Stopping server..."
+        "$SCRIPT_DIR/stop-server.sh"
+    fi
 }
 
 # Trap to ensure cleanup happens even if script fails
 cleanup() {
     echo "🧹 Cleaning up..."
-    "$SCRIPT_DIR/stop-server.sh" 2>/dev/null || true
+    if [ "$MCP_MODE" = "http" ]; then
+        "$SCRIPT_DIR/stop-server.sh" 2>/dev/null || true
+    fi
 }
 trap cleanup EXIT
 

client/src/lib/cli-parser.js

@@ -133,6 +133,12 @@ OPTIONS:
                        Example: --timeout 600
 
   --help               Display help information
+
+ENVIRONMENT VARIABLES:
+  MCP_MODE             MCP transport mode: stdio (default) or http
+  MCP_SERVER_PATH      Path to the MCP server JS entry point (stdio mode only)
+  MCP_SERVER_URL       MCP server URL (http mode only, default: http://localhost:3000/mcp)
+  ENABLE_MONITORING_TOOLS  Enable session_* monitoring tools (default: false)
 `;
 }