Run MCP smoke tests with Claude Code

mneudert · mneudert · commit 51a0b6a759a0 · 2026-04-08T17:39:14.000+02:00
diff --git a/.github/scripts/mcp-smoke/README.md b/.github/scripts/mcp-smoke/README.md
@@ -19,7 +19,7 @@ This harness powers `.github/workflows/mcp-ai-smoke.yml`.
 - Additional MCP tool calls are accepted if the expected tool succeeds at least once.
 - Handles empty `cases.json` safely and emits an empty `results.json`.
 
-3. `run-gemini-smoke.sh` / `run-codex-smoke.sh`
+3. `run-gemini-smoke.sh` / `run-codex-smoke.sh` / `run-claude-smoke.sh`
 - Thin wrappers that select the provider-specific runtime setup before invoking the shared harness.
 
 4. `.github/actions/setup-mcp-smoke-env`
@@ -38,9 +38,11 @@ This harness powers `.github/workflows/mcp-ai-smoke.yml`.
 
 - Gemini uses a temporary `settings.json` (`mcpServers`) generated at runtime outside the uploaded artifact tree and remains configurable via `GEMINI_CLI_CMD` and `GEMINI_EXTRA_ARGS`.
 - Codex uses a temporary CI-local home/config outside the uploaded artifact tree and registers the Matomo MCP endpoint there at runtime.
+- Claude uses a temporary CI-local home/config outside the uploaded artifact tree and loads the Matomo MCP endpoint via a generated HTTP MCP config file at runtime.
 - Provider login state and runtime configuration stay outside uploaded artifacts.
-- The workflow can run providers independently based on secret availability (`GEMINI_APIKEY`, `OPENAI_APIKEY`).
-- Gemini and Codex run in separate CI jobs with isolated Matomo environments and log files.
+- The workflow can run providers independently based on secret availability (`GEMINI_APIKEY`, `OPENAI_APIKEY`, `CLAUDE_APIKEY`).
+- Claude defaults to the generic `sonnet` alias, with `claude_model` still available in `workflow_dispatch` for pinned-model overrides.
+- Gemini, Codex, and Claude run in separate CI jobs with isolated Matomo environments and log files.
 - Provider jobs share a local composite action for the common Matomo/bootstrap sequence.
 - The workflow is intentionally report-first/non-blocking for internal prototype usage.
 - The workflow uploads one artifact per provider job and builds a combined summary from the downloaded per-provider `results.json` files.
diff --git a/.github/scripts/mcp-smoke/run-ai-smoke.sh b/.github/scripts/mcp-smoke/run-ai-smoke.sh
@@ -36,7 +36,7 @@ mkdir -p \
   "$provider_state_dir"
 
 cleanup() {
-  unset OPENAI_APIKEY MCP_AUTH_TOKEN GEMINI_API_KEY
+  unset OPENAI_APIKEY MCP_AUTH_TOKEN GEMINI_API_KEY ANTHROPIC_API_KEY
 
   if [ -n "${provider_state_dir:-}" ] && [ -d "$provider_state_dir" ]; then
     rm -rf "$provider_state_dir"
@@ -139,6 +139,34 @@ setup_provider() {
         --bearer-token-env-var MCP_AUTH_TOKEN \
         >/dev/null 2>&1
       ;;
+    claude)
+      ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY is required}
+      CLAUDE_MODEL=${CLAUDE_MODEL:-sonnet}
+      CLAUDE_CLI_CMD=${CLAUDE_CLI_CMD:-claude}
+      CLAUDE_HOME_DIR="$provider_state_dir/home"
+      CLAUDE_MCP_CONFIG_FILE="$provider_state_dir/mcp.json"
+
+      if ! command -v "$CLAUDE_CLI_CMD" >/dev/null 2>&1; then
+        echo "Claude CLI command not found: $CLAUDE_CLI_CMD" >&2
+        exit 1
+      fi
+
+      mkdir -p "$CLAUDE_HOME_DIR"
+      jq -n \
+        --arg mcp_url "$MCP_URL" \
+        --arg auth_header "Bearer $TOKEN_AUTH" \
+        '{
+          "mcpServers": {
+            "matomo": {
+              "type": "http",
+              "url": $mcp_url,
+              "headers": {
+                "Authorization": $auth_header
+              }
+            }
+          }
+        }' > "$CLAUDE_MCP_CONFIG_FILE"
+      ;;
     *)
       echo "Unsupported SMOKE_PROVIDER: $SMOKE_PROVIDER" >&2
       exit 1
@@ -198,6 +226,29 @@ run_provider() {
           > /dev/null 2>&1 < "$prompt_file"
       fi
       ;;
+    claude)
+      if [ "$CASE_TIMEOUT_SECONDS" -gt 0 ]; then
+        ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY" HOME="$CLAUDE_HOME_DIR" \
+        timeout "${CASE_TIMEOUT_SECONDS}s" "$CLAUDE_CLI_CMD" \
+          --mcp-config "$CLAUDE_MCP_CONFIG_FILE" \
+          --strict-mcp-config \
+          --dangerously-skip-permissions \
+          -p \
+          --model "$CLAUDE_MODEL" \
+          "$(cat "$prompt_file")" \
+          > "$transcript_file" 2>&1 < /dev/null
+      else
+        ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY" HOME="$CLAUDE_HOME_DIR" \
+        "$CLAUDE_CLI_CMD" \
+          --mcp-config "$CLAUDE_MCP_CONFIG_FILE" \
+          --strict-mcp-config \
+          --dangerously-skip-permissions \
+          -p \
+          --model "$CLAUDE_MODEL" \
+          "$(cat "$prompt_file")" \
+          > "$transcript_file" 2>&1 < /dev/null
+      fi
+      ;;
   esac
 }
 
diff --git a/.github/scripts/mcp-smoke/run-claude-smoke.sh b/.github/scripts/mcp-smoke/run-claude-smoke.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+export SMOKE_PROVIDER=claude
+"$(dirname "$0")/run-ai-smoke.sh"
diff --git a/.github/workflows/mcp-ai-smoke.yml b/.github/workflows/mcp-ai-smoke.yml
@@ -13,6 +13,10 @@ on:
         description: Codex model
         required: false
         default: gpt-5-mini
+      claude_model:
+        description: Claude model
+        required: false
+        default: sonnet
 
 permissions:
   actions: read
@@ -41,6 +45,7 @@ env:
   MYSQL_DATABASE: matomo
   GEMINI_MODEL: ${{ github.event.inputs.gemini_model || 'gemini-2.5-flash' }}
   CODEX_MODEL: ${{ github.event.inputs.codex_model || 'gpt-5-mini' }}
+  CLAUDE_MODEL: ${{ github.event.inputs.claude_model || 'sonnet' }}
 
 jobs:
   preflight:
@@ -50,17 +55,20 @@ jobs:
       should_run: ${{ steps.check.outputs.should_run }}
       has_gemini: ${{ steps.check.outputs.has_gemini }}
       has_codex: ${{ steps.check.outputs.has_codex }}
+      has_claude: ${{ steps.check.outputs.has_claude }}
       reason: ${{ steps.check.outputs.reason }}
     steps:
       - id: check
         name: Check secret availability
         env:
           GEMINI_APIKEY: ${{ secrets.GEMINI_APIKEY }}
           OPENAI_APIKEY: ${{ secrets.OPENAI_APIKEY }}
+          CLAUDE_APIKEY: ${{ secrets.CLAUDE_APIKEY }}
         run: |
           set -euo pipefail
           has_gemini=false
           has_codex=false
+          has_claude=false
 
           if [ -n "${GEMINI_APIKEY:-}" ]; then
             has_gemini=true
@@ -70,10 +78,15 @@ jobs:
             has_codex=true
           fi
 
+          if [ -n "${CLAUDE_APIKEY:-}" ]; then
+            has_claude=true
+          fi
+
           echo "has_gemini=${has_gemini}" >> "$GITHUB_OUTPUT"
           echo "has_codex=${has_codex}" >> "$GITHUB_OUTPUT"
+          echo "has_claude=${has_claude}" >> "$GITHUB_OUTPUT"
 
-          if [ "$has_gemini" = false ] && [ "$has_codex" = false ]; then
+          if [ "$has_gemini" = false ] && [ "$has_codex" = false ] && [ "$has_claude" = false ]; then
             echo "should_run=false" >> "$GITHUB_OUTPUT"
             echo "reason=No provider secret is available in this context (likely fork PR)." >> "$GITHUB_OUTPUT"
           else
@@ -89,6 +102,7 @@ jobs:
             echo "- should_run: ${{ steps.check.outputs.should_run }}"
             echo "- Gemini available: ${{ steps.check.outputs.has_gemini }}"
             echo "- Codex available: ${{ steps.check.outputs.has_codex }}"
+            echo "- Claude available: ${{ steps.check.outputs.has_claude }}"
             echo "- reason: ${{ steps.check.outputs.reason }}"
           } >> "$GITHUB_STEP_SUMMARY"
 
@@ -294,10 +308,111 @@ jobs:
           path: .github/scripts/mcp-smoke/artifacts/codex
           if-no-files-found: warn
 
+  smoke_claude:
+    name: API + Claude smoke
+    runs-on: ubuntu-24.04
+    needs: preflight
+    if: needs.preflight.outputs.has_claude == 'true'
+    continue-on-error: true
+    timeout-minutes: 30
+    services:
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ROOT_PASSWORD: root
+          MYSQL_DATABASE: matomo
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping -h 127.0.0.1 -uroot -proot"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=30
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 1
+          lfs: false
+          persist-credentials: false
+
+      - name: Setup shared smoke environment
+        uses: ./.github/actions/setup-mcp-smoke-env
+        with:
+          plugin_name: ${{ env.PLUGIN_NAME }}
+          matomo_dir: ${{ env.MATOMO_DIR }}
+          base_url: ${{ env.BASE_URL }}
+          mysql_host: ${{ env.MYSQL_HOST }}
+          mysql_port: ${{ env.MYSQL_PORT }}
+          mysql_user: ${{ env.MYSQL_USER }}
+          mysql_password: ${{ env.MYSQL_PASSWORD }}
+          mysql_database: ${{ env.MYSQL_DATABASE }}
+          matomo_test_target: maximum_supported_matomo
+
+      - name: Install Claude CLI
+        run: npm install -g @anthropic-ai/claude-code
+
+      - name: Claude MCP smoke suite
+        run: ./.github/scripts/mcp-smoke/run-claude-smoke.sh
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.CLAUDE_APIKEY }}
+          CLAUDE_MODEL: ${{ env.CLAUDE_MODEL }}
+          BASE_URL: ${{ env.BASE_URL }}
+          STATE_FILE: ${{ github.workspace }}/.github/scripts/mcp-smoke/.state.json
+          CASES_FILE: ${{ github.workspace }}/.github/scripts/mcp-smoke/cases.json
+          PROMPTS_DIR: ${{ github.workspace }}/.github/scripts/mcp-smoke/prompts
+          ARTIFACT_DIR: ${{ github.workspace }}/.github/scripts/mcp-smoke/artifacts
+          MATOMO_LOG_FILE: ${{ env.MATOMO_DIR }}/tmp/logs/matomo.log
+          PHP_SERVER_LOG_FILE: ${{ env.MATOMO_DIR }}/tmp/logs/php-server.log
+
+      - name: Build Claude summary
+        if: always()
+        run: |
+          set -euo pipefail
+          ARTIFACT_DIR="${{ github.workspace }}/.github/scripts/mcp-smoke/artifacts"
+          RESULTS_FILE="$ARTIFACT_DIR/claude/results.json"
+          {
+            echo "## Claude Smoke"
+            echo
+            echo "- job result: ${{ job.status }}"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          if [ ! -f "$RESULTS_FILE" ]; then
+            {
+              echo
+              echo "No results.json was produced."
+            } >> "$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
+
+          total=$(jq 'length' "$RESULTS_FILE")
+          passed=$(jq '[.[] | select(.status == "pass")] | length' "$RESULTS_FILE")
+          skipped=$(jq '[.[] | select(.status == "skip")] | length' "$RESULTS_FILE")
+          failed=$(jq '[.[] | select(.status == "fail")] | length' "$RESULTS_FILE")
+
+          {
+            echo
+            echo "| Case | Expected Tool | Status | Reason |"
+            echo "|---|---|---|---|"
+            jq -r '.[] | "| \(.case_id) | \(.expected_tool) | \(.status) | \(.reason) |"' "$RESULTS_FILE"
+            echo
+            echo "- Total checks: ${total}"
+            echo "- Passed: ${passed}"
+            echo "- Skipped: ${skipped}"
+            echo "- Failed: ${failed}"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload Claude artifacts
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: mcp-ai-smoke-claude
+          path: .github/scripts/mcp-smoke/artifacts/claude
+          if-no-files-found: warn
+
   summary:
     name: Final Summary
     runs-on: ubuntu-24.04
-    needs: [preflight, smoke_gemini, smoke_codex]
+    needs: [preflight, smoke_gemini, smoke_codex, smoke_claude]
     if: always()
     steps:
       - name: Download Gemini artifacts
@@ -314,6 +429,13 @@ jobs:
           name: mcp-ai-smoke-codex
           path: .github/scripts/mcp-smoke/artifacts/codex
 
+      - name: Download Claude artifacts
+        if: needs.smoke_claude.result != 'skipped'
+        uses: actions/download-artifact@v5
+        with:
+          name: mcp-ai-smoke-claude
+          path: .github/scripts/mcp-smoke/artifacts/claude
+
       - name: Final status summary
         run: |
           set -euo pipefail
@@ -359,9 +481,11 @@ jobs:
             echo "- preflight: ${{ needs.preflight.outputs.should_run }}"
             echo "- Gemini available: ${{ needs.preflight.outputs.has_gemini }}"
             echo "- Codex available: ${{ needs.preflight.outputs.has_codex }}"
+            echo "- Claude available: ${{ needs.preflight.outputs.has_claude }}"
             echo "- reason: ${{ needs.preflight.outputs.reason }}"
             echo "- Gemini job result: ${{ needs.smoke_gemini.result || 'skipped' }}"
             echo "- Codex job result: ${{ needs.smoke_codex.result || 'skipped' }}"
+            echo "- Claude job result: ${{ needs.smoke_claude.result || 'skipped' }}"
             echo
             echo "Prototype smoke is intentionally non-blocking."
           } >> "$GITHUB_STEP_SUMMARY"
@@ -373,3 +497,7 @@ jobs:
           if [ "${{ needs.preflight.outputs.has_codex }}" = "true" ]; then
             summarize_provider codex
           fi
+
+          if [ "${{ needs.preflight.outputs.has_claude }}" = "true" ]; then
+            summarize_provider claude
+          fi
