external-validation.yml

name: External Validation

on:
  push:
    branches: [main, develop]
  pull_request:
    branches: [main]
  schedule:
    # Run daily at 2 AM UTC to catch any external validator changes
    - cron: "0 2 * * *"
  workflow_dispatch:
    inputs:
      server_url:
        description: "MCP Server URL to validate"
        required: false
        default: "http://localhost:3000"
      protocol_version:
        description: "Protocol version to test"
        required: false
        default: "2024-11-05"

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1
  MCP_VALIDATOR_API_URL: https://api.mcp-validator.com
  JSONRPC_VALIDATOR_URL: https://json-rpc.dev/api/validate

permissions:
  contents: read
  packages: read

jobs:
  # Fast validation for PRs - Ubuntu only
  validate-framework-fast:
    name: Fast Framework Validation
    runs-on: [self-hosted, linux, x64, rust-cpu]
    timeout-minutes: 25
    if: github.event_name == 'pull_request'

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Setup Rust
        uses: dtolnay/rust-toolchain@1.88

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Cache Rust dependencies
        uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            target
          key: ${{ runner.os }}-cargo-external-fast-1.88-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }}
          restore-keys: |
            ${{ runner.os }}-cargo-external-fast-1.88-

      - name: Clean procedural macro artifacts
        run: |
          cargo clean -p pulseengine-mcp-macros

      - name: Build framework (parallel)
        run: |
          # Cargo automatically detects CPU count for parallel builds
          cargo build --all-features --release

      - name: Run core tests (parallel)
        run: |
          # Cargo automatically detects CPU count for parallel tests
          cargo test --all-features --release --workspace --exclude pulseengine-mcp-external-validation

      - name: Run external validation tests
        run: |
          cargo test --package pulseengine-mcp-external-validation --features "proptest" --release

      - name: Test validation tools
        run: |
          cargo run --bin mcp-validate -- --help
          cargo run --bin mcp-compliance-report -- --help

  # Full cross-platform validation for main branch and scheduled runs
  validate-framework:
    name: Full Framework Validation
    # Stays on ubuntu-latest/macos-latest/windows-latest: matrix spans macOS + Windows; smithy is Linux-only.
    runs-on: ${{ matrix.os }}
    timeout-minutes: 35
    if: github.event_name != 'pull_request'
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        rust: [stable]
        include:
          - os: ubuntu-latest
            python: "3.11"
          - os: macos-latest
            python: "3.11"
          - os: windows-latest
            python: "3.11"
          # Only run nightly on Ubuntu to reduce resource usage
          - os: ubuntu-latest
            rust: nightly
            python: "3.11"

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Setup Rust
        uses: dtolnay/rust-toolchain@1.88

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}

      - name: Free up disk space
        if: runner.os == 'Linux'
        run: |
          # Remove unnecessary tools and files to free up ~10GB
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /opt/ghc
          sudo rm -rf /opt/hostedtoolcache/CodeQL
          df -h

      - name: Cache Rust dependencies
        uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            target
          key: ${{ runner.os }}-cargo-external-full-${{ matrix.rust }}-1.88-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }}
          restore-keys: |
            ${{ runner.os }}-cargo-external-full-${{ matrix.rust }}-1.88-

      - name: Cache Python dependencies
        uses: actions/cache@v4
        with:
          path: |
            ~/.cache/pip
            ~/Library/Caches/pip
            ~\AppData\Local\pip\Cache
          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}

      - name: Install MCP Inspector (Linux/macOS)
        if: runner.os != 'Windows'
        run: |
          # Download and install MCP Inspector
          # Note: MCP Inspector may not be publicly available yet, so we skip if it fails
          if curl -L https://github.com/anthropics/mcp-inspector/releases/latest/download/mcp-inspector-${{ runner.os }}.tar.gz -o mcp-inspector.tar.gz 2>/dev/null && [ -s mcp-inspector.tar.gz ]; then
            # Validate that the downloaded file is actually a valid gzip archive
            if file mcp-inspector.tar.gz | grep -q "gzip compressed"; then
              tar -xzf mcp-inspector.tar.gz
              chmod +x mcp-inspector
              echo "$PWD" >> $GITHUB_PATH
              echo "MCP Inspector installed successfully"
            else
              echo "Downloaded file is not a valid gzip archive, skipping installation"
            fi
          else
            echo "MCP Inspector not available, skipping installation"
          fi

      - name: Install MCP Inspector (Windows)
        if: runner.os == 'Windows'
        run: |
          # Download and install MCP Inspector for Windows
          # Note: MCP Inspector may not be publicly available yet, so we skip if it fails
          try {
            Invoke-WebRequest -Uri https://github.com/anthropics/mcp-inspector/releases/latest/download/mcp-inspector-Windows.zip -OutFile mcp-inspector.zip -ErrorAction Stop
            if ((Get-Item mcp-inspector.zip).Length -gt 100) {
              Expand-Archive -Path mcp-inspector.zip -DestinationPath .
              echo "$PWD" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
            } else {
              Write-Host "MCP Inspector not available, skipping installation"
            }
          } catch {
            Write-Host "MCP Inspector not available, skipping installation"
          }

      - name: Clean procedural macro artifacts
        run: |
          # Clean procedural macro artifacts to prevent version conflicts
          cargo clean -p pulseengine-mcp-macros

      - name: Build framework (parallel)
        run: |
          # Use parallel compilation and release mode (Cargo automatically detects CPU count)
          cargo build --all-features --release

      - name: Run core tests (parallel)
        run: |
          # Run tests in parallel, excluding external validation to reduce time (Cargo automatically detects CPU count)
          cargo test --all-features --release --workspace --exclude pulseengine-mcp-external-validation

      - name: Run external validation tests
        run: |
          # Reduced feature set for faster execution
          cargo test --package pulseengine-mcp-external-validation --features "proptest" --release

      - name: Test validation tools
        run: |
          # Build and test CLI tools
          cargo run --bin mcp-validate -- --help
          cargo run --bin mcp-compliance-report -- --help
          echo "✅ Validation tools built successfully"

      - name: Build timedate-mcp-server for stdio testing
        shell: bash
        run: |
          # Check if timedate-mcp-server exists in external workspace
          if [ -d "../timedate-mcp" ]; then
            echo "Building timedate-mcp-server from external workspace"
            cd ../timedate-mcp
            cargo build --release
            cd -
            # Copy binary to expected location for tests
            mkdir -p ./target/release
            cp ../timedate-mcp/target/release/timedate-mcp-server ./target/release/
          else
            echo "timedate-mcp workspace not found - stdio integration tests will be skipped"
          fi
        continue-on-error: true

      - name: Run stdio integration tests
        shell: bash
        run: |
          # Only run if we have the timedate-mcp-server binary
          if [ -f "./target/release/timedate-mcp-server" ]; then
            echo "Running stdio integration tests with timedate-mcp-server"
            cargo test --package pulseengine-mcp-external-validation stdio_integration --release -- --nocapture
          else
            echo "Skipping stdio integration tests - timedate-mcp-server not available"
          fi
        continue-on-error: true
        env:
          RUST_LOG: info

      - name: Cleanup cache before saving
        run: |
          echo "📊 Disk usage before cleanup:"
          du -sh target 2>/dev/null || true

          # Remove test binaries (large and rebuilt every time)
          find target -type f -name '*-????????????????' -executable -delete 2>/dev/null || true

          # Remove incremental compilation data (doesn't help across runs)
          rm -rf target/debug/incremental
          rm -rf target/release/incremental

          # Keep: compiled dependencies in target/*/deps/*.rlib
          # Keep: build script outputs in target/*/build/*/out

          echo "📊 Disk usage after cleanup:"
          du -sh target 2>/dev/null || true

    # TODO: Re-enable server validation once we have a proper HTTP test server
    # - name: Run full compliance validation
    #   run: |
    #     SERVER_URL="${{ github.event.inputs.server_url || 'http://localhost:3000' }}"
    #     cargo run --bin mcp-validate -- --server-url "$SERVER_URL" --all
    #   env:
    #     RUST_LOG: debug

  stdio-integration-tests:
    name: Stdio + Inspector Integration Tests
    runs-on: [self-hosted, linux, x64, rust-cpu]
    timeout-minutes: 20
    if: github.event_name == 'push' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch'

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Setup Rust
        uses: dtolnay/rust-toolchain@1.88

      - name: Setup Node.js for MCP Inspector
        uses: actions/setup-node@v4
        with:
          node-version: "18"

      - name: Cache Rust dependencies
        uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            target
          key: ${{ runner.os }}-cargo-stdio-1.88-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }}
          restore-keys: |
            ${{ runner.os }}-cargo-stdio-1.88-

      - name: Build framework
        run: |
          cargo build --all-features --release

      - name: Check out timedate-mcp-server
        uses: actions/checkout@v4
        with:
          repository: "pulseengine/timedate-mcp"
          path: "timedate-mcp"
          ref: "main"
        continue-on-error: true

      - name: Build timedate-mcp-server
        shell: bash
        run: |
          if [ -d "timedate-mcp" ]; then
            echo "Building timedate-mcp-server"
            cd timedate-mcp
            if cargo build --release; then
              cd ..
              # Copy binary to expected location
              mkdir -p ./target/release
              cp timedate-mcp/target/release/timedate-mcp-server ./target/release/
              echo "✅ timedate-mcp-server built successfully"
            else
              cd ..
              echo "❌ timedate-mcp-server build failed - tests will be skipped"
            fi
          else
            echo "⚠️  timedate-mcp repository not available - tests will be skipped"
          fi
        continue-on-error: true

      - name: Install MCP Inspector
        run: |
          npm install -g @modelcontextprotocol/inspector
          echo "✅ MCP Inspector installed"

      - name: Run stdio integration tests
        run: |
          echo "🧪 Running comprehensive stdio integration tests"
          cargo test --package pulseengine-mcp-external-validation stdio_integration --release -- --nocapture --test-threads=1
        env:
          RUST_LOG: info
          CI: true

      - name: Run stdio stress tests
        run: |
          echo "💪 Running stdio stress and performance tests"
          cargo test --package pulseengine-mcp-external-validation stdio_stress --release -- --nocapture --test-threads=1
        env:
          RUST_LOG: info
          CI: true
        continue-on-error: true

  python-sdk-compatibility:
    name: Python SDK Compatibility
    runs-on: [self-hosted, linux, x64, rust-cpu]
    timeout-minutes: 15
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main'

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Setup Rust
        uses: dtolnay/rust-toolchain@1.88

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install Python MCP SDK
        run: |
          pip install --upgrade pip
          pip install mcp aiohttp websockets pytest pytest-asyncio

      - name: Build framework
        run: cargo build --all-features --release

      - name: Run Python compatibility tests
        run: |
          # Python compatibility example not implemented yet
          echo "Python compatibility tests not implemented yet"
        env:
          RUST_LOG: info

      - name: Test cross-language scenarios
        run: |
          # TODO: Implement cross-language testing once we have a proper HTTP test server
          echo "Cross-language testing not implemented yet - requires HTTP server"

  external-validator-integration:
    name: External Validator Integration
    runs-on: [self-hosted, linux, x64, rust-cpu]
    timeout-minutes: 20
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Setup Rust
        uses: dtolnay/rust-toolchain@1.88

      - name: Build validation tools
        run: cargo build --package pulseengine-mcp-external-validation --features "proptest,fuzzing" --release

      - name: Test MCP Validator connectivity
        run: |
          # Test if external validators are accessible
          # Basic validation example not implemented yet
          echo "Basic validation example not implemented yet"
        continue-on-error: true

      - name: Run validation against reference implementations
        run: |
          # Test against known good MCP servers (currently no public servers available)
          echo "No public MCP reference servers available yet"
          echo "Skipping external server validation"

          # TODO: Add real server URLs when available:
          # SERVERS=(
          #   "https://mcp-test-server.example.com"
          #   "https://reference.mcp-server.org"
          # )
          # for server in "${SERVERS[@]}"; do
          #   echo "Testing $server..."
          #   cargo run --bin mcp-validate -- --server-url "$server" --quick || true
          # done
        continue-on-error: true

  security-validation:
    name: Security Validation
    # Stays on ubuntu-latest: runs cargo audit; smithy's pinned cargo-audit (0.21.x) rejects CVSS 4.0
    # advisories (e.g. RUSTSEC-2026-0037). Move once smithy bumps cargo-audit to >=0.22.1.
    runs-on: ubuntu-latest
    timeout-minutes: 15
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main'

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Setup Rust
        uses: dtolnay/rust-toolchain@1.88

      - name: Run cargo audit
        run: |
          cargo install cargo-audit
          cargo audit || echo "Warning: cargo audit failed due to edition2024 issue, continuing..."

      - name: Run security lints
        run: |
          cargo clippy --all-features --all-targets --release -- -D warnings

      - name: Check for security patterns
        run: |
          # Check for common security anti-patterns
          ! grep -r "unwrap()" --include="*.rs" src/ || echo "Warning: Found unwrap() calls"
          ! grep -r "panic!" --include="*.rs" src/ || echo "Warning: Found panic! macros"
          ! grep -r "unsafe" --include="*.rs" src/ || echo "Warning: Found unsafe blocks"

  benchmark-validation:
    name: Performance Benchmarks
    runs-on: [self-hosted, linux, x64, rust-cpu]
    timeout-minutes: 25
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Setup Rust
        uses: dtolnay/rust-toolchain@1.88

      - name: Run benchmarks
        run: |
          cargo bench --package pulseengine-mcp-external-validation

      - name: Upload benchmark results
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-results
          path: target/criterion