eval-benchmarks.yaml

name: Run eval benchmarks

on:
  # Allow manual trigger
  workflow_dispatch:
    inputs:
      benchmark_type:
        description: 'Benchmark type (fast-benchmark or full-benchmark). Cannot be combined with custom test_markers.'
        required: false
        type: choice
        options:
          - 'fast-benchmark'
          - 'full-benchmark'
        default: 'fast-benchmark'
      models:
        description: 'Comma-separated list of models to test'
        required: false
        # NOTE: Keep in sync with DEFAULT_BENCHMARK_MODELS env var
        default: 'opus-4.7,opus-4.6,sonnet-4.6,haiku-4.5,gpt-5.4,gemini-3.1-pro-preview,qwen-next-80B-instruct,qwen-next-80B-thinking,deepseek-r1-reasoner,deepseek-v3.2-chat,gpt-5.3-codex'
      test_markers:
        description: 'Custom pytest markers (ONLY use if not using benchmark_type). Cannot be combined with benchmark_type.'
        required: false
        default: ''
      iterations:
        description: 'Number of iterations per test (max 10)'
        required: false
        default: '5'

  # Run weekly on Sunday at 2 AM UTC
  schedule:
    - cron: '0 2 * * 0'

env:
  # Default models for benchmarks - single source of truth
  # NOTE: Keep workflow_dispatch default in sync with this for UI display
  DEFAULT_BENCHMARK_MODELS: 'opus-4.7,opus-4.6,sonnet-4.6,haiku-4.5,gpt-5.4,gemini-3.1-pro-preview,qwen-next-80B-instruct,qwen-next-80B-thinking,deepseek-r1-reasoner,deepseek-v3.2-chat,gpt-5.3-codex'

jobs:
  run-benchmarks:
    runs-on: ubuntu-latest
    timeout-minutes: 240

    permissions:
      contents: write
      pull-requests: write

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Setup HolmesGPT environment
        uses: ./.github/actions/setup-holmes-env
        with:
          python-version: '3.12'
          install-kubectl: 'true'

      - name: Setup KIND cluster
        uses: ./.github/actions/setup-kind-cluster
        with:
          cluster-name: 'kind'
          wait-for-ready: 'true'

      - name: Build benchmark command
        id: build-command
        run: |
          # Start with base command
          CMD="./run_benchmarks_local.py"

          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
            BENCHMARK_TYPE="${{ github.event.inputs.benchmark_type }}"
            CUSTOM_MARKERS="${{ github.event.inputs.test_markers }}"
            MODELS="${{ github.event.inputs.models }}"
            ITERATIONS="${{ github.event.inputs.iterations }}"

            # Add benchmark type or custom markers (script validates they're mutually exclusive)
            if [ -n "$CUSTOM_MARKERS" ]; then
              CMD="$CMD --markers \"$CUSTOM_MARKERS\""
            elif [ -n "$BENCHMARK_TYPE" ]; then
              CMD="$CMD --benchmark-type $BENCHMARK_TYPE"
            fi

            # Add models only if specified (otherwise use script defaults)
            if [ -n "$MODELS" ]; then
              CMD="$CMD --models $MODELS"
            fi

            # Add iterations if specified
            if [ -n "$ITERATIONS" ] && [ "$ITERATIONS" != "1" ]; then
              CMD="$CMD --iterations $ITERATIONS"
            fi
          else
            # Scheduled run: use fast-benchmark with default models
            BENCHMARK_TYPE="fast-benchmark"
            CMD="$CMD --benchmark-type fast-benchmark --models $DEFAULT_BENCHMARK_MODELS"
          fi

          echo "command=$CMD" >> $GITHUB_OUTPUT
          echo "benchmark_type=$BENCHMARK_TYPE" >> $GITHUB_OUTPUT

      - name: Create model list file
        run: |
          cat > /tmp/model_list.yaml << 'EOF'
          ${{ secrets.MODEL_LIST_YAML }}
          EOF

      - name: Run evaluation benchmarks
        id: run-benchmarks
        env:
          AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
          AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
          AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }}
          AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
          AWS_REGION_NAME: ${{ vars.AWS_REGION_NAME }}
          BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
          MOONSHOT_API_KEY: ${{ secrets.MOONSHOT_API_KEY }}
          CONFLUENCE_BASE_URL: ${{ secrets.CONFLUENCE_BASE_URL }}
          CONFLUENCE_API_KEY: ${{ secrets.CONFLUENCE_API_KEY }}
          MODEL_LIST_FILE_LOCATION: /tmp/model_list.yaml
          EXPERIMENT_ID: "ci-benchmark-${{ github.run_id }}"
        run: |
          echo "Running: ${{ steps.build-command.outputs.command }}"
          ${{ steps.build-command.outputs.command }}

      - name: Upload eval results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: eval-results-${{ github.run_id }}
          path: |
            docs/development/evaluations/fast-benchmark-results.md
            docs/development/evaluations/full-benchmark-results.md
            docs/development/evaluations/latest-results.md
            eval_results.json

      - name: Create PR with benchmark results
        id: create-pr
        if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          # Configure git
          git config --local user.email "github-actions[bot]@users.noreply.github.com"
          git config --local user.name "github-actions[bot]"

          # Create timestamped branch
          BRANCH_NAME="automated/benchmark-$(date +%Y%m%d_%H%M%S)"
          git checkout -b "$BRANCH_NAME"

          # Add all evaluation results
          git add docs/development/evaluations/

          # Check if there are changes to commit
          if git diff --staged --quiet; then
            echo "No changes to commit"
            exit 0
          fi

          # Commit and push
          BENCHMARK_TYPE="${{ steps.build-command.outputs.benchmark_type }}"
          git commit -m "Update ${BENCHMARK_TYPE:-benchmark} results"
          git push origin "$BRANCH_NAME" --force

          # Create PR (or update existing)
          PR_TITLE="Weekly Benchmark Results $(date +%Y-%m-%d_%H-%M)"
          PR_BODY="Automated weekly benchmark results from CI.

          **Benchmark Type**: ${BENCHMARK_TYPE:-custom}
          **Command**: ${{ steps.build-command.outputs.command }}"

          # Check if PR already exists
          EXISTING_PR=$(gh pr list --head "$BRANCH_NAME" --json number --jq '.[0].number')
          if [ -n "$EXISTING_PR" ]; then
            echo "Updating existing PR #$EXISTING_PR"
            gh pr edit "$EXISTING_PR" \
              --title "$PR_TITLE" \
              --body "$PR_BODY"
            echo "pr_url=$(gh pr view "$EXISTING_PR" --json url --jq '.url')" >> $GITHUB_OUTPUT
          else
            PR_URL=$(gh pr create \
              --title "$PR_TITLE" \
              --body "$PR_BODY" \
              --base master \
              --head "$BRANCH_NAME" 2>&1 | tail -1)
            echo "pr_url=$PR_URL" >> $GITHUB_OUTPUT
          fi

      - name: Notify Slack on success
        if: steps.run-benchmarks.outcome == 'success'
        env:
          SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
        run: |
          PR_URL="${{ steps.create-pr.outputs.pr_url }}"
          curl -X POST https://slack.com/api/chat.postMessage \
            -H "Authorization: Bearer $SLACK_TOKEN" \
            -H "Content-Type: application/json" \
            -d "{\"channel\": \"#backend\", \"text\": \"Weekly eval benchmarks finished! Here is the PR: ${PR_URL}\"}"

      - name: Notify Slack on failure or cancellation
        if: steps.run-benchmarks.outcome != 'success'
        env:
          SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
        run: |
          RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
          if [ "${{ job.status }}" = "cancelled" ]; then
            STATUS="cancelled"
          else
            STATUS="failed"
          fi
          curl -X POST https://slack.com/api/chat.postMessage \
            -H "Authorization: Bearer $SLACK_TOKEN" \
            -H "Content-Type: application/json" \
            -d "{\"channel\": \"#backend-oncall\", \"text\": \"Weekly eval benchmarks ${STATUS}! See the run: ${RUN_URL}\"}"