benchmark-append.yml

name: Add new model to benchmark

on:
  workflow_dispatch:
    inputs:
      model:
        description: "Model to benchmark"
        required: true
        default: "openai/gpt-4o"

jobs:
  benchmark:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '18'
          cache: 'npm'
          cache-dependency-path: src/package-lock.json

      - name: Install dependencies
        working-directory: src
        run: npm ci

      - name: Verify Tinybird credentials
        env:
          TINYBIRD_API_HOST: ${{ vars.TINYBIRD_API_HOST }}
          TINYBIRD_WORKSPACE_TOKEN: ${{ secrets.TINYBIRD_WORKSPACE_TOKEN }}
        run: |
          STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
            "${TINYBIRD_API_HOST}/v0/sql?q=SELECT+1+FORMAT+JSON" \
            -H "Authorization: Bearer ${TINYBIRD_WORKSPACE_TOKEN}")
          if [ "$STATUS" != "200" ]; then
            echo "::error::Tinybird credentials check failed (HTTP $STATUS). Check TINYBIRD_API_HOST and TINYBIRD_WORKSPACE_TOKEN."
            exit 1
          fi
          echo "Tinybird credentials verified"

      - name: Run benchmark
        id: run-benchmark
        working-directory: src
        run: npm run benchmark -- --model="${{ github.event.inputs.model }}" --debug
        env:
          TINYBIRD_API_HOST: ${{ vars.TINYBIRD_API_HOST }}
          TINYBIRD_WORKSPACE_TOKEN: ${{ secrets.TINYBIRD_WORKSPACE_TOKEN }}
          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}

      - name: Verify benchmark quality
        id: verify-quality
        if: success()
        working-directory: src
        env:
          TINYBIRD_API_HOST: ${{ vars.TINYBIRD_API_HOST }}
          TINYBIRD_WORKSPACE_TOKEN: ${{ secrets.TINYBIRD_WORKSPACE_TOKEN }}
        run: |
          MODEL="${{ github.event.inputs.model }}"
          PROVIDER=$(echo "$MODEL" | cut -d'/' -f1)
          MODEL_NAME=$(echo "$MODEL" | cut -d'/' -f2-)

          SUCCESS_RATE=$(curl -s "${TINYBIRD_API_HOST}/v0/pipes/api_model_metrics.json?include_unvalidated=1" \
            -H "Authorization: Bearer ${TINYBIRD_WORKSPACE_TOKEN}" | node -e "
            const data = JSON.parse(require('fs').readFileSync('/dev/stdin','utf8'));
            const rows = data.data || [];
            const m = rows.find(r => r.model === '${MODEL_NAME}' && r.provider === '${PROVIDER}');
            console.log(m ? (m.success_rate || 0).toFixed(1) : '0');
          ")

          echo "Success rate for $MODEL: ${SUCCESS_RATE}%"
          if (( $(echo "$SUCCESS_RATE < 20" | bc -l) )); then
            echo "::error::Benchmark quality too low (${SUCCESS_RATE}% success rate)"
            exit 1
          fi

      - name: Report benchmark failure
        if: failure() && (steps.run-benchmark.outcome == 'failure' || steps.verify-quality.outcome == 'failure')
        working-directory: src
        run: |
          MODEL="${{ github.event.inputs.model }}"
          PROVIDER=$(echo "$MODEL" | cut -d'/' -f1)
          MODEL_NAME=$(echo "$MODEL" | cut -d'/' -f2)

          REASON="Benchmark workflow failed"
          if [ "${{ steps.verify-quality.outcome }}" = "failure" ]; then
            REASON="Benchmark quality too low (success rate below 20%)"
          fi

          echo "Benchmark failed for $MODEL: $REASON"
          echo "Adding to failed models list..."

          npm run manage-failed-models add "$PROVIDER" "$MODEL_NAME" "$MODEL" "$REASON"

          echo "Model $MODEL has been added to the failed list"

      - name: Commit failure changes
        if: failure() && (steps.run-benchmark.outcome == 'failure' || steps.verify-quality.outcome == 'failure')
        run: |
          git config --local user.email "action@github.com"
          git config --local user.name "GitHub Action"

          # Check if there are changes to commit
          if git diff --quiet; then
            echo "No changes to commit"
          else
            git add src/failed-models.json
            git commit -m "Add failed model ${{ github.event.inputs.model }} to failed list

          - Model: ${{ github.event.inputs.model }}
          - Reason: Benchmark workflow failed
          - Run ID: ${{ github.run_id }}

          This prevents the model from being automatically benchmarked again."

            git pull --rebase origin main
            git push origin main
            echo "Failure changes committed and pushed to main branch"
          fi

      - name: Create normalized branch name
        if: success() && steps.verify-quality.outcome == 'success'
        id: branch-name
        run: |
          MODEL_NAME="${{ github.event.inputs.model }}"
          NORMALIZED_MODEL=$(echo "$MODEL_NAME" | sed 's/[\/\s_\.:]/-/g')
          echo "branch_name=benchmark/$NORMALIZED_MODEL-${{ github.run_id }}" >> $GITHUB_OUTPUT

      - name: Create benchmark branch
        if: success() && steps.verify-quality.outcome == 'success'
        run: |
          MODEL="${{ github.event.inputs.model }}"
          BRANCH="${{ steps.branch-name.outputs.branch_name }}"

          git config --local user.email "action@github.com"
          git config --local user.name "GitHub Action"
          git checkout -b "$BRANCH"

          mkdir -p src/benchmark-runs
          echo "{\"model\":\"$MODEL\",\"run_id\":${{ github.run_id }},\"timestamp\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}" \
            > "src/benchmark-runs/$(echo "$MODEL" | sed 's/\//__/g').json"

          git add src/benchmark-runs/
          git commit -m "feat: add benchmark results for $MODEL"
          git push origin "$BRANCH"

      - name: Create Pull Request
        if: success() && steps.verify-quality.outcome == 'success'
        id: create-pr
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          MODEL="${{ github.event.inputs.model }}"
          BRANCH="${{ steps.branch-name.outputs.branch_name }}"
          ASSIGNEES="${{ vars.PR_ASSIGNEES || '' }}"

          PR_URL=$(gh pr create \
            --title "Add benchmark results for $MODEL" \
            --body "$(cat <<'BODY'
          This PR adds benchmark results for the **${{ github.event.inputs.model }}** model.

          Results have been pushed to Tinybird with `validated=0` (pending review).
          Merging this PR will validate the results, making them visible on the production dashboard.

          This PR was automatically generated by the benchmark workflow.

          **Note:** If you don't want to merge this PR, close it and the model will be added to the failed list.

          ${{ vars.PR_ASSIGNEES && format('/cc {0}', vars.PR_ASSIGNEES) || '' }}
          BODY
          )" \
            --base main \
            --head "$BRANCH" \
            ${ASSIGNEES:+--assignee "$ASSIGNEES"})

          PR_NUMBER=$(echo "$PR_URL" | grep -oE '[0-9]+$')
          echo "pull-request-number=$PR_NUMBER" >> $GITHUB_OUTPUT
          echo "Created PR #$PR_NUMBER: $PR_URL"

      - name: LLM review and auto-merge
        if: success() && steps.create-pr.outputs.pull-request-number
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
          TINYBIRD_API_HOST: ${{ vars.TINYBIRD_API_HOST }}
          TINYBIRD_WORKSPACE_TOKEN: ${{ secrets.TINYBIRD_WORKSPACE_TOKEN }}
        run: |
          PR_NUMBER="${{ steps.create-pr.outputs.pull-request-number }}"
          MODEL="${{ github.event.inputs.model }}"
          REVIEW_MODEL="${{ vars.REVIEW_MODEL || 'openai/gpt-5.4-nano' }}"
          AUTO_MERGE="${{ vars.AUTO_MERGE || 'true' }}"

          echo "Generating LLM review for PR #$PR_NUMBER (model: $MODEL)"
          echo "Using review model: $REVIEW_MODEL"

          PROVIDER=$(echo "$MODEL" | cut -d'/' -f1)
          MODEL_NAME=$(echo "$MODEL" | cut -d'/' -f2-)

          # Query Tinybird for benchmark metrics (include unvalidated results)
          METRICS=$(curl -s "${TINYBIRD_API_HOST}/v0/pipes/api_model_metrics.json?include_unvalidated=1" \
            -H "Authorization: Bearer ${TINYBIRD_WORKSPACE_TOKEN}" | node -e "
            const data = JSON.parse(require('fs').readFileSync('/dev/stdin','utf8'));
            const rows = data.data || [];
            const m = rows.find(r => r.model === '${MODEL_NAME}' && r.provider === '${PROVIDER}');
            if (m) {
              console.log(JSON.stringify({
                total: m.total_queries,
                successful: m.successful_queries,
                errors: m.total_queries - m.successful_queries,
                avgLatency: (m.avg_total_duration || 0).toFixed(2),
                successRate: (m.success_rate || 0).toFixed(1),
                firstAttemptRate: (m.first_attempt_rate || 0).toFixed(1),
                avgExecutionTime: (m.avg_execution_time || 0).toFixed(4)
              }));
            } else {
              console.log(JSON.stringify({ total: 0, successful: 0, errors: 0, avgLatency: '0', successRate: '0' }));
            }
          ")

          SUCCESS_RATE=$(echo "$METRICS" | node -e "const d=JSON.parse(require('fs').readFileSync('/dev/stdin','utf8')); console.log(d.successRate)")

          # Call OpenRouter for LLM review
          REVIEW_PROMPT="You are reviewing benchmark results for the LLM model '$MODEL'.

          Metrics:
          $METRICS

          Analyze these results and provide:
          1. A brief quality summary (2-3 sentences)
          2. Any concerns or anomalies
          3. Your recommendation: MERGE (results look reasonable) or REVIEW (needs human attention)

          Keep your response concise. End with exactly one line: 'Recommendation: MERGE' or 'Recommendation: REVIEW'"

          REVIEW_RESPONSE=$(curl -s https://openrouter.ai/api/v1/chat/completions \
            -H "Authorization: Bearer $OPENROUTER_API_KEY" \
            -H "Content-Type: application/json" \
            -d "$(node -e "console.log(JSON.stringify({
              model: '$REVIEW_MODEL',
              messages: [{ role: 'user', content: $(echo "$REVIEW_PROMPT" | node -e "process.stdout.write(JSON.stringify(require('fs').readFileSync('/dev/stdin','utf8')))") }],
              max_tokens: 500
            }))")" | node -e "
              const data = JSON.parse(require('fs').readFileSync('/dev/stdin','utf8'));
              console.log(data.choices?.[0]?.message?.content || 'Review unavailable');
            ")

          # Post review comment on PR
          COMMENT_BODY="## Automated Benchmark Review

          **Model:** \`$MODEL\`
          **Review model:** \`$REVIEW_MODEL\`
          **Success rate:** ${SUCCESS_RATE}%

          ---

          $REVIEW_RESPONSE

          ---
          *This review was automatically generated. Set \`AUTO_MERGE=false\` in repo variables to disable auto-merge.*"

          gh pr comment "$PR_NUMBER" --body "$COMMENT_BODY"

          # Auto-merge if review recommends it and AUTO_MERGE is enabled
          if [ "$AUTO_MERGE" = "true" ] && echo "$REVIEW_RESPONSE" | grep -qi "Recommendation: MERGE"; then
            echo "LLM recommends MERGE. Validating results in Tinybird..."
            cd src
            npm run benchmark -- --model="${MODEL}" --validate
            cd ..
            echo "Results validated. Merging PR..."
            gh pr merge "$PR_NUMBER" --merge --auto || echo "Auto-merge failed (may need branch protection rules)"
          else
            echo "Skipping auto-merge. Review response or AUTO_MERGE setting requires manual review."
            gh pr edit "$PR_NUMBER" --add-label "needs-review" 2>/dev/null || echo "Could not add label"
          fi

      - name: Update benchmark config after merge
        # When the PR is auto-merged with GITHUB_TOKEN, GitHub does not fire
        # downstream workflows (recursion prevention), so benchmark-validate-on-merge.yml
        # never runs. We perform the config update and run-file cleanup inline here.
        if: success() && steps.create-pr.outputs.pull-request-number
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          PR_NUMBER="${{ steps.create-pr.outputs.pull-request-number }}"
          MODEL="${{ github.event.inputs.model }}"
          PROVIDER=$(echo "$MODEL" | cut -d'/' -f1)
          MODEL_NAME=$(echo "$MODEL" | cut -d'/' -f2-)

          # Poll for merge completion. Auto-merge usually settles in seconds
          # but may take longer if branch protection requires status checks.
          echo "Waiting for PR #$PR_NUMBER to merge..."
          MERGED_AT=""
          for i in $(seq 1 30); do
            INFO=$(gh pr view "$PR_NUMBER" --json state,mergedAt --jq '.state + "|" + (.mergedAt // "")')
            PR_STATE=$(echo "$INFO" | cut -d'|' -f1)
            MERGED_AT=$(echo "$INFO" | cut -d'|' -f2)
            if [ -n "$MERGED_AT" ]; then
              echo "PR merged at $MERGED_AT"
              break
            fi
            if [ "$PR_STATE" = "CLOSED" ]; then
              echo "::warning::PR closed without merging. Skipping config update."
              exit 0
            fi
            echo "  Not merged yet (attempt $i/30). Waiting 10s..."
            sleep 10
          done

          if [ -z "$MERGED_AT" ]; then
            echo "::warning::PR not merged after 5 minutes. Config update skipped."
            exit 0
          fi

          git config --local user.email "action@github.com"
          git config --local user.name "GitHub Action"
          git fetch origin main
          git checkout -B main origin/main

          node -e "
            const fs = require('fs');
            const path = 'src/benchmark-config.json';
            const config = JSON.parse(fs.readFileSync(path, 'utf8'));
            const provider = process.argv[1];
            const modelName = process.argv[2];
            if (!config.providers[provider]) config.providers[provider] = { models: [] };
            if (!config.providers[provider].models.includes(modelName)) {
              config.providers[provider].models.push(modelName);
              fs.writeFileSync(path, JSON.stringify(config, null, 2));
              console.log('Added ' + modelName + ' to ' + provider);
            } else {
              console.log(modelName + ' already in ' + provider);
            }
          " "$PROVIDER" "$MODEL_NAME"

          RUN_FILE="src/benchmark-runs/$(echo "$MODEL" | sed 's/\//__/g').json"
          if [ -f "$RUN_FILE" ]; then
            git rm "$RUN_FILE"
            echo "Removed $RUN_FILE"
          fi

          git add src/benchmark-config.json
          if git diff --cached --quiet; then
            echo "No changes to commit"
            exit 0
          fi

          git commit -m "chore: update benchmark config for $MODEL"

          # Retry on push race when multiple append workflows finish near the same time
          for attempt in 1 2 3; do
            if git push origin main; then
              echo "Pushed config update on attempt $attempt"
              exit 0
            fi
            echo "Push failed on attempt $attempt. Rebasing..."
            git pull --rebase origin main
          done
          echo "::error::Could not push config update after 3 attempts"
          exit 1