benchmark.yml

name: Benchmark

on:
  pull_request:
    branches: [main]
  push:
    branches: [main]

concurrency:
  group: benchmark-${{ github.ref }}
  cancel-in-progress: true

jobs:
  benchmark:
    runs-on: ubuntu-latest
    timeout-minutes: 5
    permissions:
      contents: write
      pull-requests: write

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Setup pnpm
        uses: pnpm/action-setup@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 22
          cache: 'pnpm'

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      - name: Build core and benchmarks
        run: pnpm --filter core build && pnpm --filter benchmarks build

      - name: Run Tier 1 synthetic benchmark
        id: benchmark
        run: |
          node packages/benchmarks/dist/ci/run-ci-entrypoint.js \
            --commit "${{ github.sha }}" \
            --branch "${{ github.head_ref || github.ref_name }}" \
            --baseline "packages/benchmarks/results/baseline.json" \
            --history "packages/benchmarks/results/history/" \
            --output "/tmp/benchmark-report.json"

      - name: Read benchmark report
        id: report
        if: always()
        run: |
          if [ -f /tmp/benchmark-report.json ]; then
            # Extract status and comment from report
            STATUS=$(node -e "const r=JSON.parse(require('fs').readFileSync('/tmp/benchmark-report.json','utf-8'));console.log(r.hasRegression?'failure':'success')")
            echo "status=$STATUS" >> "$GITHUB_OUTPUT"

            # Extract markdown comment
            COMMENT=$(node -e "const r=JSON.parse(require('fs').readFileSync('/tmp/benchmark-report.json','utf-8'));console.log(r.comment)")
            # Write to file for multi-line output
            echo "$COMMENT" > /tmp/benchmark-comment.md
          else
            echo "status=failure" >> "$GITHUB_OUTPUT"
            echo "Benchmark report not found" > /tmp/benchmark-comment.md
          fi

      - name: Post PR comment
        if: github.event_name == 'pull_request' && always()
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const comment = fs.readFileSync('/tmp/benchmark-comment.md', 'utf-8');

            // Find existing benchmark comment
            const { data: comments } = await github.rest.issues.listComments({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
            });

            const botComment = comments.find(c =>
              c.body?.includes('Generated by CodeRAG Benchmark CI')
            );

            if (botComment) {
              // Update existing comment
              await github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: botComment.id,
                body: comment,
              });
            } else {
              // Create new comment
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: context.issue.number,
                body: comment,
              });
            }

      - name: Update baseline on main
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        continue-on-error: true
        run: |
          if [ -f /tmp/benchmark-new-baseline.json ]; then
            cp /tmp/benchmark-new-baseline.json packages/benchmarks/results/baseline.json
            git config user.name "github-actions[bot]"
            git config user.email "github-actions[bot]@users.noreply.github.com"
            git add packages/benchmarks/results/baseline.json packages/benchmarks/results/history/
            git diff --cached --quiet || git commit -m "chore: update benchmark baseline [skip ci]"
            git push || echo "::warning::Could not push baseline update (branch protection). Baseline will be updated on next unprotected push."
          fi

      - name: Fail on regression
        if: steps.report.outputs.status == 'failure'
        run: |
          echo "Benchmark regression detected! See PR comment for details."
          exit 1