evals.yml

# Eval regression gate.
#
# Prompts are code. They ship, they regress, and they deserve a test suite.
# This runs the golden set on any PR that touches the parts of the system that
# can change answer quality, and fails if a gated metric drops more than the
# tolerance against the latest run on main.
#
# It costs real money per run, which is why the path filter is narrow and the
# job is skipped for forks — a public repo whose CI spends the owner's API
# budget on every drive-by PR is a mistake you only make once.

name: Evals

on:
  pull_request:
    paths:
      - "app/generation/prompts/**"
      - "app/generation/**"
      - "app/retrieval/**"
      - "app/quality/**"
      - "app/ingest/chunk.py"
      - "app/evals/configs.py"
      - "evals/golden_set.jsonl"
  workflow_dispatch:
    inputs:
      config:
        description: "Config to evaluate"
        default: "D"
      tolerance:
        description: "Maximum tolerated drop (negative)"
        default: "-0.03"

concurrency:
  group: evals-${{ github.ref }}
  cancel-in-progress: true

jobs:
  eval:
    runs-on: ubuntu-latest
    # Forks do not receive secrets, so the run would fail confusingly rather
    # than usefully.
    if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'workflow_dispatch'
    timeout-minutes: 30

    env:
      ENVIRONMENT: ci
      DATABASE_URL: ${{ secrets.EVAL_DATABASE_URL }}
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      CONFIG: ${{ github.event.inputs.config || 'D' }}
      TOLERANCE: ${{ github.event.inputs.tolerance || '-0.03' }}

    steps:
      - name: Check that the eval secrets are configured
        id: secrets
        # The eval gate needs a database and an API key. Until those secrets
        # exist the whole job is skipped with an explanatory notice rather than
        # failing — a workflow that goes red because an optional secret is
        # missing is a workflow people learn to ignore, and this one is meant
        # to be trusted enough to block a merge.
        run: |
          if [ -z "${{ secrets.EVAL_DATABASE_URL }}" ] || [ -z "${{ secrets.OPENAI_API_KEY }}" ]; then
            echo "configured=false" >> "$GITHUB_OUTPUT"
            echo "::notice title=Eval gate skipped::Set the EVAL_DATABASE_URL and OPENAI_API_KEY repository secrets to enable it. See the README."
          else
            echo "configured=true" >> "$GITHUB_OUTPUT"
          fi

      - uses: actions/checkout@v4
        if: steps.secrets.outputs.configured == 'true'
        with:
          # compare.py reads the branch name from git, so it needs real refs.
          fetch-depth: 0

      - uses: actions/setup-python@v5
        if: steps.secrets.outputs.configured == 'true'
        with:
          python-version: "3.12"
          cache: pip

      - name: Install dependencies
        if: steps.secrets.outputs.configured == 'true'
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt

      - name: Migrate
        if: steps.secrets.outputs.configured == 'true'
        run: alembic upgrade head

      - name: Run evals
        if: steps.secrets.outputs.configured == 'true'
        # The corpus is assumed already ingested into the shared eval database.
        # Re-ingesting on every PR would cost more than the eval itself and add
        # embedding drift as a confounder — the corpus is a fixture, not part of
        # what is under test.
        run: |
          python -m app.evals.run \
            --config "$CONFIG" \
            --golden evals/golden_set.jsonl \
            --concurrency 4 \
            --output evals/out/run.json \
            --notes "PR #${{ github.event.pull_request.number }}"

      - name: Compare against main
        if: steps.secrets.outputs.configured == 'true'
        id: compare
        run: |
          python -m app.evals.compare \
            --config "$CONFIG" \
            --baseline main \
            --fail-under "$TOLERANCE" \
            | tee evals/out/comparison.txt

      - name: Comment the results on the PR
        if: always() && steps.secrets.outputs.configured == 'true' && github.event_name == 'pull_request'
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const read = (p) => { try { return fs.readFileSync(p, 'utf8'); } catch { return null; } };
            const comparison = read('evals/out/comparison.txt') ?? '_comparison did not run_';
            const passed = '${{ steps.compare.outcome }}' === 'success';
            const body = [
              `### ${passed ? 'Eval gate passed' : 'Eval gate FAILED'} — config \`${process.env.CONFIG}\``,
              '',
              '```',
              comparison.trim(),
              '```',
              '',
              '<sub>Gated on retrieval and answer-quality metrics. Cost and latency are reported but not gated — they are a trade, not a failure.</sub>',
            ].join('\n');
            await github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body,
            });

      - name: Upload artefacts
        if: always() && steps.secrets.outputs.configured == 'true'
        uses: actions/upload-artifact@v4
        with:
          name: eval-run-${{ github.run_id }}
          path: evals/out/
          retention-days: 30