eval-nightly.yml

# Eval harness nightly — disabled-by-default.
#
# This workflow runs the golden QA dataset against the agent / LLM loop. It
# is `workflow_dispatch`-only by default to prevent accidental LLM API
# spend. To enable nightly runs:
#
#   1. Set the LLM secrets in repo settings (LLM_API_KEY at minimum;
#      LLM_BASE_URL / LLM_MODEL / LLM_PROVIDER if your judge differs from
#      OpenAI defaults).
#   2. Replace the `on:` block below with:
#
#        on:
#          schedule:
#            - cron: "0 6 * * *"   # daily 06:00 UTC
#          workflow_dispatch:
#
#   3. Add the `eval-nightly.yml` to EXEMPT_WORKFLOWS in
#      `.github/scripts/check_required_contexts.py` if it's not already
#      there (it is, by default — scheduled runs never gate PRs).
#
# See docs/EVAL_HARNESS.md for the full setup story.

name: Eval nightly

on:
  workflow_dispatch:
    inputs:
      python_version:
        description: "Python version to use"
        required: false
        default: "3.14"

permissions:
  contents: read

jobs:
  eval:
    name: Run golden QA dataset
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4
      - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57  # v8
      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
        with:
          python-version: ${{ inputs.python_version || '3.14' }}
      - run: uv sync --frozen --extra dev
      - name: Run pytest eval/
        env:
          LLM_PROVIDER: ${{ secrets.LLM_PROVIDER }}
          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
          LLM_MODEL: ${{ secrets.LLM_MODEL }}
        run: uv run pytest eval/ -v