benchmark.yml

name: benchmark

# Nightly ASV benchmark trend (advisory; never blocks PRs). Results accumulate
# across runs via the cache so step-detection sees the whole history; a detected
# wall-time regression opens/updates a tracking issue. Deterministic call-count
# regressions are gated separately by test/integration/test_perf_counters.py (blocking, PRs).

on:
  schedule:
    - cron: "0 7 * * *"  # nightly, 07:00 UTC
  workflow_dispatch: {}    # manual trigger for verification

permissions:
  issues: write

jobs:
  asv:
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0   # asv needs history to benchmark commits

      - name: Set up uv
        uses: astral-sh/setup-uv@v5
        with:
          enable-cache: true
          cache-dependency-glob: "uv.lock"

      - name: Set up Python 3.13
        run: uv python install 3.13

      - name: Install dependencies
        run: uv sync --python 3.13 --group dev --all-extras

      - name: Restore accumulated ASV results
        uses: actions/cache@v4
        with:
          path: benchmarks/results
          key: asv-results-${{ github.run_id }}
          restore-keys: asv-results-

      - name: Configure ASV machine
        working-directory: benchmarks
        run: uv run asv machine --yes

      - name: Run benchmarks for new commits
        working-directory: benchmarks
        run: uv run asv run NEW --skip-existing --show-stderr || true

      - name: Detect regression (HEAD vs previous commit, same runner)
        id: detect
        working-directory: benchmarks
        run: |
          # `asv continuous` exits non-zero for BOTH a real regression and a
          # benchmark build/run failure, so the exit code alone can't tell them
          # apart. Parse the sentinel line, which asv prints only after a
          # comparison actually completes.
          out=$(uv run asv continuous --factor 1.5 HEAD~1 HEAD --show-stderr 2>&1) || true
          echo "$out"
          if grep -q "PERFORMANCE DECREASED" <<<"$out"; then
            echo "regressed=true" >> "$GITHUB_OUTPUT"
          elif grep -qE "BENCHMARKS NOT SIGNIFICANTLY CHANGED|PERFORMANCE INCREASED" <<<"$out"; then
            echo "regressed=false" >> "$GITHUB_OUTPUT"
          else
            echo "::error::asv continuous did not complete a comparison; benchmark build/run failure (not a regression)"
            exit 1
          fi

      - name: Open/update regression issue
        if: steps.detect.outputs.regressed == 'true'
        uses: actions/github-script@v7
        with:
          script: |
            const title = "Nightly benchmark regression detected";
            const url = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
            const body = `ASV flagged a wall-time regression on \`main\`. Workflow run: ${url}`;
            const existing = await github.rest.issues.listForRepo({
              owner: context.repo.owner,
              repo: context.repo.repo,
              state: "open",
              labels: "benchmark-regression",
            });
            if (existing.data.length) {
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: existing.data[0].number,
                body,
              });
            } else {
              await github.rest.issues.create({
                owner: context.repo.owner,
                repo: context.repo.repo,
                title,
                body,
                labels: ["benchmark-regression"],
              });
            }