perf.yml

name: Perf

# Phase 4 deliverables W0.4 (manual baseline lock) + W0.5 (per-PR
# advisory delta check). Both modes share the same measurement
# script (scripts/measure_perf_baseline.py) so a baseline locked
# here and a per-PR run are directly comparable.
#
# Two jobs:
#
# 1. `lock-baseline` — triggered manually via Actions UI. Runs
#    N=50 iterations (default) of all benchmarks, optionally
#    `--include-llm`, commits the locked files to
#    docs/specs/downstream-validation/. This is the mechanism for
#    W0.4 (the spec's "Lock initial perf baseline" deliverable).
#    N was bumped 30→50 pre-W3.1 to tighten the baseline mean
#    estimate on sub-millisecond benchmarks where the original
#    N=30 ran into inter-run variance the locked stdev didn't
#    capture (see PR after #74).
#
# 2. `delta-check` — triggered on every PR. Runs an N=30 LLM-free
#    measurement, compares against the locked baseline, posts an
#    advisory comment. Bumped from N=10 pre-W3.1: the original
#    N=10 produced ±23% swings between consecutive PRs on
#    identical perf-relevant code (rag_pipeline_run, PR #72/#74),
#    making the gate unreliable. N=30 cuts that swing by √3 ≈ 1.73x
#    and brings per-PR variance closer to the baseline's intra-run
#    measurement methodology. Advisory mode through W3.1 (per
#    design.md §2's promotion ramp); doesn't fail the PR.

on:
  pull_request:
    branches: [main]
  workflow_dispatch:
    inputs:
      runs:
        description: "Iterations per benchmark (>= 10)."
        required: false
        default: "50"
      include_llm:
        description: "Include LLMReranker.rerank (needs ANTHROPIC_API_KEY)?"
        required: false
        default: "false"
        type: choice
        options: ["false", "true"]

# Default: read-only. Each job widens only what it needs.
# This narrows the blast radius of the delta-check job, which
# executes PR-author-modified scripts on every PR — it must not
# inherit contents:write from the workflow-level scope.
permissions:
  contents: read

concurrency:
  group: perf-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  # ── W0.4 / Phase 5 M2: manual baseline lock (multi-run v2) ────
  #
  # Phase 5 perf-baseline-multi-run M2: split the single-invocation
  # lock into K=5 parallel matrix invocations + an aggregator job.
  # Each matrix entry runs N trials and emits a per-invocation JSON;
  # the aggregator combines K JSONs into the v2 locked baseline
  # (mean + intra_run_stdev + inter_run_stdev + threshold per
  # docs/specs/perf-baseline-multi-run/design.md §3).
  #
  # K=5 hardcoded per scoping decision #2 (requirements.md R3).
  # N defaults to inputs.runs (M2 spec calls 20; default kept at 50
  # for backward compat with v1 dispatches — pass runs=20 to switch).
  lock-baseline:
    if: github.event_name == 'workflow_dispatch'
    name: Lock baseline (invocation ${{ matrix.invocation_index }} / ${{ strategy.job-total }})
    runs-on: ubuntu-latest
    timeout-minutes: 20
    strategy:
      # One bad invocation shouldn't kill the others. The aggregator
      # job's `_validate_consistent` will fail loudly if the surviving
      # set isn't a contiguous 0..K-1 range — surfacing the gap.
      fail-fast: false
      matrix:
        invocation_index: [0, 1, 2, 3, 4]  # K=5
    permissions:
      # Matrix entries are read-only; the aggregator widens to
      # contents:write + pull-requests:write for the bot PR.
      contents: read
    env:
      RUNS: ${{ inputs.runs }}
      INCLUDE_LLM: ${{ inputs.include_llm }}
      INVOCATIONS: 5
      PER_INV_JSON: per-invocation-${{ matrix.invocation_index }}.json

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2

      - name: Set up Python 3.11
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
        with:
          python-version: "3.11"
          cache: pip

      - name: Install (LLM extras only if --include-llm)
        run: |
          set -euo pipefail
          python -m pip install --upgrade pip
          if [ "$INCLUDE_LLM" = "true" ]; then
            python -m pip install -e ".[claude,dev]"
          else
            python -m pip install -e ".[dev]"
          fi
        shell: bash

      - name: Validate input
        run: |
          set -euo pipefail
          if [ "$RUNS" -lt 10 ]; then
            echo "::error::runs must be >= 10 (got $RUNS)"
            exit 1
          fi
          if [ "$INCLUDE_LLM" = "true" ] && [ -z "${ANTHROPIC_API_KEY:-}" ]; then
            echo "::error::--include-llm requested but ANTHROPIC_API_KEY is not set in repo Secrets."
            exit 1
          fi
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
        shell: bash

      - name: Run per-invocation measurement
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
        run: |
          set -euo pipefail
          ARGS=(
            --runs "$RUNS"
            --invocations "$INVOCATIONS"
            --invocation-index "${{ matrix.invocation_index }}"
            --per-invocation-out "$PER_INV_JSON"
          )
          if [ "$INCLUDE_LLM" = "true" ]; then
            ARGS+=(--include-llm)
          fi
          python scripts/measure_perf_baseline.py "${ARGS[@]}"
          # Log runner fingerprint so design.md §3's clustering check
          # (M5 validation) has the data it needs.
          echo "::group::Runner fingerprint"
          python -c "import json,sys; p=json.load(open('$PER_INV_JSON')); print(json.dumps(p['environment'], indent=2))"
          echo "::endgroup::"
        shell: bash

      - name: Upload per-invocation artifact
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f  # v7.0.1
        with:
          name: perf-per-invocation-${{ matrix.invocation_index }}
          path: ${{ env.PER_INV_JSON }}
          retention-days: 7
          if-no-files-found: error

  # ── Phase 5 M2: aggregate K artifacts → v2 locked baseline ────
  aggregate-baseline:
    if: github.event_name == 'workflow_dispatch'
    name: Aggregate per-invocation JSONs (v2 lock)
    needs: lock-baseline
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      # Bot opens a PR with the locked baseline (direct push to
      # main is rejected by branch protection).
      contents: write       # to push the auto-branch
      pull-requests: write  # to open the PR
    env:
      BASELINE_MD: docs/specs/downstream-validation/perf-baseline.md
      THRESHOLDS_JSON: docs/specs/downstream-validation/perf-thresholds.json
      RUNS: ${{ inputs.runs }}
      INCLUDE_LLM: ${{ inputs.include_llm }}

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          # Need write access + full history to commit the locked
          # baseline back to main.
          token: ${{ secrets.GITHUB_TOKEN }}
          fetch-depth: 0

      - name: Set up Python 3.11
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
        with:
          python-version: "3.11"
          cache: pip

      - name: Install (aggregator is pure stdlib)
        run: |
          set -euo pipefail
          python -m pip install --upgrade pip
          python -m pip install -e ".[dev]"
        shell: bash

      - name: Download all per-invocation artifacts
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
        with:
          pattern: perf-per-invocation-*
          path: per-invocation-artifacts/
          merge-multiple: true

      - name: Aggregate to v2 locked baseline
        run: |
          set -euo pipefail
          ls -la per-invocation-artifacts/
          PER_INV_ARGS=()
          for f in per-invocation-artifacts/*.json; do
            PER_INV_ARGS+=(--per-invocation "$f")
          done
          if [ "${#PER_INV_ARGS[@]}" -lt 2 ]; then
            echo "::error::aggregator requires ≥2 per-invocation JSONs (got ${#PER_INV_ARGS[@]})"
            exit 1
          fi
          python scripts/aggregate_perf_baseline.py \
            "${PER_INV_ARGS[@]}" \
            --out "$BASELINE_MD" \
            --thresholds-out "$THRESHOLDS_JSON"
        shell: bash

      - name: Open PR with the locked baseline
        # main is a protected branch — direct push is rejected. Bot
        # creates a branch + commits + opens a PR; maintainer (or
        # `gh pr merge --admin`) lands it.
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          RUN_ID: ${{ github.run_id }}
        run: |
          set -euo pipefail
          git config user.name 'github-actions[bot]'
          git config user.email 'github-actions[bot]@users.noreply.github.com'
          git add "$BASELINE_MD" "$THRESHOLDS_JSON"
          if git diff --cached --quiet; then
            echo "::notice::No changes to baseline files; skipping PR."
            exit 0
          fi
          STAMP=$(date -u +%Y%m%d-%H%M%S)
          BRANCH="auto/perf-baseline-v2-${STAMP}-r${RUN_ID}"
          git checkout -b "$BRANCH"
          git commit -m "perf: lock perf baseline v2 (K=5, runs=$RUNS, include_llm=$INCLUDE_LLM)

          Phase 5 perf-baseline-multi-run M3. K=5 parallel invocations
          (matrix), each N=$RUNS trials. Aggregator emits the v2
          methodology schema: mean / intra_run_stdev / inter_run_stdev /
          threshold (= mean + sigma × inter_run_stdev). See
          docs/specs/perf-baseline-multi-run/design.md for the noise-
          decomposition rationale."
          git push --set-upstream origin "$BRANCH"
          gh pr create \
            --base main \
            --head "$BRANCH" \
            --title "perf: lock perf baseline v2 (K=5, runs=$RUNS, include_llm=$INCLUDE_LLM)" \
            --body "Auto-generated by the \`perf\` workflow's \`lock-baseline\` dispatch (run id \`$RUN_ID\`).

          **Multi-run methodology v2** (\`docs/specs/perf-baseline-multi-run/\`):

          - **K**: 5 invocations (parallel matrix)
          - **N**: $RUNS trials per invocation
          - **include_llm**: $INCLUDE_LLM
          - **Schema additions**: \`intra_run_stdev\` (within-invocation jitter), \`inter_run_stdev\` (between-invocation drift), \`runs_per_invocation\`, \`invocations\`, \`methodology_version: 2\`. Backward-compat aliases held: \`mean\` / \`stdev\` (= \`inter_run_stdev\`) / \`threshold\` (= mean + sigma × inter_run_stdev).
          - **Files**: \`docs/specs/downstream-validation/perf-baseline.md\` + \`docs/specs/downstream-validation/perf-thresholds.json\`.

          Merging this lands the v2 locked baseline on \`main\`; subsequent per-PR \`delta-check\` comments will compare against the inter-run-stdev-based threshold. Follow-up (M3.2 σ rollback PR) drops \`DEFAULT_SIGMA\` 3.0 → 2.0 once a v2 baseline is on main."
        shell: bash

  # ── W0.5: per-PR advisory delta check ─────────────────────────
  delta-check:
    if: github.event_name == 'pull_request'
    name: Perf delta (advisory)
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      # Runs PR-author code. Read-only on contents; only the bot
      # comment needs write.
      contents: read
      pull-requests: write
    env:
      BASELINE_JSON: docs/specs/downstream-validation/perf-thresholds.json
      # Workspace-relative paths so hashFiles() works in step
      # conditionals (it only matches files under GITHUB_WORKSPACE
      # — silently returns '' for /tmp paths, defeating the guard).
      CURRENT_JSON: perf-current.json
      CURRENT_MD: perf-current.md  # not used downstream, but required by the script
      COMMENT_PATH: perf-comment.md

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          fetch-depth: 0

      - name: Set up Python 3.11
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
        with:
          python-version: "3.11"
          cache: pip

      - name: Install (dev extras only — LLM-free measurement)
        run: |
          python -m pip install --upgrade pip
          python -m pip install -e ".[dev]"
        shell: bash

      # ── Measure current HEAD on this runner (no LLM, N=10) ────
      # continue-on-error so a script crash doesn't kill the
      # whole job silently — the next step branches on outcome
      # and emits a "measurement failed" comment instead.
      - name: Measure current
        id: measure
        continue-on-error: true
        run: |
          set -euo pipefail
          python scripts/measure_perf_baseline.py \
            --runs 30 \
            --out "$CURRENT_MD" \
            --thresholds-out "$CURRENT_JSON"
        shell: bash

      # ── Format the delta comment ─────────────────────────────
      # Branches on the measure step's outcome:
      # - success → run format_perf_delta.py as usual.
      # - failure → emit a static "measurement failed" comment
      #   directly so the PR still gets a visible signal that
      #   the gate plumbing is broken (rather than silent CI red).
      #
      # W3.1 (2026-05-22): CPU-time axis of KeywordRetriever.retrieve
      # and RagPipeline.run is now BLOCKING via --gate-metric. Other
      # metrics (wall-clock, reranker, directory_corpus_load) stay
      # advisory — they appear in the comment table with the ⚠️ icon
      # but don't fail the job. Re-evaluate wall-clock promotion in
      # W4 based on observed σ. Measurement-failure branch stays
      # advisory (RC=0) — a script crash shouldn't block merge.
      - name: Format delta
        id: delta
        run: |
          set +e
          if [ "${{ steps.measure.outcome }}" = "failure" ]; then
            cat > "$COMMENT_PATH" <<'EOF'
          <!-- attune-rag-perf-gate -->
          ## Perf delta — measurement failed

          The per-PR perf measurement step crashed before producing
          data. See the workflow run log for details (top-level CI
          summary links the failing step).

          Measurement-failure stays non-blocking even under W3.1
          gating — the alternative would make every PR depend on the
          perf-script infrastructure being green. Fix the measurement
          step in a follow-up; real regressions in gated metrics will
          remain detectable once measurement runs again.

          <!-- attune-rag-perf-gate -->
          EOF
            RC=0
          else
            python scripts/format_perf_delta.py \
              --baseline "$BASELINE_JSON" \
              --current "$CURRENT_JSON" \
              --comment-out "$COMMENT_PATH" \
              --gate-metric keyword_retriever_retrieve.cpu \
              --gate-metric rag_pipeline_run.cpu
            RC=$?
          fi
          echo "rc=$RC" >> "$GITHUB_OUTPUT"
          # W3.1: propagate the script's exit code. RC=1 (regression
          # in a gated metric) fails the job and blocks merge under
          # branch protection. Measurement-failure branch above
          # explicitly sets RC=0 so it doesn't trip the gate.
          exit $RC
        shell: bash

      # ── Post or update the PR comment ─────────────────────────
      # W3.1: `always()` ensures the comment posts even when the
      # Format delta step exited non-zero (gated regression). The
      # devloop needs the comment to know WHY the gate failed; the
      # job's overall non-zero exit still blocks merge.
      - name: Post or update PR comment
        if: |
          always()
          && github.event_name == 'pull_request'
          && hashFiles(env.COMMENT_PATH) != ''
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PR_NUMBER: ${{ github.event.pull_request.number }}
          REPO: ${{ github.repository }}
        run: |
          set -euo pipefail
          EXISTING=$(gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" \
            --paginate \
            --jq '.[] | select(.body | contains("<!-- attune-rag-perf-gate -->")) | .id' \
            | head -1 || true)
          if [ -n "$EXISTING" ]; then
            echo "Updating existing comment id=$EXISTING"
            gh api -X PATCH "repos/${REPO}/issues/comments/${EXISTING}" \
              -F body=@"$COMMENT_PATH"
          else
            echo "Posting new comment on PR #$PR_NUMBER"
            gh pr comment "$PR_NUMBER" --body-file "$COMMENT_PATH"
          fi
        shell: bash

      - name: Summary
        if: always()
        env:
          DELTA_RC: ${{ steps.delta.outputs.rc }}
        run: |
          {
            echo "### attune-rag perf delta"
            echo ""
            echo "- format_perf_delta.py exit: ${DELTA_RC:-skipped}"
            echo "- Mode: W3.1 — CPU-time blocking on \`keyword_retriever_retrieve.cpu\` + \`rag_pipeline_run.cpu\`; other metrics advisory"
          } >> "$GITHUB_STEP_SUMMARY"
        shell: bash