ci: add RAG P@1 regression gate (overall >=73%)

- benchmark_all_fixtures.py: --overall-gate flag exits 1 when overall
  P@1 drops below threshold (default disabled, not a breaking change).
- .github/workflows/rag-benchmark.yml: runs on changes to summaries,
  fixtures, or the benchmark script. Gate is 73% (current baseline
  73.1%). Skips unrelated commits to keep CI fast.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: silversurfer562

.github/workflows/rag-benchmark.yml

@@ -0,0 +1,49 @@
+name: RAG Benchmark
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "src/attune_help/templates/summaries_by_path.json"
+      - "src/attune_help/templates/fixtures/**"
+      - "scripts/benchmark_all_fixtures.py"
+  pull_request:
+    paths:
+      - "src/attune_help/templates/summaries_by_path.json"
+      - "src/attune_help/templates/fixtures/**"
+      - "scripts/benchmark_all_fixtures.py"
+  workflow_dispatch:
+    inputs:
+      overall_gate:
+        description: "Overall P@1 gate (0.0–1.0)"
+        default: "0.73"
+
+permissions:
+  contents: read
+
+jobs:
+  benchmark:
+    name: RAG P@1 gate
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Set up Python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.11"
+          cache: pip
+
+      - name: Install attune-help + rag extra
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[dev]"
+          python -m pip install "attune-rag>=0.1.0,<0.2"
+
+      - name: Run RAG benchmark
+        run: |
+          GATE=${{ github.event.inputs.overall_gate || '0.73' }}
+          python scripts/benchmark_all_fixtures.py \
+            --gate 0.0 \
+            --overall-gate "$GATE"

scripts/benchmark_all_fixtures.py

@@ -41,7 +41,10 @@ def _load_fixtures() -> list[dict]:
 
 def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser()
-    parser.add_argument("--gate", type=float, default=0.60)
+    parser.add_argument("--gate", type=float, default=0.60,
+                        help="Per-feature P@1 gate (default: 0.60)")
+    parser.add_argument("--overall-gate", type=float, default=0.0,
+                        help="Overall P@1 gate — exit 1 if overall drops below (default: disabled)")
     parser.add_argument(
         "--summaries",
         default="summaries_by_path.json",
@@ -124,6 +127,17 @@ def main(argv: list[str] | None = None) -> int:
 
     print(f"\nFeatures below {args.gate:.0%} P@1 gate: " f"{below_gate}/{len(rows)}")
 
+    overall_p1 = total_top1 / total_queries if total_queries else 0.0
+    if args.overall_gate > 0:
+        if overall_p1 < args.overall_gate:
+            print(
+                f"\n✖ FAIL — overall P@1 {overall_p1:.1%} is below "
+                f"--overall-gate {args.overall_gate:.0%}",
+                file=sys.stderr,
+            )
+            return 1
+        print(f"\n✔ PASS — overall P@1 {overall_p1:.1%} >= {args.overall_gate:.0%} gate")
+
     return 1 if below_gate > 0 else 0