refactor: store summaries (17KB) instead of full results (220KB) per model

- New: evaluations/summaries/ — scores only, no raw responses or
  per-permutation details. 17KB vs 220KB per model.
- evaluations/results/ added to .gitignore (full results reproducible
  via pilot.py)
- Report generator reads from summaries/
- pilot.py auto-generates summary after each run
- Fixed filename suffix to use exact model IDs (CodeRabbit feedback)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: raifdmueller

evaluations/.gitignore

@@ -1 +1,3 @@
 *.pyc
+# Full results with raw responses (large, reproducible via pilot.py)
+results/

evaluations/generate-report.py

@@ -19,7 +19,7 @@
 from collections import defaultdict
 from pathlib import Path
 
-RESULTS_DIR = Path(__file__).parent / "results"
+RESULTS_DIR = Path(__file__).parent / "summaries"
 SPECS_DIR = Path(__file__).parent / "specs"
 
 # Fallback display names (used when config doesn't have exact model ID)

evaluations/pilot.py

@@ -312,19 +312,15 @@ def run_pilot(models, dry_run=False, verbose=False, ollama_model="qwen3:4b", no_
     print()
 
     ts = datetime.now().strftime("%Y%m%d-%H%M%S")
-    # Include model names in filename to prevent race conditions on parallel runs
-    model_suffix = "_".join(models)
+    # Include exact model IDs in filename to prevent race conditions
+    model_ids = []
     for m in models:
-        if m == "openai":
-            model_suffix = model_suffix.replace("openai", openai_model)
-        elif m == "mistral":
-            model_suffix = model_suffix.replace("mistral", mistral_model)
-        elif m == "deepseek":
-            model_suffix = model_suffix.replace("deepseek", deepseek_model)
-        elif m == "ollama":
-            model_suffix = model_suffix.replace("ollama", f"ollama-{ollama_model}")
-    # Sanitize for filename
-    model_suffix = model_suffix.replace(":", "-").replace("/", "-")
+        if m == "openai": model_ids.append(openai_model)
+        elif m == "mistral": model_ids.append(mistral_model)
+        elif m == "deepseek": model_ids.append(deepseek_model)
+        elif m == "ollama": model_ids.append(f"ollama-{ollama_model}")
+        else: model_ids.append(m)
+    model_suffix = "_".join(model_ids).replace(":", "-").replace("/", "-")
     out_file = RESULTS_DIR / f"pilot-{ts}_{model_suffix}.json"
 
     all_results = {
@@ -462,6 +458,18 @@ def append_and_save(r):
         save_results(all_results, out_file)
         print(f"\nResults saved to {out_file}")
 
+        # Also save a stripped summary (scores only, no raw responses)
+        summary_dir = RESULTS_DIR.parent / "summaries"
+        summary_dir.mkdir(parents=True, exist_ok=True)
+        summary = json.loads(json.dumps(all_results))  # deep copy
+        for m_results in summary.get("models", {}).values():
+            for r in m_results:
+                r.pop("results", None)
+        summary_file = summary_dir / out_file.name
+        with open(summary_file, "w", encoding="utf-8") as fh:
+            json.dump(summary, fh, indent=2, ensure_ascii=False)
+        print(f"Summary saved to {summary_file}")
+
         # Summary
         print("\n=== SUMMARY ===")
         print(f"Models: {', '.join(models)}")