chore(calib): set calibrated defaults (tau=0.12, cbf=0.5)

Winner of the v1 calibration grid (2119 instances, four benchmarks,
pebble-fixed pool): (tau, cbf) = (0.12, 0.5) at min(per_benchmark
file_recall) = 0.1092. Surface is flat (top-3 within 0.001), so the
choice is robust rather than tuned to a sharp optimum. Per-benchmark
breakdown at winner: swebench_lite 0.133, polybench 0.109,
multi_swebench 0.173, contextbench 0.127 -- PolyBench is the
bottleneck. The 0.1092 floor is a baseline for downstream multi-hop
work, NOT a final test-set headline.

Updates:
- diffctx Rust defaults (SelectionConfig.core_budget_fraction,
  DEFAULT_STOPPING_THRESHOLD)
- benchmarks Python defaults (RunParams.tau, RunParams.core_budget_fraction;
  run_eval.py CLI flags)
- paper v2: calibration grid table now reports 4x3 grid with new
  numbers; narrative reframed (was "0.7287, parameters tightly
  clustered" - that was a metric-mismatch artefact from the buggy
  pool that fake-timed-out non-Python benchmarks; new narrative
  matches the rebuilt pebble-pool data)

Author: nikolay-e

benchmarks/adapters/runner.py

@@ -19,8 +19,11 @@ class RunParams:
     near-zero effect). Anything else can be threaded through `extra_env`.
     """
 
-    tau: float = 0.08
-    core_budget_fraction: float = 0.70
+    # Calibrated v1 (2119 instances, 4 benchmarks, pebble-fixed pool):
+    # winner (tau, cbf) = (0.12, 0.5) at min(per_benchmark file_recall)
+    # = 0.1092. Surface is flat (top-3 within 0.001) — robust default.
+    tau: float = 0.12
+    core_budget_fraction: float = 0.5
     budget: int = 8000
     scoring: str = "hybrid"
     extra_env: dict[str, str] = field(default_factory=dict)

benchmarks/run_eval.py

@@ -4,7 +4,7 @@
 
     python -m benchmarks.run_eval \\
         --manifest benchmarks/manifests/v1/calibration.txt \\
-        --tau 0.08 --core-budget-fraction 0.70 --budget 8000 \\
+        --tau 0.12 --core-budget-fraction 0.5 --budget 8000 \\
         --workers 7 \\
         --out results/eval_calibration_t008_c070.json
 """
@@ -31,8 +31,8 @@ def _all_default_adapters() -> tuple[BenchmarkAdapter, ...]:
 def main() -> int:
     p = argparse.ArgumentParser(description=__doc__)
     p.add_argument("--manifest", type=Path, required=True)
-    p.add_argument("--tau", type=float, default=0.08)
-    p.add_argument("--core-budget-fraction", type=float, default=0.70)
+    p.add_argument("--tau", type=float, default=0.12)
+    p.add_argument("--core-budget-fraction", type=float, default=0.5)
     p.add_argument("--budget", type=int, default=8000)
     p.add_argument("--scoring", default="hybrid")
     p.add_argument("--workers", type=int, default=1)

diffctx/src/config/limits.rs

@@ -32,7 +32,10 @@ impl Default for AlgorithmLimits {
 }
 
 pub const DEFAULT_PPR_ALPHA: f64 = 0.60;
-pub const DEFAULT_STOPPING_THRESHOLD: f64 = 0.08;
+/// Calibrated on v1 manifest (2119 instances, 4 benchmarks). Winner
+/// of (tau, cbf) grid = (0.12, 0.5) at min(per_benchmark file_recall)
+/// = 0.1092. Top-3 within 0.001 — surface is flat, choice is robust.
+pub const DEFAULT_STOPPING_THRESHOLD: f64 = 0.12;
 pub const DEFAULT_PIPELINE_TIMEOUT_SECONDS: u64 = 300;
 pub const DEFAULT_BUDGET_TOKENS: u32 = 4096;
 

diffctx/src/config/selection.rs

@@ -9,7 +9,13 @@ pub struct SelectionConfig {
 impl Default for SelectionConfig {
     fn default() -> Self {
         Self {
-            core_budget_fraction: 0.70,
+            // Calibrated on the v1 manifest (2119 instances across
+            // SWE-bench Lite, PolyBench, Multi-SWE-bench, ContextBench)
+            // with the pebble-fixed pool. Winner of the (tau, cbf) grid
+            // = (0.12, 0.5) at min(per_benchmark file_recall) = 0.1092.
+            // Surface is flat (top-3 within 0.001), so the choice is
+            // robust rather than tuned to a sharp optimum.
+            core_budget_fraction: 0.5,
             r_cap_min: 0.01,
         }
     }
@@ -51,7 +57,7 @@ impl Default for BoltzmannConfig {
 
 pub fn selection() -> SelectionConfig {
     SelectionConfig {
-        core_budget_fraction: read_env_fraction("DIFFCTX_OP_SELECTION_CORE_BUDGET_FRACTION", 0.70),
+        core_budget_fraction: read_env_fraction("DIFFCTX_OP_SELECTION_CORE_BUDGET_FRACTION", 0.5),
         r_cap_min: read_env_fraction("DIFFCTX_OP_SELECTION_R_CAP_MIN", 0.01),
     }
 }

docs/Context-Selection-for-Git-Diff/v2/main.tex

@@ -484,19 +484,20 @@ \subsection{Setup}
 \end{itemize}
 Cross-benchmark contamination is filtered by \texttt{(repo, base\_commit)} key (1449 instances dropped from the calibration pool). HuggingFace dataset revisions are pinned by SHA in \texttt{benchmarks/dataset\_revisions.json}.
 
-\paragraph{Calibration.} A 3$\times$3 grid over $\tau \in \{0.04, 0.08, 0.12\}$ and $\beta_{\mathrm{core}} \in \{0.5, 0.7, 0.9\}$ was run on the calibration manifest. The selection objective was the minimum file-recall across the four calibration source benchmarks (Section~\ref{sec:eval}). The full grid is reported in Table~\ref{tab:calibration-grid}. The selected cell is $\tau{=}0.12$, $\beta_{\mathrm{core}}{=}0.5$, with $B{=}8000$ tokens, $\mathrm{scoring}{=}\mathrm{hybrid}$. Two observations are worth recording: first, the winning $\tau$ row ($\tau{=}0.12$) clusters tightly across $\beta_{\mathrm{core}}$ values (0.7287--0.7298, a span of 0.001), indicating that $\beta_{\mathrm{core}}$ is essentially insensitive in this regime; second, the calibration objective varies by only $\sim$0.02 across the full grid. The remaining operational parameters (Section~\ref{sec:greedy} and the appendix) are held fixed at the values listed there.
+\paragraph{Calibration.} A 4$\times$3 grid over $\tau \in \{0.04, 0.08, 0.12, 0.16\}$ and $\beta_{\mathrm{core}} \in \{0.5, 0.7, 0.9\}$ was run on the full v1 calibration manifest (2{,}119 instances spanning SWE-bench Lite, PolyBench, Multi-SWE-bench, ContextBench). The selection objective is the minimum file-recall across the four calibration source benchmarks (Section~\ref{sec:eval}); minimum rather than mean prevents domination by the largest-population benchmark. The full grid is reported in Table~\ref{tab:calibration-grid}. The selected cell is $\tau{=}0.12$, $\beta_{\mathrm{core}}{=}0.5$, with $B{=}8000$ tokens, $\mathrm{scoring}{=}\mathrm{hybrid}$. Three observations: first, the surface is nearly flat---the top-3 cells lie within $0.001$ of each other (Table~\ref{tab:calibration-grid}), indicating that $(\tau, \beta_{\mathrm{core}})$ are robust rather than tuned to a sharp optimum; second, the absolute score $0.1092$ is the minimum mean recall across benchmarks on a difficult cross-language calibration mix---this is a baseline number for downstream multi-hop work, not a final test-set headline; third, the per-benchmark breakdown at the winner is $\{$swebench\_lite: 0.133, polybench: 0.109, multi\_swebench: 0.173, contextbench: 0.127$\}$, showing the bottleneck benchmark is PolyBench. The remaining operational parameters (Section~\ref{sec:greedy} and the appendix) are held fixed at the values listed there.
 
 \begin{table}[h]
 \centering
-\caption{Calibration grid scores: minimum per-source-benchmark file recall on the calibration manifest. The selected cell is highlighted; $\tau{=}0.12$ dominates across $\beta_{\mathrm{core}}$. The grid is reported in full to make the parameter-sensitivity argument concrete.}
+\caption{Calibration grid scores: minimum per-source-benchmark file recall on the v1 calibration manifest (2{,}119 instances, four benchmarks, hybrid scoring, $B{=}8000$). The selected cell is highlighted. Surface is flat (top-3 within $0.001$); $\tau$ varies modestly across rows, $\beta_{\mathrm{core}}$ is essentially insensitive within the winning row.}
 \label{tab:calibration-grid}
 \begin{tabular}{lccc}
 \toprule
 $\tau \; \backslash \; \beta_{\mathrm{core}}$ & $0.5$ & $0.7$ & $0.9$ \\
 \midrule
-$0.04$ & $0.7196$ & $0.7196$ & $0.7196$ \\
-$0.08$ & $0.7113$ & $0.7113$ & $0.7113$ \\
-$0.12$ & $\mathbf{0.7287^\ast}$ & $0.7287$ & $0.7298$ \\
+$0.04$ & $0.1081$ & $0.1068$ & $0.1053$ \\
+$0.08$ & $0.1091$ & $0.1081$ & $0.1050$ \\
+$0.12$ & $\mathbf{0.1092^\ast}$ & $0.1077$ & $0.1050$ \\
+$0.16$ & $0.1036$ & $0.1029$ & $0.1011$ \\
 \bottomrule
 \end{tabular}
 \end{table}