fix(overeng): BM25 divergence, dead env vars, memory_pipeline postpass

Author: nikolay-e

benchmarks/adapters/runner.py

@@ -213,15 +213,18 @@ def _run_parallel(  # noqa: C901 — pool-shape branching + multi-failure-mode d
     )
 
     def _drain(active_pool: ProcessPoolExecutor) -> None:
+        from concurrent.futures import CancelledError
         from concurrent.futures.process import BrokenProcessPool
 
         futures: dict = {}
         submit_failed: list[BenchmarkInstance] = []
+        pool_broken = False
         for inst in pending:
             try:
                 futures[active_pool.submit(eval_fn, inst, params)] = inst
             except BrokenProcessPool:
                 submit_failed.extend([inst, *pending[pending.index(inst) + 1 :]])
+                pool_broken = True
                 break
         outer_deadline = time.monotonic() + timeout_per_instance * max(1, (len(pending) + workers - 1) // workers)
         completed: set[str] = set()
@@ -230,10 +233,11 @@ def _drain(active_pool: ProcessPoolExecutor) -> None:
                 inst = futures[future]
                 try:
                     r = future.result(timeout=0)
-                except FuturesTimeoutError:
+                except (FuturesTimeoutError, CancelledError):
                     r = _failure_result(inst, params, "timeout", f"after {timeout_per_instance}s")
                 except BrokenProcessPool:
                     r = _failure_result(inst, params, "error", "BrokenProcessPool: worker died")
+                    pool_broken = True
                 except Exception as e:
                     r = _failure_result(inst, params, "error", f"{type(e).__name__}: {e}")
                 completed.add(inst.instance_id)
@@ -243,12 +247,13 @@ def _drain(active_pool: ProcessPoolExecutor) -> None:
                 if inst.instance_id not in completed:
                     record(_failure_result(inst, params, "timeout", "exceeded global deadline"))
         except BrokenProcessPool:
+            pool_broken = True
             for inst in futures.values():
                 if inst.instance_id not in completed:
                     record(_failure_result(inst, params, "error", "BrokenProcessPool: worker died"))
         for inst in submit_failed:
             record(_failure_result(inst, params, "error", "BrokenProcessPool: submit failed"))
-        if submit_failed or any(inst.instance_id not in completed for inst in futures.values()):
+        if pool_broken:
             raise BrokenProcessPool("pool degraded mid-cell")
 
     if pool is not None:

diffctx/Cargo.toml

@@ -11,7 +11,7 @@ crate-type = ["cdylib", "rlib"]
 name = "diffctx"
 path = "src/main.rs"
 
-[[bin]]
+[[example]]
 name = "diffctx-test"
 path = "src/test_harness.rs"
 

diffctx/src/config/bm25.rs

@@ -3,7 +3,6 @@ use once_cell::sync::Lazy;
 pub struct Bm25Config {
     pub k1: f64,
     pub b: f64,
-    pub doc_len_factor: f64,
     pub idf_smoothing: f64,
     pub min_query_token_length: usize,
 }
@@ -13,7 +12,6 @@ impl Default for Bm25Config {
         Self {
             k1: 2.5,
             b: 0.75,
-            doc_len_factor: 1.5,
             idf_smoothing: 0.5,
             min_query_token_length: 3,
         }

diffctx/src/config/category_weights.rs

@@ -8,24 +8,11 @@
 //! (Bayesian opt / grid search) against a labeled corpus, as described in
 //! paper Section 4.3 (Edge-Type Weight Calibration).
 //!
-//! Override at runtime via environment variables:
-//!   DIFFCTX_CATWEIGHT_SEMANTIC=0.65
-//!   DIFFCTX_CATWEIGHT_STRUCTURAL=0.50
-//!   DIFFCTX_CATWEIGHT_SIBLING=0.10
-//!   DIFFCTX_CATWEIGHT_CONFIG=0.40
-//!   DIFFCTX_CATWEIGHT_CONFIGGENERIC=0.30
-//!   DIFFCTX_CATWEIGHT_DOCUMENT=0.30
-//!   DIFFCTX_CATWEIGHT_SIMILARITY=0.20
-//!   DIFFCTX_CATWEIGHT_HISTORY=0.40
-//!   DIFFCTX_CATWEIGHT_TESTEDGE=0.60
-//!   DIFFCTX_CATWEIGHT_GENERIC=0.10
-//!
 //! Default for every variant is 1.0 (no scaling — the fine-grained
 //! prior weights from `weights.rs` apply unchanged).
 
 use once_cell::sync::Lazy;
 
-use crate::config::env_overrides::read_env_f64;
 use crate::graph::EdgeCategory;
 
 #[derive(Debug, Clone, Copy)]
@@ -77,18 +64,7 @@ impl CategoryWeights {
     }
 }
 
-pub static CATEGORY_WEIGHTS: Lazy<CategoryWeights> = Lazy::new(|| CategoryWeights {
-    semantic: read_env_f64("DIFFCTX_CATWEIGHT_SEMANTIC", 1.0),
-    structural: read_env_f64("DIFFCTX_CATWEIGHT_STRUCTURAL", 1.0),
-    sibling: read_env_f64("DIFFCTX_CATWEIGHT_SIBLING", 1.0),
-    config: read_env_f64("DIFFCTX_CATWEIGHT_CONFIG", 1.0),
-    config_generic: read_env_f64("DIFFCTX_CATWEIGHT_CONFIGGENERIC", 1.0),
-    document: read_env_f64("DIFFCTX_CATWEIGHT_DOCUMENT", 1.0),
-    similarity: read_env_f64("DIFFCTX_CATWEIGHT_SIMILARITY", 1.0),
-    history: read_env_f64("DIFFCTX_CATWEIGHT_HISTORY", 1.0),
-    test_edge: read_env_f64("DIFFCTX_CATWEIGHT_TESTEDGE", 1.0),
-    generic: read_env_f64("DIFFCTX_CATWEIGHT_GENERIC", 1.0),
-});
+pub static CATEGORY_WEIGHTS: Lazy<CategoryWeights> = Lazy::new(CategoryWeights::default);
 
 #[cfg(test)]
 mod tests {

diffctx/src/config/limits.rs

@@ -85,22 +85,18 @@ impl Default for LexicalConfig {
 }
 
 pub struct CochangeConfig {
-    pub weight: f64,
     pub min_count: usize,
     pub max_files_per_commit: usize,
     pub commits_limit: usize,
-    pub timeout_seconds: u64,
     pub log_scale_factor: f64,
 }
 
 impl Default for CochangeConfig {
     fn default() -> Self {
         Self {
-            weight: 0.40,
             min_count: 2,
             max_files_per_commit: 30,
             commits_limit: 500,
-            timeout_seconds: 10,
             log_scale_factor: 0.1,
         }
     }

diffctx/src/config/scoring.rs

@@ -1,24 +1,17 @@
 use once_cell::sync::Lazy;
 
-use crate::config::env_overrides::read_env_f64;
-
 pub struct EgoScoringConfig {
     pub identifier_overlap_epsilon: f64,
     pub identifier_overlap_cap: usize,
-    pub per_hop_decay: f64,
 }
 
 impl Default for EgoScoringConfig {
     fn default() -> Self {
         Self {
             identifier_overlap_epsilon: 0.1,
             identifier_overlap_cap: 10,
-            per_hop_decay: 1.0,
         }
     }
 }
 
-pub static EGO: Lazy<EgoScoringConfig> = Lazy::new(|| EgoScoringConfig {
-    per_hop_decay: read_env_f64("DIFFCTX_OP_EGO_PER_HOP_DECAY", 1.0),
-    ..EgoScoringConfig::default()
-});
+pub static EGO: Lazy<EgoScoringConfig> = Lazy::new(EgoScoringConfig::default);

diffctx/src/config/selection.rs

@@ -4,15 +4,13 @@ use crate::config::env_overrides::{read_env_fraction, read_env_u32};
 pub struct SelectionConfig {
     pub core_budget_fraction: f64,
     pub r_cap_min: f64,
-    pub stopping_threshold: f64,
 }
 
 impl Default for SelectionConfig {
     fn default() -> Self {
         Self {
             core_budget_fraction: 0.70,
             r_cap_min: 0.01,
-            stopping_threshold: 0.08,
         }
     }
 }
@@ -55,7 +53,6 @@ pub fn selection() -> SelectionConfig {
     SelectionConfig {
         core_budget_fraction: read_env_fraction("DIFFCTX_OP_SELECTION_CORE_BUDGET_FRACTION", 0.70),
         r_cap_min: read_env_fraction("DIFFCTX_OP_SELECTION_R_CAP_MIN", 0.01),
-        stopping_threshold: read_env_fraction("DIFFCTX_OP_SELECTION_STOPPING_THRESHOLD", 0.08),
     }
 }
 

diffctx/src/discovery.rs

@@ -172,7 +172,7 @@ impl BM25Discovery {
             }
             let idf_val = idf.get(t).copied().unwrap_or(0.0);
             s += idf_val * (freq * BM25.k1)
-                / (freq + BM25.doc_len_factor * (1.0 - BM25.b + BM25.b * dl / avgdl));
+                / (freq + BM25.k1 * (1.0 - BM25.b + BM25.b * dl / avgdl));
         }
         s
     }

diffctx/src/memory_pipeline.rs

@@ -135,8 +135,38 @@ pub fn build_diff_context_in_memory(
         None,
     );
 
+    let mut selected = selection.selected;
+
+    crate::postpass::coherence_post_pass(
+        &mut selected,
+        &scoring_result.filtered_fragments,
+        &scoring_result.graph,
+        effective_budget,
+    );
+
+    crate::postpass::rescue_nontrivial_context(
+        &mut selected,
+        &all_fragments,
+        &scoring_result.rel_scores,
+        &core_ids,
+        effective_budget,
+    );
+
+    let used: u32 = selected.iter().map(|f| f.token_count).sum();
+    let remaining = effective_budget.saturating_sub(used);
+    let changed_files: Vec<PathBuf> = changed_paths.iter().map(PathBuf::from).collect();
+    crate::postpass::ensure_changed_files_represented(
+        &mut selected,
+        &all_fragments,
+        &changed_files,
+        remaining,
+        Path::new("."),
+        &[],
+        None,
+    );
+
     let dummy_root = Path::new(".");
-    build_diff_context_output(dummy_root, &selection.selected, no_content)
+    build_diff_context_output(dummy_root, &selected, no_content)
 }
 
 fn compute_memory_hunks(

diffctx/src/pybridge.rs

@@ -183,51 +183,6 @@ impl FragmentIterator {
     }
 }
 
-#[pyfunction]
-#[pyo3(signature = (
-    root_dir,
-    diff_range = None,
-    budget_tokens = None,
-    alpha = DEFAULT_PPR_ALPHA,
-    tau = DEFAULT_STOPPING_THRESHOLD,
-    no_content = false,
-    full = false,
-    scoring_mode = "hybrid",
-    timeout = DEFAULT_PIPELINE_TIMEOUT_SECONDS,
-))]
-fn build_diff_context_native(
-    py: Python<'_>,
-    root_dir: &str,
-    diff_range: Option<&str>,
-    budget_tokens: Option<u32>,
-    alpha: f64,
-    tau: f64,
-    no_content: bool,
-    full: bool,
-    scoring_mode: &str,
-    timeout: u64,
-) -> PyResult<DiffContextResult> {
-    let mode =
-        ScoringMode::from_str(scoring_mode).map_err(pyo3::exceptions::PyValueError::new_err)?;
-    let path = Path::new(root_dir);
-
-    py.allow_threads(|| {
-        pipeline::build_diff_context(
-            path,
-            diff_range,
-            budget_tokens,
-            alpha,
-            tau,
-            no_content,
-            full,
-            mode,
-            timeout,
-        )
-    })
-    .map(DiffContextResult::from)
-    .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))
-}
-
 #[pyfunction]
 #[pyo3(signature = (
     root_dir,
@@ -598,7 +553,6 @@ fn graph_summary<'py>(
 #[pymodule]
 pub fn _diffctx(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(build_diff_context, m)?)?;
-    m.add_function(wrap_pyfunction!(build_diff_context_native, m)?)?;
     m.add_function(wrap_pyfunction!(get_language_for_file, m)?)?;
     m.add_function(wrap_pyfunction!(count_tokens, m)?)?;
     m.add_function(wrap_pyfunction!(build_project_graph, m)?)?;