refactor(diffctx): remove artificial caps, tune edge weights

Remove max_discovered_files (200), max_expansion_files (50), and
max_candidate_files (5000) caps — PPR + tau are now the sole quality
filters. Lower all same-package/module/namespace edge weights to 0.05
and reduce lexical similarity weights across all languages to prevent
over-discovery noise. Add threshold sanity checks and error handling
to benchmark scripts.

YAML tests: 1408 passed (+8), 29 xfailed (-8), avg score 84.0% (+0.8pp)

Author: nikolay-e

benchmarks/contextbench_diffctx.py

@@ -143,7 +143,7 @@ def run_diffctx(repo_dir: Path, budget: int = 8000) -> dict | None:
         ],
         capture_output=True,
         text=True,
-        timeout=120,
+        timeout=600,
     )
     if r.returncode != 0:
         print(f"  DIFFCTX FAIL (rc={r.returncode}): {r.stderr[:300]}")
@@ -351,7 +351,22 @@ def aggregate(results: list[dict]) -> None:
         print(f"\nFailures: {dict(by_status)}")
 
 
+def _print_threshold_sanity_check():
+    v = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            "from treemapper.diffctx.filtering import _LOW_RELEVANCE_THRESHOLD; print(_LOW_RELEVANCE_THRESHOLD)",
+        ],
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+    print(f"diffctx _LOW_RELEVANCE_THRESHOLD = {v}", file=sys.stderr)
+
+
 def main():
+    _print_threshold_sanity_check()
+
     import argparse
 
     parser = argparse.ArgumentParser()
@@ -389,9 +404,12 @@ def main():
 
     results = []
     for inst in instances:
-        r = evaluate_instance(inst, args.budget)
-        if r:
-            results.append(r)
+        try:
+            r = evaluate_instance(inst, args.budget)
+            if r:
+                results.append(r)
+        except Exception as e:
+            print(f"  ERROR: {type(e).__name__}: {e}")
 
     aggregate(results)
 

benchmarks/forensic_contextbench.py

@@ -363,7 +363,22 @@ def evaluate_one(inst: dict, budget: int) -> dict:
     }
 
 
+def _print_threshold_sanity_check():
+    v = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            "from treemapper.diffctx.filtering import _LOW_RELEVANCE_THRESHOLD; print(_LOW_RELEVANCE_THRESHOLD)",
+        ],
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+    print(f"diffctx _LOW_RELEVANCE_THRESHOLD = {v}", file=sys.stderr)
+
+
 def main():
+    _print_threshold_sanity_check()
+
     ap = argparse.ArgumentParser()
     ap.add_argument("--limit", type=int, default=5)
     ap.add_argument("--budget", type=int, default=8000)

src/treemapper/diffctx/config/limits.py

@@ -20,11 +20,8 @@ class AlgorithmLimits:
     max_fragments: int = field(default_factory=lambda: _env_int("TREEMAPPER_MAX_FRAGMENTS", 200))
     max_generated_fragments: int = 5
     max_generated_lines: int = 30
-    max_candidate_files: int = 5000
-    max_discovered_files: int = field(default_factory=lambda: _env_int("TREEMAPPER_MAX_DISCOVERED", 200))
     skip_expensive_threshold: int = 2000
     rare_identifier_threshold: int = 3
-    max_expansion_files: int = field(default_factory=lambda: _env_int("TREEMAPPER_MAX_EXPANSION", 50))
     overhead_per_fragment: int = 18
 
 

src/treemapper/diffctx/config/weights.py

@@ -30,43 +30,43 @@ class EdgeWeightConfig:
     "go_import": EdgeWeightConfig(0.70, 0.40),
     "go_type": EdgeWeightConfig(0.65, 0.40),
     "go_func": EdgeWeightConfig(0.60, 0.40),
-    "go_same_package": EdgeWeightConfig(0.55, 0.40),
+    "go_same_package": EdgeWeightConfig(0.05, 0.40),
     "rust_mod": EdgeWeightConfig(0.70, 0.40),
     "rust_use": EdgeWeightConfig(0.65, 0.40),
     "rust_type": EdgeWeightConfig(0.65, 0.40),
     "rust_fn": EdgeWeightConfig(0.60, 0.40),
-    "rust_same_crate": EdgeWeightConfig(0.50, 0.40),
+    "rust_same_crate": EdgeWeightConfig(0.05, 0.40),
     "jvm_import": EdgeWeightConfig(0.75, 0.40),
     "jvm_inheritance": EdgeWeightConfig(0.80, 0.40),
     "jvm_type": EdgeWeightConfig(0.60, 0.40),
-    "jvm_same_package": EdgeWeightConfig(0.10, 0.40),
+    "jvm_same_package": EdgeWeightConfig(0.05, 0.40),
     "jvm_annotation": EdgeWeightConfig(0.50, 0.40),
     "c_include": EdgeWeightConfig(0.65, 0.40),
     "c_call": EdgeWeightConfig(0.55, 0.40),
     "c_type": EdgeWeightConfig(0.50, 0.40),
     "dotnet_using": EdgeWeightConfig(0.65, 0.40),
     "dotnet_inheritance": EdgeWeightConfig(0.75, 0.40),
     "dotnet_type": EdgeWeightConfig(0.60, 0.40),
-    "dotnet_same_namespace": EdgeWeightConfig(0.50, 0.40),
+    "dotnet_same_namespace": EdgeWeightConfig(0.05, 0.40),
     "dotnet_attribute": EdgeWeightConfig(0.50, 0.40),
     "dotnet_partial": EdgeWeightConfig(0.80, 0.40),
     "ruby_require": EdgeWeightConfig(0.65, 0.40),
     "ruby_include": EdgeWeightConfig(0.60, 0.40),
     "ruby_const": EdgeWeightConfig(0.55, 0.40),
-    "ruby_same_dir": EdgeWeightConfig(0.45, 0.40),
+    "ruby_same_dir": EdgeWeightConfig(0.05, 0.40),
     "php_use": EdgeWeightConfig(0.65, 0.40),
     "php_require": EdgeWeightConfig(0.60, 0.40),
     "php_inheritance": EdgeWeightConfig(0.75, 0.40),
     "php_type": EdgeWeightConfig(0.55, 0.40),
-    "php_same_namespace": EdgeWeightConfig(0.10, 0.40),
+    "php_same_namespace": EdgeWeightConfig(0.05, 0.40),
     "shell_source": EdgeWeightConfig(0.60, 0.35),
     "shell_script": EdgeWeightConfig(0.50, 0.35),
     "swift_import": EdgeWeightConfig(0.65, 0.40),
     "swift_conformance": EdgeWeightConfig(0.70, 0.40),
     "swift_extension": EdgeWeightConfig(0.65, 0.40),
     "swift_type": EdgeWeightConfig(0.60, 0.40),
     "swift_func": EdgeWeightConfig(0.55, 0.40),
-    "swift_same_module": EdgeWeightConfig(0.15, 0.40),
+    "swift_same_module": EdgeWeightConfig(0.05, 0.40),
     "zig_import": EdgeWeightConfig(0.65, 0.40),
     "zig_type": EdgeWeightConfig(0.60, 0.40),
     "zig_fn": EdgeWeightConfig(0.55, 0.40),
@@ -156,24 +156,24 @@ class LangWeights:
 
 
 LANG_WEIGHTS: dict[str, LangWeights] = {
-    "python": LangWeights(0.65, 0.70, 0.50, 0.20, 0.35),
-    "javascript": LangWeights(0.50, 0.55, 0.45, 0.25, 0.35),
-    "jsx": LangWeights(0.50, 0.55, 0.45, 0.25, 0.35),
-    "typescript": LangWeights(0.70, 0.75, 0.65, 0.15, 0.25),
-    "tsx": LangWeights(0.70, 0.75, 0.65, 0.15, 0.25),
-    "rust": LangWeights(0.90, 0.95, 0.85, 0.10, 0.15),
-    "java": LangWeights(0.85, 0.90, 0.80, 0.10, 0.15),
-    "kotlin": LangWeights(0.80, 0.85, 0.75, 0.12, 0.18),
-    "scala": LangWeights(0.80, 0.85, 0.75, 0.12, 0.18),
-    "go": LangWeights(0.80, 0.85, 0.75, 0.12, 0.20),
-    "c": LangWeights(0.60, 0.65, 0.55, 0.15, 0.25),
-    "cpp": LangWeights(0.65, 0.70, 0.60, 0.15, 0.25),
-    "csharp": LangWeights(0.75, 0.80, 0.70, 0.12, 0.20),
-    "fsharp": LangWeights(0.70, 0.75, 0.65, 0.12, 0.20),
-    "ruby": LangWeights(0.60, 0.65, 0.55, 0.15, 0.25),
-    "php": LangWeights(0.60, 0.65, 0.55, 0.15, 0.25),
-    "shell": LangWeights(0.40, 0.45, 0.35, 0.20, 0.30),
-    "swift": LangWeights(0.75, 0.80, 0.70, 0.12, 0.20),
+    "python": LangWeights(0.65, 0.70, 0.50, 0.10, 0.20),
+    "javascript": LangWeights(0.50, 0.55, 0.45, 0.10, 0.20),
+    "jsx": LangWeights(0.50, 0.55, 0.45, 0.10, 0.20),
+    "typescript": LangWeights(0.70, 0.75, 0.65, 0.10, 0.18),
+    "tsx": LangWeights(0.70, 0.75, 0.65, 0.10, 0.18),
+    "rust": LangWeights(0.90, 0.95, 0.85, 0.05, 0.10),
+    "java": LangWeights(0.85, 0.90, 0.80, 0.05, 0.10),
+    "kotlin": LangWeights(0.80, 0.85, 0.75, 0.05, 0.12),
+    "scala": LangWeights(0.80, 0.85, 0.75, 0.05, 0.12),
+    "go": LangWeights(0.80, 0.85, 0.75, 0.05, 0.12),
+    "c": LangWeights(0.60, 0.65, 0.55, 0.08, 0.15),
+    "cpp": LangWeights(0.65, 0.70, 0.60, 0.08, 0.15),
+    "csharp": LangWeights(0.75, 0.80, 0.70, 0.05, 0.12),
+    "fsharp": LangWeights(0.70, 0.75, 0.65, 0.05, 0.12),
+    "ruby": LangWeights(0.60, 0.65, 0.55, 0.08, 0.15),
+    "php": LangWeights(0.60, 0.65, 0.55, 0.08, 0.15),
+    "shell": LangWeights(0.40, 0.45, 0.35, 0.10, 0.18),
+    "swift": LangWeights(0.75, 0.80, 0.70, 0.05, 0.12),
 }
 
-DEFAULT_LANG_WEIGHTS = LangWeights(0.55, 0.60, 0.50, 0.15, 0.25)
+DEFAULT_LANG_WEIGHTS = LangWeights(0.55, 0.60, 0.50, 0.08, 0.15)

src/treemapper/diffctx/pipeline.py

@@ -45,7 +45,6 @@
 logger = logging.getLogger(__name__)
 
 _OVERHEAD_PER_FRAGMENT = LIMITS.overhead_per_fragment
-_MAX_DISCOVERED_FILES = LIMITS.max_discovered_files
 _UNLIMITED_BUDGET = 10_000_000
 
 
@@ -253,19 +252,12 @@ def build_diff_context(
         seen_frag_ids: set[FragmentId] = set()
         all_fragments = _process_files_for_fragments(changed_files, root_dir, preferred_revs, seen_frag_ids, batch_reader)
 
-        all_candidate_files, _ = _collect_candidate_files(root_dir, set(changed_files), combined_spec)
+        all_candidate_files = _collect_candidate_files(root_dir, set(changed_files), combined_spec)
         all_candidate_files = _filter_whitelist(all_candidate_files, root_dir, wl_spec)
 
         t1 = time.perf_counter()
 
         edge_discovered = discover_all_related_files(changed_files, all_candidate_files, root_dir)
-        if len(edge_discovered) > _MAX_DISCOVERED_FILES:
-            logger.debug(
-                "diffctx: capping edge-discovered files from %d to %d",
-                len(edge_discovered),
-                _MAX_DISCOVERED_FILES,
-            )
-            edge_discovered = edge_discovered[:_MAX_DISCOVERED_FILES]
         edge_discovered = [_normalize_path(p, root_dir) for p in edge_discovered]
         all_fragments.extend(_process_files_for_fragments(edge_discovered, root_dir, preferred_revs, seen_frag_ids, batch_reader))
 
@@ -277,7 +269,6 @@ def build_diff_context(
             changed_files + edge_discovered,
             combined_spec,
             candidate_files=all_candidate_files,
-            changed_files=changed_files,
         )
         expanded_files = [_normalize_path(p, root_dir) for p in expanded_files]
         all_fragments.extend(_process_files_for_fragments(expanded_files, root_dir, preferred_revs, seen_frag_ids, batch_reader))

src/treemapper/diffctx/project_graph.py

@@ -149,7 +149,7 @@ def build_project_graph(
     combined_spec = get_ignore_specs(root_dir, ignore_file, no_default_ignores, None)
     wl_spec = get_whitelist_spec(whitelist_file, root_dir)
 
-    all_candidate_files, _ = _collect_candidate_files(root_dir, set(), combined_spec)
+    all_candidate_files = _collect_candidate_files(root_dir, set(), combined_spec)
     all_candidate_files = _filter_whitelist(all_candidate_files, root_dir, wl_spec)
 
     logger.info("project_graph: found %d candidate files", len(all_candidate_files))

src/treemapper/diffctx/universe.py

@@ -19,11 +19,8 @@
 from . import git as _git  # noqa: E402
 
 _RARE_THRESHOLD = LIMITS.rare_identifier_threshold
-_MAX_EXPANSION_FILES = LIMITS.max_expansion_files
 _FALLBACK_MAX_FILES = 10_000
-_MAX_DISCOVERED_FILES = LIMITS.max_discovered_files
 _MAX_FILE_SIZE = LIMITS.max_file_size
-_MAX_CANDIDATE_FILES = LIMITS.max_candidate_files
 _MIN_CONCEPT_LENGTH = 4
 
 _BUILD_SYSTEM_NAMES = frozenset(
@@ -106,36 +103,7 @@ def _is_candidate_file(file_path: Path, root_dir: Path, included_set: set[Path],
     return True
 
 
-def _prioritize_candidates(
-    candidates: list[Path],
-    changed_files: set[Path],
-) -> list[Path]:
-    changed_dirs: set[Path] = set()
-    changed_extensions: set[str] = set()
-    for f in changed_files:
-        changed_dirs.add(f.parent)
-        if f.parent.parent != f.parent:
-            changed_dirs.add(f.parent.parent)
-        if f.suffix:
-            changed_extensions.add(f.suffix.lower())
-
-    priority: list[Path] = []
-    rest: list[Path] = []
-    for c in candidates:
-        if c.parent in changed_dirs or c.suffix.lower() in changed_extensions:
-            priority.append(c)
-        else:
-            rest.append(c)
-
-    budget = _MAX_CANDIDATE_FILES - len(priority)
-    if budget > 0:
-        priority.extend(rest[:budget])
-    return priority[:_MAX_CANDIDATE_FILES]
-
-
-def _collect_candidate_files(
-    root_dir: Path, included_set: set[Path], combined_spec: pathspec.PathSpec
-) -> tuple[list[Path], bool]:
+def _collect_candidate_files(root_dir: Path, included_set: set[Path], combined_spec: pathspec.PathSpec) -> list[Path]:
     try:
         result = subprocess.run(
             ["git", "ls-files", "-z"],
@@ -147,16 +115,7 @@ def _collect_candidate_files(
         if result.returncode == 0 and result.stdout:
             out = result.stdout.decode("utf-8", errors="surrogateescape")
             files = [root_dir / f for f in out.split("\0") if f]
-            candidates = [f for f in files if _is_candidate_file(f, root_dir, included_set, combined_spec)]
-            is_large_repo = len(candidates) > _MAX_CANDIDATE_FILES
-            if is_large_repo:
-                logger.debug(
-                    "diffctx: %d candidates exceed cap %d, prioritizing by proximity",
-                    len(candidates),
-                    _MAX_CANDIDATE_FILES,
-                )
-                candidates = _prioritize_candidates(candidates, included_set)
-            return candidates, is_large_repo
+            return [f for f in files if _is_candidate_file(f, root_dir, included_set, combined_spec)]
     except (subprocess.SubprocessError, OSError):
         pass
     logger.warning("diffctx: git ls-files failed, falling back to rglob (limit %d files)", _FALLBACK_MAX_FILES)
@@ -167,39 +126,15 @@ def _collect_candidate_files(
             break
         if _is_candidate_file(f, root_dir, included_set, combined_spec):
             fallback.append(f)
-    return fallback, False
-
-
-def _path_distance(a: Path, b: Path) -> int:
-    a_parts = a.parent.parts
-    b_parts = b.parent.parts
-    common = 0
-    for x, y in zip(a_parts, b_parts):
-        if x != y:
-            break
-        common += 1
-    return (len(a_parts) - common) + (len(b_parts) - common)
+    return fallback
 
 
 def _build_ident_index(
     files: list[Path],
     concepts: frozenset[str],
-    changed_files: list[Path] | None = None,
 ) -> dict[str, list[Path]]:
-    if changed_files:
-        changed_dirs = {f.parent for f in changed_files}
-
-        def sort_key(p: Path) -> tuple[int, int, str]:
-            in_same_dir = 0 if p.parent in changed_dirs else 1
-            min_dist = min((_path_distance(p, cf) for cf in changed_files), default=0)
-            return (in_same_dir, min_dist, str(p))
-
-        prioritized = sorted(files, key=sort_key)[:2000]
-    else:
-        prioritized = sorted(files)[:2000]
-
     inverted_index: dict[str, list[Path]] = defaultdict(list)
-    for file_path in prioritized:
+    for file_path in files:
         try:
             content = file_path.read_text(encoding="utf-8")
             file_idents = extract_identifiers(content, skip_stopwords=False)
@@ -240,8 +175,6 @@ def _collect_expansion_files(
         for file_path in inverted_index.get(concept, []):
             if file_path not in included_set and not _is_build_or_manifest(file_path):
                 expansion_files.add(file_path)
-                if len(expansion_files) >= _MAX_EXPANSION_FILES:
-                    return list(expansion_files)
 
     return list(expansion_files)
 
@@ -286,7 +219,6 @@ def _expand_universe_by_rare_identifiers(
     already_included: list[Path],
     combined_spec: pathspec.PathSpec,
     candidate_files: list[Path] | None = None,
-    changed_files: list[Path] | None = None,
 ) -> list[Path]:
     if not concepts:
         return []
@@ -295,8 +227,8 @@ def _expand_universe_by_rare_identifiers(
     if candidate_files is not None:
         files = [f for f in candidate_files if f not in included_set]
     else:
-        files, _ = _collect_candidate_files(root_dir, included_set, combined_spec)
-    inverted_index = _build_ident_index(files, concepts, changed_files=changed_files)
+        files = _collect_candidate_files(root_dir, included_set, combined_spec)
+    inverted_index = _build_ident_index(files, concepts)
 
     included_concept_counts: dict[str, int] = {}
     for f in already_included:

tests/cases/diff/go_035_http_handler.yaml

@@ -1,6 +1,4 @@
 name: go_035_http_handler
-xfail:
-  category: go-cross-file-discovery
 repo:
   initial_files:
     api/handlers.go: |

tests/cases/diff/jvm_and_compiled_060_swift_async_await.yaml

@@ -1,6 +1,4 @@
 name: jvm_and_compiled_060_swift_async_await
-xfail:
-  category: swift-same-module-edges
 repo:
   initial_files:
     User.swift: |

tests/cases/diff/jvm_and_compiled_061_swift_codable.yaml

@@ -1,6 +1,4 @@
 name: jvm_and_compiled_061_swift_codable
-xfail:
-  category: swift-same-module-edges
 repo:
   initial_files:
     User.swift: |