feat: introduce configurable hybrid search alpha and reranker top-k multiplier with telemetry and quality evaluation tools

Author: deepakdgupta1

aimodels.yaml

@@ -20,3 +20,5 @@ eval_models:
 # Configuration
 config:
   sufficiency_threshold: 0.8 # Configurable threshold for local-first answering
+  hybrid_alpha: 0.2
+  reranker_top_k_multiplier: 5

src/knowcode/config.py

@@ -33,12 +33,14 @@ class AppConfig:
         "eval_models",
         "config",
     }
-    KNOWN_CONFIG_KEYS = {"sufficiency_threshold"}
+    KNOWN_CONFIG_KEYS = {"sufficiency_threshold", "hybrid_alpha", "reranker_top_k_multiplier"}
 
     models: list[ModelConfig] = field(default_factory=list)
     embedding_models: list[ModelConfig] = field(default_factory=list)
     reranking_models: list[ModelConfig] = field(default_factory=list)
     sufficiency_threshold: float = 0.8  # For local-first answering
+    hybrid_alpha: float = 0.2
+    reranker_top_k_multiplier: int = 5
 
     @classmethod
     def load(cls, config_path: Optional[str] = None, strict: bool = False) -> "AppConfig":
@@ -87,6 +89,8 @@ def default(cls) -> "AppConfig":
                 )
             ],
             sufficiency_threshold=0.8,
+            hybrid_alpha=0.2,
+            reranker_top_k_multiplier=5,
         )
 
     @classmethod
@@ -153,6 +157,14 @@ def _load_from_yaml(cls, path: Path, strict: bool = False) -> "AppConfig":
             if not isinstance(sufficiency_threshold, (int, float)):
                 raise ValueError("'config.sufficiency_threshold' must be a number.")
 
+            hybrid_alpha = config_section.get("hybrid_alpha", 0.5)
+            if not isinstance(hybrid_alpha, (int, float)):
+                raise ValueError("'config.hybrid_alpha' must be a number.")
+            
+            reranker_top_k_multiplier = config_section.get("reranker_top_k_multiplier", 5)
+            if not isinstance(reranker_top_k_multiplier, int):
+                raise ValueError("'config.reranker_top_k_multiplier' must be an integer.")
+            
             if not models:
                 models = cls.default().models
 
@@ -161,6 +173,8 @@ def _load_from_yaml(cls, path: Path, strict: bool = False) -> "AppConfig":
                 embedding_models=embedding_models,
                 reranking_models=reranking_models,
                 sufficiency_threshold=sufficiency_threshold,
+                hybrid_alpha=hybrid_alpha,
+                reranker_top_k_multiplier=reranker_top_k_multiplier,
             )
         except Exception as e:
             message = f"Failed to load config from {path}: {e}"

src/knowcode/retrieval/hybrid_index.py

@@ -47,10 +47,10 @@ def search(
         # 1. BM25 Search
         query_tokens = tokenize_code(query)
         # We get more results for fusion
-        sparse_results = self.chunk_repo.search_by_tokens(query_tokens, limit=limit * 2)
+        sparse_results = self.chunk_repo.search_by_tokens(query_tokens, limit=limit * 3)
 
         # 2. Vector Search
-        dense_results = self.vector_store.search(query_embedding, limit=limit * 2)
+        dense_results = self.vector_store.search(query_embedding, limit=limit * 3)
 
         # 3. Combine scores (RRF)
         combined_scores: dict[str, float] = {}

src/knowcode/retrieval/reranker.py

@@ -44,8 +44,10 @@ def __init__(
             try:
                 from knowcode.llm.voyageai_client import get_voyageai_client
                 self.voyage_client = get_voyageai_client(api_key_env)
-            except Exception:
-                pass  # Fall back to signal-based
+            except ImportError as e:
+                print(f"  ⚠️ VoyageAI client not available (missing package): {e}")
+            except ValueError as e:
+                print(f"  ⚠️ VoyageAI client not available (configuration error): {e}")
 
     def rerank(
         self,
@@ -75,15 +77,45 @@ def rerank(
 
         # Try VoyageAI cross-encoder reranking
         if self.voyage_client:
+            start_time = time.time()
             try:
-                return self._rerank_with_voyageai(query, chunks, top_k)
+                result = self._rerank_with_voyageai(query, chunks, top_k)
+                latency = time.time() - start_time
+                self._log_telemetry("voyageai", latency, len(chunks))
+                return result
+            except ValueError as e:
+                print(f"  ⚠️ VoyageAI reranking failed (invalid input): {e}. Using signal-based fallback.")
+            except ConnectionError as e:
+                print(f"  ⚠️ VoyageAI reranking failed (network error): {e}. Using signal-based fallback.")
             except Exception as e:
+                import logging
+                logging.error(f"VoyageAI reranking API failed: {e}")
                 print(f"  ⚠️ VoyageAI reranking failed: {e}. Using signal-based fallback.")
 
         # Fallback to signal-based reranking
-        return self._rerank_with_signals(
+        start_time = time.time()
+        result = self._rerank_with_signals(
             query, chunks, boost_recent, boost_documented, top_k
         )
+        latency = time.time() - start_time
+        self._log_telemetry("signal_based", latency, len(chunks))
+        return result
+        
+    def _log_telemetry(self, method: str, latency: float, num_chunks: int) -> None:
+        """Log reranking latency telemetry."""
+        from knowcode.telemetry import log_event
+        try:
+            log_event(
+                ".",  # We don't have store_path here, telemetry.py handles fallback to current dir
+                {
+                    "event_type": "reranker_latency",
+                    "method": method,
+                    "latency_seconds": latency,
+                    "num_chunks": num_chunks,
+                }
+            )
+        except Exception:
+            pass
 
     def _rerank_with_voyageai(
         self,

src/knowcode/retrieval/search_engine.py

@@ -53,6 +53,7 @@ def __init__(
             use_voyageai=use_voyageai_reranking,
             config=config,
         )
+        self.multiplier = config.reranker_top_k_multiplier if config else 5
 
     def search_scored(
         self,
@@ -74,7 +75,7 @@ def search_scored(
             Ranked list of ScoredChunk objects.
         """
         query_embedding = self.embedding_provider.embed_single(query)
-        results = self.hybrid_index.search(query, query_embedding, limit=limit * 2)
+        results = self.hybrid_index.search(query, query_embedding, limit=limit * self.multiplier)
         reranked = self.reranker.rerank(query, results, top_k=limit)
         primary = [ScoredChunk(chunk=c, score=s, source="retrieved") for c, s in reranked]
 

src/knowcode/service.py

@@ -169,7 +169,11 @@ def get_search_engine(
             from knowcode.retrieval.search_engine import SearchEngine
 
             indexer = self.get_indexer(index_path)
-            hybrid_index = HybridIndex(indexer.chunk_repo, indexer.vector_store)
+            hybrid_index = HybridIndex(
+                indexer.chunk_repo, 
+                indexer.vector_store,
+                alpha=self.app_config.hybrid_alpha
+            )
 
             self._search_engine = SearchEngine(
                 indexer.chunk_repo,