feat: enhance system robustness, diagnostics, and retrieval precision

- Implement MockVectorStore with .npy persistence as a fallback for FAISS-less environments.
- Enhance 'knowcode doctor' with comprehensive native and optional dependency checks.
- Fix Reranker initialization to respect AppConfig model settings regardless of VoyageAI status.
- Improve tokenizer to preserve compound identifiers (CamelCase/snake_case) for superior BM25 precision.
- Integrate SqliteChunkRepository for robust, scalable metadata persistence.
- Add comprehensive unit tests for Reranker fallbacks, Search Engine error handling, and MockVectorStore persistence.
- Improve telemetry reliability in test environments with synchronous flushing.

Author: deepakdgupta1

conftest.py

@@ -19,3 +19,9 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         default=False,
         help="Skip the eval golden SHA guard for local exploration.",
     )
+
+
+def pytest_configure(config: pytest.Config) -> None:
+    """Configure repository-wide settings before tests run."""
+    import os
+    os.environ["KNOWCODE_TESTING"] = "1"

src/knowcode/doctor.py

@@ -4,6 +4,7 @@
 
 import asyncio
 import contextlib
+import importlib
 import json
 import os
 import sys
@@ -80,6 +81,8 @@ def run_doctor(
     store_root = store_file.parent
     _check_knowledge_store(store_file, checks)
 
+    _check_dependencies(checks)
+
     resolved_index = Path(index_path) if index_path is not None else store_root / "knowcode_index"
     _check_semantic_index(resolved_index, config, checks)
 
@@ -208,6 +211,73 @@ def _check_api_keys(config: AppConfig, checks: list[DoctorCheck]) -> None:
     )
 
 
+def _check_dependencies(checks: list[DoctorCheck]) -> None:
+    """Check for optional and native dependencies."""
+    from importlib.util import find_spec
+
+    deps = [
+        ("faiss", "knowcode[search]", "Required for fast dense vector retrieval."),
+        ("mcp", "knowcode[mcp]", "Required for Model Context Protocol server."),
+        ("voyageai", "knowcode[voyageai]", "Required for VoyageAI embeddings and reranking."),
+        ("openai", "knowcode[llm]", "Required for OpenAI embeddings."),
+        ("google.genai", "knowcode[llm]", "Required for Google Gemini integration."),
+        ("watchdog", "knowcode[watch]", "Required for background file indexing."),
+    ]
+
+    missing = []
+    for module, extra, reason in deps:
+        try:
+            if find_spec(module) is None:
+                missing.append((module, extra, reason))
+        except (ModuleNotFoundError, ImportError):
+            missing.append((module, extra, reason))
+
+    # FAISS is special - it's a native binary
+    faiss_installed = True
+    try:
+        importlib.import_module("faiss")
+    except (ImportError, RuntimeError):
+        faiss_installed = False
+
+    if not faiss_installed:
+        checks.append(
+            DoctorCheck(
+                name="Native dependencies",
+                status="warn",
+                message="FAISS native binaries not found. Using MockVectorStore fallback.",
+                hint='Install with `pip install "faiss-cpu>=1.7.0"`.',
+            )
+        )
+    else:
+        checks.append(
+            DoctorCheck(
+                name="Native dependencies",
+                status="pass",
+                message="FAISS native binaries are installed and functional.",
+            )
+        )
+
+    if missing:
+        msg = f"Missing {len(missing)} optional dependencies."
+        hint = "Install them using: pip install " + " ".join(f'"{extra}"' for _, extra, _ in missing)
+        checks.append(
+            DoctorCheck(
+                name="Optional dependencies",
+                status="warn",
+                message=msg,
+                hint=hint,
+            )
+        )
+    else:
+        checks.append(
+            DoctorCheck(
+                name="Optional dependencies",
+                status="pass",
+                message="All optional dependencies are installed.",
+            )
+        )
+
+
 def _check_knowledge_store(store_file: Path, checks: list[DoctorCheck]) -> None:
     """Validate knowledge store presence and schema."""
     if not store_file.exists():

src/knowcode/indexing/indexer.py

@@ -5,7 +5,7 @@
 from pathlib import Path
 from typing import Any, Optional
 
-from knowcode.storage.chunk_repository import InMemoryChunkRepository
+from knowcode.storage.chunk_repository import ChunkRepository, InMemoryChunkRepository
 from knowcode.indexing.chunker import Chunker
 from knowcode.indexing.graph_builder import GraphBuilder
 from knowcode.indexing.scanner import Scanner
@@ -26,7 +26,7 @@ class Indexer:
     def __init__(
         self,
         embedding_provider: EmbeddingProviderProtocol,
-        chunk_repo: Optional[InMemoryChunkRepository] = None,
+        chunk_repo: Optional[ChunkRepository] = None,
         vector_store: Optional[VectorStoreProtocol] = None,
     ) -> None:
         """Initialize an indexer with optional storage backends.
@@ -37,7 +37,7 @@ def __init__(
             vector_store: Optional vector store (defaults to FAISS-backed store).
         """
         self.embedding_provider = embedding_provider
-        self.chunk_repo = chunk_repo or InMemoryChunkRepository()
+        self.chunk_repo: ChunkRepository = chunk_repo or InMemoryChunkRepository()
         self.vector_store = vector_store or VectorStore(dimension=embedding_provider.config.dimension)
         self.chunker = Chunker()
         self.manifest: dict[str, Any] = {}
@@ -93,34 +93,20 @@ def save(self, path: str | Path) -> None:
         Args:
             path: Directory path to write index files into.
         """
+        import json
+        import time
+        from dataclasses import asdict
+
         path = Path(path)
         path.mkdir(parents=True, exist_ok=True)
-        
+
         # Save vector store
         self.vector_store.save(path / "vectors")
-        
-        # Save chunk metadata (BM25 tokens and content)
-        import json
-        metadata = {
-            "schema_version": self.SCHEMA_VERSION,
-            "chunks": [
-                {
-                    "id": c.id,
-                    "entity_id": c.entity_id,
-                    "content": c.content,
-                    "tokens": c.tokens,
-                    "metadata": c.metadata
-                }
-                for c in self.chunk_repo._chunks.values()
-            ]
-        }
-        with open(path / "chunks.json", "w") as f:
-            json.dump(metadata, f)
 
-        # Save index manifest for compatibility checks at query time.
-        from dataclasses import asdict
-        import time
+        # Save chunk metadata via repository
+        self.chunk_repo.save(path)
 
+        # Save index manifest for compatibility checks at query time.
         self.manifest = {
             "schema_version": self.SCHEMA_VERSION,
             "version": 1,
@@ -160,22 +146,8 @@ def load(self, path: str | Path) -> None:
                 "version": self.LEGACY_MANIFEST_VERSION,
             }
 
-        # Load chunks
-        from knowcode.models import CodeChunk  # type: ignore
-        
-        chunks_file = path / "chunks.json"
-        if chunks_file.exists():
-            with open(chunks_file) as f:
-                data = json.load(f)
-            if isinstance(data, dict):
-                validated_chunks = self._validate_and_migrate_chunks_payload(data)
-                chunk_entries = validated_chunks.get("chunks", [])
-                if isinstance(chunk_entries, list):
-                    for c_data in chunk_entries:
-                        if not isinstance(c_data, dict):
-                            continue
-                        chunk = CodeChunk(**c_data)
-                        self.chunk_repo.add(chunk)
+        # Load chunks via repository
+        self.chunk_repo.load(path)
 
     @classmethod
     def _validate_and_migrate_manifest(cls, manifest: dict[str, Any]) -> dict[str, Any]:
@@ -260,12 +232,13 @@ def remove_file(self, file_path: str | Path) -> None:
         file_path_str = str(Path(file_path).resolve())
         removed_ids = self.chunk_repo.remove_by_file(file_path_str)
         if removed_ids:
-            # Rebuild vector store with remaining chunks
-            remaining_chunks = list(self.chunk_repo._chunks.values())
-            self.vector_store.clear()
-            for c in remaining_chunks:
-                if c.embedding:
-                    self.vector_store.add(c.id, c.embedding)
+            # Rebuild the vector store from remaining chunks if embeddings are available.
+            remaining_chunks = self.chunk_repo.get_all()
+            if any(c.embedding for c in remaining_chunks):
+                self.vector_store.clear()
+                for c in remaining_chunks:
+                    if c.embedding:
+                        self.vector_store.add(c.id, c.embedding)
 
     def index_file(self, file_path: str | Path) -> int:
 

src/knowcode/retrieval/reranker.py

@@ -30,14 +30,15 @@ def __init__(
         """
         self.voyage_client = None
         self.model = "rerank-2.5"
+
+        if config and config.reranking_models:
+            self.model = config.reranking_models[0].name
+            if not api_key_env:
+                api_key_env = config.reranking_models[0].api_key_env
 
         if use_voyageai:
-            # Determine API key env from config or default
-            if config and config.reranking_models:
-                api_key_env = config.reranking_models[0].api_key_env
-                self.model = config.reranking_models[0].name
-            else:
-                api_key_env = api_key_env or "VOYAGE_API_KEY_1"
+            if not api_key_env:
+                api_key_env = "VOYAGE_API_KEY_1"
 
             # Try to initialize VoyageAI client
             try:

src/knowcode/service.py

@@ -139,16 +139,17 @@ def get_indexer(self, index_path: Optional[str | Path] = None) -> "Indexer":
         if self._indexer is None:
             from knowcode.llm.embedding import create_embedding_provider
             from knowcode.indexing.indexer import Indexer
+            from knowcode.storage.sqlite_chunk_repository import SqliteChunkRepository
 
             provider = create_embedding_provider(app_config=self.app_config)
-            self._indexer = Indexer(provider)
+            resolved_index_path = Path(index_path) if index_path else self._index_path()
+            db_path = resolved_index_path / "chunks.db"
+            chunk_repo = SqliteChunkRepository(db_path)
 
-            if index_path:
-                self._indexer.load(Path(index_path))
-            else:
-                default_index = self._index_path()
-                if default_index.exists():
-                    self._indexer.load(default_index)
+            self._indexer = Indexer(provider, chunk_repo=chunk_repo)
+
+            if resolved_index_path.exists():
+                self._indexer.load(resolved_index_path)
 
         return self._indexer
 
@@ -294,11 +295,23 @@ def _build_index(self, directory: str | Path, index_path: str | Path) -> int:
         """Build a semantic index for a directory and persist it."""
         from knowcode.llm.embedding import create_embedding_provider
         from knowcode.indexing.indexer import Indexer
+        from knowcode.storage.sqlite_chunk_repository import SqliteChunkRepository
 
         provider = create_embedding_provider(app_config=self.app_config)
-        indexer = Indexer(provider)
+        resolved_index_path = Path(index_path)
+        
+        # Clear/initialize directory
+        import shutil
+        if resolved_index_path.exists():
+            shutil.rmtree(resolved_index_path)
+        resolved_index_path.mkdir(parents=True, exist_ok=True)
+        
+        db_path = resolved_index_path / "chunks.db"
+        chunk_repo = SqliteChunkRepository(db_path)
+
+        indexer = Indexer(provider, chunk_repo=chunk_repo)
         count = indexer.index_directory(directory)
-        indexer.save(index_path)
+        indexer.save(resolved_index_path)
         self._indexer = indexer
         return count
 
@@ -538,7 +551,7 @@ def get_stats(self) -> dict[str, Any]:
 
         # Add index stats if indexer is loaded
         if self._indexer:
-            stats["total_chunks"] = len(self._indexer.chunk_repo._chunks)
+            stats["total_chunks"] = self._indexer.chunk_repo.count()
             if (
                 hasattr(self._indexer.vector_store, "index")
                 and self._indexer.vector_store.index