feat: migrate to persistent storage (SQLite/LanceDB) and introduce exact query matching

- Storage Evolution: Replaced the legacy InMemoryChunkRepository with SqliteChunkRepository featuring FTS5 for BM25 search.
- LanceDB Integration: Added LancedbVectorStore as a scalable vector storage backend, improving retrieval performance for large codebases.
- Exact Query Engine: Implemented ExactQueryEngine to handle quoted queries, allowing for precise substring matching across the codebase.
- Live Context Hydration: Introduced LiveSourceLoader to ContextSynthesizer, ensuring context bundles can be hydrated with the most up-to-date source code from the file system.
- Architectural Refinement: Migrated KnowledgeStore to SQLite persistence and updated CLI/API defaults to use persistent stores.
- Verification: Comprehensive updates to unit and integration tests to support the new persistence layer and verify the exact matching logic.

Author: deepakdgupta1

pyproject.toml

@@ -24,6 +24,7 @@ server = [
 search = [
     "faiss-cpu>=1.7.0",
     "numpy>=1.24.0",
+    "lancedb>=0.12.0",
 ]
 llm = [
     "openai>=1.0.0",
@@ -37,6 +38,7 @@ all = [
     "slowapi>=0.1.9",
     "faiss-cpu>=1.7.0",
     "numpy>=1.24.0",
+    "lancedb>=0.12.0",
     "openai>=1.0.0",
     "google-genai>=0.3.0",
     "google-api-core>=2.29.0",

scripts/fix_swallowed.py

@@ -0,0 +1,23 @@
+import os
+import re
+
+target_dir = "/Users/deepg/Desktop/KnowCode/src/knowcode"
+pattern = re.compile(r'except Exception:\s+pass')
+replacement = r'''except Exception as e:
+            import logging
+            logging.getLogger(__name__).warning("Ignored exception: %s", e)'''
+
+def fix_swallowed(path):
+    with open(path, 'r', encoding='utf-8') as f:
+        content = f.read()
+    
+    new_content = pattern.sub(replacement, content)
+    if new_content != content:
+        with open(path, 'w', encoding='utf-8') as f:
+            f.write(new_content)
+        print(f"Fixed {path}")
+
+for root, _, files in os.walk(target_dir):
+    for file in files:
+        if file.endswith('.py'):
+            fix_swallowed(os.path.join(root, file))

src/knowcode/__init__.py

@@ -5,12 +5,10 @@
 from knowcode.data_models import CodeChunk as CodeChunk
 from knowcode.data_models import EmbeddingConfig as EmbeddingConfig
 from knowcode.storage.chunk_repository import ChunkRepository as ChunkRepository
-from knowcode.storage.chunk_repository import InMemoryChunkRepository as InMemoryChunkRepository
 
 __all__ = [
     "__version__",
     "ChunkRepository",
     "CodeChunk",
     "EmbeddingConfig",
-    "InMemoryChunkRepository",
 ]

src/knowcode/analysis/context_synthesizer.py

@@ -1,7 +1,10 @@
 """Context synthesizer for generating AI-ready context bundles."""
 
 from dataclasses import dataclass
-from typing import Optional
+from typing import Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from knowcode.analysis.live_source_loader import LiveSourceLoader
 
 from knowcode.storage.knowledge_store import KnowledgeStore
 from knowcode.data_models import Entity, EntityKind, TaskType
@@ -74,6 +77,7 @@ def __init__(
         store: KnowledgeStore,
         max_tokens: int = DEFAULT_MAX_TOKENS,
         model: str = "gpt-4",
+        live_loader: Optional["LiveSourceLoader"] = None,
     ) -> None:
         """Initialize context synthesizer.
 
@@ -85,6 +89,15 @@ def __init__(
         self.store = store
         self.max_tokens = max_tokens
         self.tokenizer = TokenCounter(model)
+        self.live_loader = live_loader
+
+    def _get_entity_source(self, entity: Entity) -> str:
+        """Get entity source code, preferring live file if available."""
+        if self.live_loader:
+            live_source = self.live_loader.load_source(entity)
+            if live_source is not None:
+                return live_source
+        return entity.source_code or ""
 
     def synthesize(self, entity_id: str, summarize: bool = False) -> Optional[ContextBundle]:
         """Synthesize context bundle for an entity.
@@ -146,14 +159,15 @@ def synthesize(self, entity_id: str, summarize: bool = False) -> Optional[Contex
                 current_tokens += t
 
         # Priority 2: Source Code (Huge consumer, often truncated)
-        if entity.source_code and not summarize:
+        source_code = self._get_entity_source(entity)
+        if source_code and not summarize:
             code_header = "## Source Code\n\n```python\n"
             code_footer = "\n```"
             overhead = self.tokenizer.count_tokens(code_header + code_footer)
             remaining = self.max_tokens - current_tokens - overhead
 
             if remaining > 100: # Only add if we have decent space
-                code_body = entity.source_code
+                code_body = source_code
                 code_tokens = self.tokenizer.count_tokens(code_body)
 
                 if code_tokens > remaining:
@@ -211,7 +225,7 @@ def synthesize(self, entity_id: str, summarize: bool = False) -> Optional[Contex
         context_text = "\n\n---\n\n".join(sections)
 
         # Check if we skipped source code but had it
-        if entity.source_code and "## Source Code" not in context_text:
+        if source_code and "## Source Code" not in context_text:
              is_truncated = True
 
         return ContextBundle(
@@ -403,10 +417,11 @@ def synthesize_with_task(
         if entity.docstring:
             content_sections["docstring"] = f"## Description\n\n{entity.docstring}"
 
-        if entity.source_code and not summarize:
+        source_code = self._get_entity_source(entity)
+        if source_code and not summarize:
             code_header = "## Source Code\n\n```python\n"
             code_footer = "\n```"
-            code_body = entity.source_code
+            code_body = source_code
             # Pre-truncate if too long
             max_code_tokens = int(self.max_tokens * 0.5)  # Reserve half for code max
             code_tokens = self.tokenizer.count_tokens(code_body)
@@ -518,7 +533,8 @@ def _calculate_sufficiency(
                 score += weight
 
         # Bonus for having source code (always valuable)
-        if entity.source_code and "## Source Code" in context_text:
+        source_code = self._get_entity_source(entity)
+        if source_code and "## Source Code" in context_text:
             score += 0.2
             max_score += 0.2
 

src/knowcode/analysis/live_source_loader.py

@@ -0,0 +1,48 @@
+"""Loader for authoritative source code reads from disk."""
+
+from pathlib import Path
+from knowcode.data_models import Entity
+
+
+class LiveSourceLoader:
+    """Safely reads and slices live source files from disk."""
+
+    def __init__(self, root_dir: str | Path) -> None:
+        """Initialize with repository root directory."""
+        self.root_dir = Path(root_dir)
+
+    def load_source(self, entity: Entity) -> str | None:
+        """Load live source code from disk for the given entity.
+        
+        Args:
+            entity: Target entity containing location data.
+            
+        Returns:
+            The source code string sliced from the live file, 
+            or None if the file is missing or reading fails.
+        """
+        if not entity.location or not entity.location.file_path:
+            return None
+            
+        file_path = self.root_dir / entity.location.file_path
+        if not file_path.exists():
+            return None
+            
+        try:
+            with open(file_path, "r", encoding="utf-8") as f:
+                lines = f.readlines()
+                
+            start = entity.location.line_start - 1
+            end = entity.location.line_end
+            
+            if start < 0:
+                start = 0
+            if end > len(lines):
+                end = len(lines)
+                
+            if start >= len(lines):
+                return None
+                
+            return "".join(lines[start:end])
+        except Exception:
+            return None

src/knowcode/api/api.py

@@ -364,9 +364,10 @@ def trace_calls(
     up to the specified depth. Each result includes the call_depth indicating
     how many hops from the starting entity.
     """
-    return service.store.trace_calls(
+    from typing import cast
+    return cast(list[dict[str, Any]], service.store.trace_calls(
         entity_id, direction=direction.value, depth=depth, max_results=max_results
-    )
+    ))
 
 
 @router.get("/impact/{entity_id:path}", summary="Impact Analysis")
@@ -387,4 +388,5 @@ def get_impact(
     - affected_files: Files that would need review
     - risk_score: 0.0-1.0 indicating modification risk
     """
-    return service.store.get_impact(entity_id, max_depth=max_depth)
+    from typing import cast
+    return cast(dict[str, Any], service.store.get_impact(entity_id, max_depth=max_depth))

src/knowcode/cli/cli.py

@@ -105,11 +105,17 @@ def analyze(directory: str, output: str, ignore: tuple[str, ...], temporal: bool
     type=click.Path(exists=True, dir_okay=False),
     help="Path to configuration file (aimodels.yaml) for embedding models.",
 )
+@click.option(
+    "--incremental",
+    is_flag=True,
+    help="Use incremental indexing to speed up subsequent builds.",
+)
 def build(
     directory: str,
     ignore: tuple[str, ...],
     temporal: bool,
     config: Optional[str],
+    incremental: bool,
 ) -> None:
     """Build the knowledge base and semantic index for a directory.
 
@@ -134,6 +140,7 @@ def build(
         output=str(target),
         ignore=list(ignore),
         temporal=temporal,
+        incremental=incremental,
     )
 
     click.echo("\n✓ Build complete!")
@@ -174,7 +181,12 @@ def build(
     type=click.Path(exists=True, dir_okay=False),
     help="Path to configuration file (aimodels.yaml) for embedding models.",
 )
-def index(directory: str, output: str, config: Optional[str]) -> None:
+@click.option(
+    "--incremental",
+    is_flag=True,
+    help="Use incremental indexing to speed up subsequent builds.",
+)
+def index(directory: str, output: str, config: Optional[str], incremental: bool) -> None:
     """Build semantic search index for a codebase.
 
     DIRECTORY: Path to the codebase to index.
@@ -191,7 +203,27 @@ def index(directory: str, output: str, config: Optional[str]) -> None:
         provider = create_embedding_provider(app_config=app_config)
         indexer = Indexer(provider)
 
-        count = indexer.index_directory(directory)
+        index_path = Path(output)
+        if incremental and (index_path / "index_manifest.json").exists():
+            from knowcode.storage.sqlite_chunk_repository import SqliteChunkRepository
+            db_path = index_path / "chunks.db"
+            indexer.chunk_repo = SqliteChunkRepository(db_path)
+            
+            vector_backend = app_config.vector_backend
+            dimension = provider.config.dimension
+            if vector_backend == "lancedb":
+                from knowcode.storage.lancedb_vector_store import LanceDBVectorStore
+                vs_path = index_path / "vectors.lancedb"
+                indexer.vector_store = LanceDBVectorStore(dimension=dimension, path=vs_path)
+            else:
+                from knowcode.storage.vector_store import VectorStore
+                indexer.vector_store = VectorStore(dimension=dimension, index_path=index_path)
+
+            indexer.load(index_path)
+            count = indexer.index_incremental(directory)
+        else:
+            count = indexer.index_directory(directory)
+            
         indexer.save(output)
 
         click.echo(f"✓ Indexing complete! Created {count} chunks.")
@@ -607,7 +639,7 @@ def history(target: Optional[str], store: str, limit: int) -> None:
         changes = []
         for rel in rels:
             if rel.kind == RelationshipKind.CHANGED_BY:
-                commit = knowledge.get_entity(rel.target_id)  # type: ignore
+                commit = knowledge.get_entity(rel.target_id)
                 if commit:
                     # Get modification stats from edge metadata
                     stats = f"(+{rel.metadata.get('insertions', 0)}/-{rel.metadata.get('deletions', 0)})"
@@ -622,7 +654,7 @@ def history(target: Optional[str], store: str, limit: int) -> None:
 
         for _, commit, stats in changes[:limit]:
             date = commit.metadata.get("date", "")
-            click.echo(f"  {date} {commit.name} {stats}: {commit.docstring.splitlines()[0]}")  # type: ignore
+            click.echo(f"  {date} {commit.name} {stats}: {commit.docstring.splitlines()[0]}")
 
 
 @cli.command()

src/knowcode/config.py

@@ -33,14 +33,15 @@ class AppConfig:
         "eval_models",
         "config",
     }
-    KNOWN_CONFIG_KEYS = {"sufficiency_threshold", "hybrid_alpha", "reranker_top_k_multiplier"}
+    KNOWN_CONFIG_KEYS = {"sufficiency_threshold", "hybrid_alpha", "reranker_top_k_multiplier", "vector_backend"}
 
     models: list[ModelConfig] = field(default_factory=list)
     embedding_models: list[ModelConfig] = field(default_factory=list)
     reranking_models: list[ModelConfig] = field(default_factory=list)
     sufficiency_threshold: float = 0.8  # For local-first answering
     hybrid_alpha: float = 0.2
     reranker_top_k_multiplier: int = 5
+    vector_backend: str = "lancedb"
 
     @classmethod
     def load(cls, config_path: Optional[str] = None, strict: bool = False) -> "AppConfig":
@@ -91,6 +92,7 @@ def default(cls) -> "AppConfig":
             sufficiency_threshold=0.8,
             hybrid_alpha=0.2,
             reranker_top_k_multiplier=5,
+            vector_backend="lancedb",
         )
 
     @classmethod
@@ -165,6 +167,10 @@ def _load_from_yaml(cls, path: Path, strict: bool = False) -> "AppConfig":
             if not isinstance(reranker_top_k_multiplier, int):
                 raise ValueError("'config.reranker_top_k_multiplier' must be an integer.")
 
+            vector_backend = config_section.get("vector_backend", "lancedb")
+            if not isinstance(vector_backend, str) or vector_backend not in ("faiss", "lancedb"):
+                raise ValueError("'config.vector_backend' must be 'faiss' or 'lancedb'.")
+            
             if not models:
                 models = cls.default().models
 
@@ -175,6 +181,7 @@ def _load_from_yaml(cls, path: Path, strict: bool = False) -> "AppConfig":
                 sufficiency_threshold=sufficiency_threshold,
                 hybrid_alpha=hybrid_alpha,
                 reranker_top_k_multiplier=reranker_top_k_multiplier,
+                vector_backend=vector_backend,
             )
         except Exception as e:
             message = f"Failed to load config from {path}: {e}"