feat: add document existence check

- Modified Chunker to fetch token count using named columns.
- FileReader to handle UTF-8 files with Unicode characters.

Author: Daniele Briggi

.gitignore

@@ -1,4 +1,5 @@
 semsearch/
+docs/
 
 # LLM models
 *.gguf

src/sqlite_rag/chunker.py

@@ -22,8 +22,8 @@ def _get_token_count(self, text: str) -> int:
         """Get token count using SQLite AI extension."""
         if text == "":
             return 0
-        cursor = self._conn.execute("SELECT llm_token_count(?)", (text,))
-        return cursor.fetchone()[0]
+        cursor = self._conn.execute("SELECT llm_token_count(?) AS count", (text,))
+        return cursor.fetchone()['count']
 
     def _estimate_tokens_count(self, text: str) -> int:
         """Estimate token count more conservatively."""

src/sqlite_rag/cli.py

@@ -122,11 +122,11 @@ def rebuild(
     """Rebuild embeddings and full-text index"""
     rag = SQLiteRag()
 
-    typer.echo("Starting rebuild process...")
+    typer.echo("Rebuild process...")
 
     result = rag.rebuild(remove_missing=remove_missing)
 
-    typer.echo(f"Rebuild completed:")
+    typer.echo("Rebuild completed:")
     typer.echo(f"  Total documents: {result['total']}")
     typer.echo(f"  Reprocessed: {result['reprocessed']}")
     typer.echo(f"  Not found: {result['not_found']}")
@@ -176,14 +176,12 @@ def search(
         return
 
     typer.echo(f"Found {len(results)} documents:")
-    # print the position, the snippet and the uri as a table
-    typer.echo(f"{'Position':<10} {'Content':<50}")
-    typer.echo("-" * 60)
-    for i, doc in enumerate(results, start=1):
-        uri_or_content = doc.uri or (
-            doc.content[:47] + "..." if len(doc.content) > 47 else doc.content
-        )
-        typer.echo(f"{i:<10} {uri_or_content:<50}")
+    typer.echo(f"{'Pos':<4} {'Preview':<60} {'URI':<50}")
+    typer.echo("-" * 116)
+    for idx, doc in enumerate(results, 1):
+        snippet = f"{doc.snippet[:57]!r}" + "..." if len(doc.snippet) > 60 else f"{doc.snippet!r}"
+        uri = doc.document.uri or "N/A"
+        typer.echo(f"{idx:<4} {snippet:<60} {uri:<50}")
 
 
 def repl_mode():

src/sqlite_rag/database.py

@@ -63,6 +63,7 @@ def _create_schema(conn: sqlite3.Connection, settings: Settings):
         cursor.execute(
             """
             CREATE TABLE IF NOT EXISTS chunks (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
                 document_id TEXT,
                 content TEXT,
                 embedding BLOB,
@@ -73,7 +74,7 @@ def _create_schema(conn: sqlite3.Connection, settings: Settings):
 
         cursor.execute(
             """
-            CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(content, content='chunks', content_rowid='rowid');
+            CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(content, content='chunks', content_rowid='id');
         """
         )
 

src/sqlite_rag/engine.py

@@ -1,6 +1,9 @@
+import re
 import sqlite3
 from pathlib import Path
 
+from sqlite_rag.models.document_result import DocumentResult
+
 from .chunker import Chunker
 from .models.chunk import Chunk
 from .models.document import Document
@@ -46,13 +49,13 @@ def generate_embedding(self, chunks: list[Chunk]) -> list[Chunk]:
         cursor = self._conn.cursor()
 
         for chunk in chunks:
-            cursor.execute("SELECT llm_embed_generate(?)", (chunk.content,))
+            cursor.execute("SELECT llm_embed_generate(?) AS embedding", (chunk.content,))
             result = cursor.fetchone()
 
             if result is None:
                 raise RuntimeError("Failed to generate embedding.")
 
-            chunk.embedding = result[0]
+            chunk.embedding = result['embedding']
 
         return chunks
 
@@ -64,42 +67,40 @@ def quantize(self) -> None:
 
         self._conn.commit()
 
-    def search(self, query: str, limit: int = 10) -> list[Document]:
+    def search(self, query: str, limit: int = 10) -> list[DocumentResult]:
         """Semantic search and full-text search sorted with Reciprocal Rank Fusion."""
         cursor = self._conn.cursor()
 
         query_embedding = self.generate_embedding([Chunk(content=query)])[0].embedding
 
+        # Clean up and split into words
+        query = " ".join(re.findall(r"\b\w+\b", query.lower()))
+
         cursor.execute(
             # TODO: use vector_convert_XXX to convert the query to the correct type
             """
             -- sqlite-vector KNN vector search results
             WITH vec_matches AS (
                 SELECT
+                    v.rowid AS chunk_id,
                     row_number() over (order by v.distance) AS rank_number,
-                    c.document_id,
                     v.distance
-                FROM chunks AS c
-                    JOIN vector_quantize_scan('chunks', 'embedding', vector_convert_f32(:query_embedding), :k) AS v
-                    ON c.rowid = v.rowid
+                FROM vector_quantize_scan('chunks', 'embedding', vector_convert_f32(:query_embedding), :k) AS v
             ),
             -- Full-text search results
             fts_matches AS (
                 SELECT
+                    chunks_fts.rowid AS chunk_id,
                     row_number() over (order by rank) AS rank_number,
-                    c.document_id,
                     rank AS score
-                FROM chunks_fts AS c_fts
-                    JOIN chunks AS c ON c_fts.rowid = c.rowid
-                WHERE c_fts.content MATCH :query
+                FROM chunks_fts
+                WHERE chunks_fts MATCH :query
                 LIMIT :k
             ),
             -- combine FTS5 + vector search results with RRF
             matches AS (
                 SELECT
-                    documents.id,
-                    documents.uri,
-                    documents.content,
+                    COALESCE(vec_matches.chunk_id, fts_matches.chunk_id) AS chunk_id,
                     vec_matches.rank_number AS vec_rank,
                     fts_matches.rank_number AS fts_rank,
                     -- Reciprocal Rank Fusion score
@@ -111,22 +112,26 @@ def search(self, query: str, limit: int = 10) -> list[Document]:
                     fts_matches.score AS fts_score
                 FROM vec_matches
                     FULL OUTER JOIN fts_matches
-                        ON vec_matches.document_id = fts_matches.document_id
-                    JOIN documents ON documents.id = COALESCE(vec_matches.document_id, fts_matches.document_id)
-                ORDER BY combined_rank DESC
+                        ON vec_matches.chunk_id = fts_matches.chunk_id
             )
             SELECT
-                id,
-                uri,
-                content,
+                documents.id,
+                documents.uri,
+                documents.content as document_content,
+                chunks.content AS snippet,
                 vec_rank,
                 fts_rank,
                 combined_rank,
                 vec_distance,
                 fts_score
-            FROM matches;
+            FROM matches
+                JOIN chunks ON chunks.id = matches.chunk_id
+                JOIN documents ON documents.id = chunks.document_id
+            ORDER BY combined_rank DESC
+            ;
             """,
             {
+                # '*' is used to match while typing
                 "query": query + "*",
                 "query_embedding": query_embedding,
                 "k": limit,
@@ -139,15 +144,18 @@ def search(self, query: str, limit: int = 10) -> list[Document]:
 
         rows = cursor.fetchall()
         return [
-            Document(
-                id=row[0],
-                uri=row[1],
-                content=row[2],
-                vec_rank=row[3],
-                fts_rank=row[4],
-                combined_rank=row[5],
-                vec_distance=row[6],
-                fts_score=row[7],
+            DocumentResult(
+                document=Document(
+                    id=row["id"],
+                    uri=row["uri"],
+                    content=row["document_content"],
+                ),
+                snippet=row["snippet"],
+                vec_rank=row["vec_rank"],
+                fts_rank=row["fts_rank"],
+                combined_rank=row["combined_rank"],
+                vec_distance=row["vec_distance"],
+                fts_score=row["fts_score"],
             )
             for row in rows
         ]

src/sqlite_rag/models/document_result.py

@@ -0,0 +1,17 @@
+from attr import dataclass
+
+from .document import Document
+
+
+@dataclass
+class DocumentResult:
+    document: Document
+
+    snippet: str
+
+    combined_rank: float
+    vec_rank: float | None = None
+    fts_rank: float | None = None
+
+    vec_distance: float | None = None
+    fts_score: float | None = None

src/sqlite_rag/reader.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-from markitdown import MarkItDown
+from markitdown import MarkItDown, StreamInfo
 
 
 class FileReader:
@@ -47,8 +47,8 @@ def is_supported(path: Path) -> bool:
     @staticmethod
     def parse_file(path: Path) -> str:
         try:
-            reader = MarkItDown()
-            return reader.convert(path).text_content
+            converter = MarkItDown()
+            return converter.convert(path, stream_info=StreamInfo(charset='utf8')).text_content
         except Exception as exc:
             raise ValueError(f"Failed to parse file {path}") from exc
 

src/sqlite_rag/repository.py

@@ -82,13 +82,21 @@ def find_document_by_id_or_uri(self, identifier: str) -> Document | None:
             )
         return None
 
+    def document_exists_by_hash(self, hash: str) -> bool:
+        """Check if a document with the given hash exists"""
+        cursor = self._conn.cursor()
+        cursor.execute("SELECT 1 FROM documents WHERE hash = ?", (hash,))
+        return cursor.fetchone() is not None
+
     def remove_document(self, document_id: str) -> bool:
         """Remove document and its chunks by document ID"""
         cursor = self._conn.cursor()
 
         # Check if document exists
-        cursor.execute("SELECT COUNT(*) FROM documents WHERE id = ?", (document_id,))
-        if cursor.fetchone()[0] == 0:
+        cursor.execute(
+            "SELECT COUNT(*) AS total FROM documents WHERE id = ?", (document_id,)
+        )
+        if cursor.fetchone()["total"] == 0:
             return False
 
         # Remove chunks first

src/sqlite_rag/settings.py

@@ -3,12 +3,13 @@ def __init__(self, model_path_or_name: str, db_path: str = "sqliterag.db"):
         self.model_path_or_name = model_path_or_name
         self.db_path = db_path
 
-        self.embedding_dim = 384  # Default for all-MiniLM-L6-v2
-        self.vector_type = "FLOAT32"  # Default vector type for sqlite-vector
+        self.embedding_dim = 384
+        self.vector_type = "FLOAT32"
 
-        self.model_config = "n_ctx=384,generate_embedding=1"  # See: https://github.com/sqliteai/sqlite-ai/blob/e172b9c9b9d76435be635d1e02c1e88b3681cc6e/src/sqlite-ai.c#L51-L57
+        self.model_config = "n_ctx=384"  # See: https://github.com/sqliteai/sqlite-ai/blob/e172b9c9b9d76435be635d1e02c1e88b3681cc6e/src/sqlite-ai.c#L51-L57
 
         self.chunk_size = 256  # Maximum tokens per chunk
         self.chunk_overlap = 32  # Token overlap between chunks
 
         self.quantize_scan = True  # Whether to quantize the vector for faster search
+

src/sqlite_rag/sqliterag.py

@@ -1,9 +1,9 @@
-import re
 import sqlite3
 from pathlib import Path
 from typing import Optional
 
 from sqlite_rag.logger import Logger
+from sqlite_rag.models.document_result import DocumentResult
 
 from .chunker import Chunker
 from .database import Database
@@ -26,13 +26,19 @@ def __init__(self, settings: Optional[Settings] = None):
         self.settings = settings
         self._logger = Logger()
 
-        self._conn = sqlite3.connect(settings.db_path)
+        self._conn = self._create_db_connection()
 
         self._repository = Repository(self._conn, settings)
         self._chunker = Chunker(self._conn, settings)
         self._engine = Engine(self._conn, settings, chunker=self._chunker)
 
         self.ready = False
+    
+    def _create_db_connection(self) -> sqlite3.Connection:
+        conn = sqlite3.connect(self.settings.db_path)
+        conn.row_factory = sqlite3.Row
+
+        return conn
 
     def _ensure_initialized(self):
         if not self.ready:
@@ -54,9 +60,14 @@ def add(self, path: str, recursive: bool = False) -> int:
         for file_path in files_to_process:
             # TODO: include metadata extraction and mdx options (see our docsearch)
             content = FileReader.parse_file(file_path)
-            document = self._engine.process(
-                Document(content=content, uri=str(file_path))
-            )
+            document = Document(content=content, uri=str(file_path.absolute()))
+            
+            exists = self._repository.document_exists_by_hash(document.hash())
+            if exists:
+                self._logger.info(f"Unchanged document: {file_path}")
+                continue
+
+            document = self._engine.process(document)
 
             self._repository.add_document(document)
             self._logger.info(str(file_path))
@@ -145,7 +156,9 @@ def rebuild(self, remove_missing: bool = False) -> dict:
                     self._repository.add_document(processed_doc)
 
                     reprocessed += 1
-                    self._logger.debug(f"Reprocessed text document: {doc.content[:20]!r}...")
+                    self._logger.debug(
+                        f"Reprocessed text document: {doc.content[:20]!r}..."
+                    )
                 except Exception as e:
                     self._logger.error(f"Error processing text document {doc.id}: {e}")
 
@@ -162,36 +175,34 @@ def rebuild(self, remove_missing: bool = False) -> dict:
     def reset(self) -> bool:
         """Reset/clear the entire database by deleting and recreating it"""
         db_path = self.settings.db_path
-        
+
         try:
             # Close the database connection
             self._conn.close()
-            
+
             # Delete the database file if it exists
             if Path(db_path).exists():
                 Path(db_path).unlink()
                 self._logger.info(f"Deleted database file: {db_path}")
-            
+
             # Recreate the database connection and initialize
-            self._conn = sqlite3.connect(db_path)
-            Database.initialize(self._conn, self.settings)
-            
+            self._conn = self._create_db_connection()
+
             # Reinitialize components with new connection
             self._repository = Repository(self._conn, self.settings)
             self._chunker = Chunker(self._conn, self.settings)
             self._engine = Engine(self._conn, self.settings, chunker=self._chunker)
-            
+
             # Reset ready flag so initialization happens on next use
             self.ready = False
-            
-            self._logger.info("Database reset completed successfully")
+
             return True
-            
+
         except Exception as e:
             self._logger.error(f"Error during database reset: {e}")
             return False
 
-    def search(self, query: str, top_k: int = 10) -> list[Document]:
+    def search(self, query: str, top_k: int = 10) -> list[DocumentResult]:
         """Search for documents matching the query"""
         self._ensure_initialized()