chore: release v0.8.9 - fix RAG indexing and enhance transparency

Author: Tim Sinaeve

VERSION_LOG.md

@@ -1,4 +1,17 @@
-## v0.8.8 - The Agentic Workspace Update
+## v0.8.9 - The RAG Indexing & Transparency Fix
+1: 
+2: ### 🐛 Fixes
+3: 
+4: -   **RAG Workspace Indexing**:
+5:     *   Fixed a critical bug where workspace indexing would find documents but fail to process them (showing "0 documents processed").
+6:     *   Corrected the `rag_vectors` virtual table schema by removing an unsupported `PRIMARY KEY` constraint.
+7:     *   Added automated schema recovery logic to detect and report missing or broken vector tables.
+8: -   **Transparency & Feedback**:
+9:     *   **Main Process Error Propagation**: Backend indexing errors are now captured and displayed directly in the Chat Panel log, providing clear feedback if document processing fails.
+10:     *   **Enhanced Status Diagnostics**: The "Build Index" process now performs a pre-flight check of the database schema and reports specific configuration or initialization errors.
+11:     *   **Detailed Content Logs**: Added diagnostic logging to identify documents that are skipped due to missing content or extraction failures.
+12: 
+13: ## v0.8.8 - The Agentic Workspace Update
 
 ### ✨ Features
 

components/ChatPanel.tsx

@@ -106,8 +106,15 @@ const ChatPanel: React.FC<ChatPanelProps> = ({
       const result = await ragService.indexAll(settings);
       if (result.success) {
         addLog('INFO', `RAG: Index build complete. Found ${result.totalDocumentsFound || 0} documents. Processed ${result.documentsProcessed} documents, ${result.totalChunks} chunks.`);
+        if (result.errors && result.errors.length > 0) {
+          addLog('WARNING', `RAG: Encountered errors while indexing ${result.errors.length} documents:`);
+          result.errors.forEach(err => addLog('ERROR', `  - ${err}`));
+        }
         if (result.documentsProcessed === 0) {
           addLog('WARNING', `RAG: No documents were processed into the index. (Found ${result.totalDocumentsFound || 0} document nodes in total).`);
+          if (!result.errors || result.errors.length === 0) {
+            addLog('INFO', 'RAG: Tip - Check if Ollama is running and the embedding model is downloaded.');
+          }
         }
       } else {
         addLog('ERROR', `RAG: Index build failed: ${result.error}`);

docs/VERSION_LOG.md

@@ -1,4 +1,17 @@
-## v0.8.8 - The Agentic Workspace Update
+## v0.8.9 - The RAG Indexing & Transparency Fix
+1: 
+2: ### 🐛 Fixes
+3: 
+4: -   **RAG Workspace Indexing**:
+5:     *   Fixed a critical bug where workspace indexing would find documents but fail to process them (showing "0 documents processed").
+6:     *   Corrected the `rag_vectors` virtual table schema by removing an unsupported `PRIMARY KEY` constraint.
+7:     *   Added automated schema recovery logic to detect and report missing or broken vector tables.
+8: -   **Transparency & Feedback**:
+9:     *   **Main Process Error Propagation**: Backend indexing errors are now captured and displayed directly in the Chat Panel log, providing clear feedback if document processing fails.
+10:     *   **Enhanced Status Diagnostics**: The "Build Index" process now performs a pre-flight check of the database schema and reports specific configuration or initialization errors.
+11:     *   **Detailed Content Logs**: Added diagnostic logging to identify documents that are skipped due to missing content or extraction failures.
+12: 
+13: ## v0.8.8 - The Agentic Workspace Update
 
 ### ✨ Features
 

docs/releases/v0.8.9.md

@@ -0,0 +1,25 @@
+# DocForge v0.8.9 Release Notes
+
+## 🏷️ The RAG Indexing & Transparency Fix
+
+This release focuses on resolving critical stability issues in the RAG (Retrieval-Augmented Generation) workspace indexing engine and improving the transparency of background operations.
+
+### 🐛 Bug Fixes
+
+-   **RAG Workspace Indexing Fix**:
+    -   Resolved a bug where the indexing process would successfully discover documents but fail to process any chunks, resulting in an empty index.
+    -   Corrected the `rag_vectors` virtual table schema in the database initialization logic.
+    -   Fixed a silent failure in the SQLite extension loading process that could cause transactions to roll back without user notification.
+
+### ✨ Improvements
+
+-   **Error Visibility**:
+    -   Indexing errors from the main process are now propagated to the renderer and displayed in the **Chat Panel log**.
+    -   Added clear troubleshooting tips when indexing results in 0 processed documents (e.g., suggestions to check Ollama status).
+-   **Diagnostic Diagnostics**:
+    -   The application now performs a detailed check of the RAG engine status on startup and when building the index, identifying missing tables or unsupported architectures.
+-   **Enhanced Logging**:
+    -   Added document-level logging to track content retrieval and chunking, making it easier to identify why specific documents might be skipped.
+
+---
+*For a full history of changes, see the [Version Log](../../VERSION_LOG.md).*

electron/database.ts

@@ -1414,6 +1414,15 @@ export const databaseService = {
         };
       }
 
+      // Check if rag_vectors table exists
+      const tableCheck = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='rag_vectors'").get();
+      if (!tableCheck) {
+        return {
+          success: false,
+          error: 'The rag_vectors table is missing. This usually happens if the database was initialized with an invalid schema. You may need to reset your database if this persists.',
+        };
+      }
+
       const totalRow = db.prepare('SELECT COUNT(*) as count FROM nodes WHERE node_type = \'document\'').get() as { count: number };
       const indexedRow = db.prepare('SELECT COUNT(DISTINCT node_id) as count FROM rag_chunks').get() as { count: number };
 
@@ -1446,7 +1455,14 @@ export const databaseService = {
       WHERE n.node_id = ? AND n.node_type = 'document'
     `).get(nodeId) as { title: string; text_content: string } | undefined;
 
-    if (!row || !row.text_content) return null;
+    if (!row) {
+      console.log(`[RAG DB] Document entry not found for node: ${nodeId}`);
+      return null;
+    }
+    if (!row.text_content) {
+      console.log(`[RAG DB] Document "${row.title}" (${nodeId}) has no text content in store (it might be empty or a different type).`);
+      return null;
+    }
     return { title: row.title, content: row.text_content };
   },
 

electron/embeddingService.ts

@@ -192,12 +192,13 @@ export const embeddingService = {
     ollamaBaseUrl: string,
     modelName: string,
     onProgress?: (current: number, total: number) => void
-  ): Promise<{ documentsProcessed: number; totalChunks: number; totalDocumentsFound: number }> {
+  ): Promise<{ documentsProcessed: number; totalChunks: number; totalDocumentsFound: number; errors?: string[] }> {
     const nodeIds = databaseService.ragGetAllDocumentNodeIds();
     console.log(`[RAG] Starting full index of ${nodeIds.length} documents.`);
     let totalChunks = 0;
     let documentsProcessed = 0;
     const totalDocumentsFound = nodeIds.length;
+    const errors: string[] = [];
 
     if (nodeIds.length === 0) {
       console.log('[RAG] No documents found to index.');
@@ -208,14 +209,19 @@ export const embeddingService = {
         const result = await this.indexDocument(nodeIds[i], ollamaBaseUrl, modelName);
         totalChunks += result.chunksCreated;
         if (result.chunksCreated > 0) documentsProcessed++;
-      } catch (error) {
-        console.error(`[RAG] Failed to index document ${nodeIds[i]}:`, error);
+      } catch (error: any) {
+        const errorMsg = error instanceof Error ? error.message : String(error);
+        // Try to get document title for the error message
+        const doc = databaseService.ragGetDocumentContent(nodeIds[i]);
+        const title = doc ? doc.title : nodeIds[i];
+        console.error(`[RAG] Failed to index document "${title}":`, errorMsg);
+        errors.push(`"${title}": ${errorMsg}`);
       }
       onProgress?.(i + 1, nodeIds.length);
     }
 
     console.log(`[RAG] Full index complete. Processed ${documentsProcessed} documents, created ${totalChunks} total chunks.`);
-    return { documentsProcessed, totalChunks, totalDocumentsFound };
+    return { documentsProcessed, totalChunks, totalDocumentsFound, errors: errors.length > 0 ? errors : undefined };
   },
 
   /**

electron/schema.ts

@@ -2,7 +2,7 @@
 // removing the need for external .sql files which can complicate the build process.
 
 export const INITIAL_SCHEMA = `
--- PRAGMA user_version is set to 7 by the database service for this schema.
+-- PRAGMA user_version is set to 9 by the database service for this schema.
 
 -- =================================================================
 --  CORE HIERARCHY & METADATA
@@ -237,7 +237,7 @@ CREATE INDEX idx_rag_chunks_node ON rag_chunks(node_id);
 -- sqlite-vec virtual table for vector similarity search
 -- nomic-embed-text produces 768-dimensional vectors
 CREATE VIRTUAL TABLE IF NOT EXISTS rag_vectors USING vec0(
-    chunk_id INTEGER PRIMARY KEY,
+    chunk_id INTEGER,
     embedding float[768]
 );
 `;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "docforge",
-  "version": "0.8.8",
+  "version": "0.8.9",
   "description": "An application to manage and refine documents.",
   "main": "dist/main.js",
   "scripts": {

types.ts

@@ -622,6 +622,7 @@ export interface RagIndexResponse {
   totalChunks: number;
   totalDocumentsFound?: number;
   error?: string;
+  errors?: string[];
 }
 
 export interface AgentToolCall {