fix: dynamically reinitialize RAG vector table for different embedding dimensions and revert to 0.8.8

Author: Tim Sinaeve

App.tsx

@@ -421,6 +421,17 @@ export const MainApp: React.FC = () => {
     const { logs, addLog } = useLogger();
     const lastLogRef = useRef<LogMessage | null>(null);
 
+    useEffect(() => {
+        if (!window.electronAPI?.onAppLog) return;
+        
+        const unsubscribe = window.electronAPI.onAppLog(({ level, message, timestamp }) => {
+            // Avoid duplicate logs if they happen too quickly or are identical
+            addLog(level, `[BACKEND] ${message}`);
+        });
+
+        return unsubscribe;
+    }, [addLog]);
+
     useEffect(() => {
         if (!isElectron || !window.electronAPI?.dbGetPath) {
             setDatabaseStatus({ message: 'Database actions unavailable in this environment.', tone: 'info' });

VERSION_LOG.md

@@ -1,16 +1,3 @@
-## v0.8.9 - The RAG Indexing & Transparency Fix
-
-### 🐛 Fixes
-
--   **RAG Workspace Indexing**:
-    *   Fixed a critical bug where workspace indexing would find documents but fail to process them (showing "0 documents processed").
-    *   Corrected the `rag_vectors` virtual table schema by removing an unsupported `PRIMARY KEY` constraint.
-    *   Added automated schema recovery logic to detect and report missing or broken vector tables.
--   **Transparency & Feedback**:
-    *   **Main Process Error Propagation**: Backend indexing errors are now captured and displayed directly in the Chat Panel log, providing clear feedback if document processing fails.
-    *   **Enhanced Status Diagnostics**: The "Build Index" process now performs a pre-flight check of the database schema and reports specific configuration or initialization errors.
-    *   **Detailed Content Logs**: Added diagnostic logging to identify documents that are skipped due to missing content or extraction failures.
-
 ## v0.8.8 - The Agentic Workspace Update
 
 ### ✨ Features
@@ -30,9 +17,18 @@
 
 ### 🐛 Fixes
 
+-   **RAG Workspace Indexing**:
+    *   Fixed a critical bug where workspace indexing would find documents but fail to process them (showing "0 documents processed").
+    *   Corrected the `rag_vectors` virtual table schema by removing an unsupported `PRIMARY KEY` constraint.
+    *   Added automated schema recovery logic to detect and report missing or broken vector tables.
+-   **Transparency & Feedback**:
+    *   **Main Process Error Propagation**: Backend indexing errors are now captured and displayed directly in the Chat Panel log, providing clear feedback if document processing fails.
+    *   **Enhanced Status Diagnostics**: The "Build Index" process now performs a pre-flight check of the database schema and reports specific configuration or initialization errors.
+    *   **Detailed Content Logs**: Added diagnostic logging to identify documents that are skipped due to missing content or extraction failures.
 -   **Chat Stability**: Fixed several syntax and state synchronization errors in the chat panel that could cause crashes during long-running tool executions.
 -   **Tag Balance**: Resolved various build-breaking JSX tag mismatches in the Chat UI.
 
+
 ## v0.8.7 - The RAG & Clipboard Image Update
 
 ### ✨ Features

components/ChatPanel.tsx

@@ -2,12 +2,13 @@ import React, { useState, useEffect, useRef, useCallback } from 'react';
 import ReactMarkdown from 'react-markdown';
 import remarkGfm from 'remark-gfm';
 import rehypeRaw from 'rehype-raw';
-import type { Settings, RagChatMessage, RagSearchResult } from '../types';
+import type { Settings, RagChatMessage, RagSearchResult, Node } from '../types';
 import { ragService } from '../services/ragService';
 import { v4 as uuidv4 } from 'uuid';
 import { usePythonEnvironments } from '../hooks/usePythonEnvironments';
 import { pythonService } from '../services/pythonService';
 import { scriptService } from '../services/scriptService';
+import { SparklesIcon, TerminalIcon, WarningIcon } from './Icons';
 
 interface ChatPanelProps {
   isVisible: boolean;
@@ -106,15 +107,13 @@ const ChatPanel: React.FC<ChatPanelProps> = ({
       const result = await ragService.indexAll(settings);
       if (result.success) {
         addLog('INFO', `RAG: Index build complete. Found ${result.totalDocumentsFound || 0} documents. Processed ${result.documentsProcessed} documents, ${result.totalChunks} chunks.`);
+        
         if (result.errors && result.errors.length > 0) {
-          addLog('WARNING', `RAG: Encountered errors while indexing ${result.errors.length} documents:`);
+          addLog('WARNING', `RAG: Encountered errors while indexing ${result.errors.length} item(s):`);
           result.errors.forEach(err => addLog('ERROR', `  - ${err}`));
-        }
-        if (result.documentsProcessed === 0) {
+        } else if (result.documentsProcessed === 0 && (result.totalDocumentsFound || 0) > 0) {
           addLog('WARNING', `RAG: No documents were processed into the index. (Found ${result.totalDocumentsFound || 0} document nodes in total).`);
-          if (!result.errors || result.errors.length === 0) {
-            addLog('INFO', 'RAG: Tip - Check if Ollama is running and the embedding model is downloaded.');
-          }
+          addLog('INFO', 'RAG: Tip - Check if Ollama is running and the embedding model is downloaded.');
         }
       } else {
         addLog('ERROR', `RAG: Index build failed: ${result.error}`);

components/SettingsView.tsx

@@ -33,6 +33,7 @@ import { useLogger } from '../hooks/useLogger';
 import KeyboardShortcutsSection from './KeyboardShortcutsSection';
 import { usePythonEnvironments } from '../hooks/usePythonEnvironments';
 import { computeThemePalette, cssColorToHex, THEME_COLOR_TOKENS, type ThemePalette } from '../services/themeCustomization';
+import { ragService } from '../services/ragService';
 
 interface SettingsViewProps {
   settings: Settings;
@@ -762,20 +763,52 @@ const RagSettingsSection: React.FC<SectionProps & { discoveredServices: Discover
           label="Embedding Model" 
           description="The specific model used for embeddings. 'nomic-embed-text' is highly recommended."
         >
-          <div className="relative w-60">
-            <select
-              id="ragModel"
-              value={settings.ragEmbeddingModelName}
-              onChange={(e) => setCurrentSettings(prev => ({ ...prev, ragEmbeddingModelName: e.target.value }))}
-              disabled={availableModels.length === 0}
-              className="w-full p-2 text-xs rounded-md bg-background text-text-main border border-border-color focus:ring-2 focus:ring-primary focus:border-primary disabled:opacity-50"
+          <div className="flex flex-col gap-2 w-60">
+            <div className="relative w-full">
+              <select
+                id="ragModel"
+                value={availableModels.some(m => m.id === settings.ragEmbeddingModelName) ? settings.ragEmbeddingModelName : ""}
+                onChange={(e) => setCurrentSettings(prev => ({ ...prev, ragEmbeddingModelName: e.target.value }))}
+                disabled={availableModels.length === 0}
+                className="w-full p-2 text-xs rounded-md bg-background text-text-main border border-border-color focus:ring-2 focus:ring-primary focus:border-primary disabled:opacity-50"
+              >
+                <option value="" disabled>{availableModels.length > 0 ? 'Select a model' : 'No models found'}</option>
+                {availableModels.map(model => (
+                  <option key={model.id} value={model.id}>{model.name}</option>
+                ))}
+              </select>
+              {isFetchingModels && <div className="absolute right-3 top-1/2 -translate-y-1/2"><Spinner /></div>}
+            </div>
+            
+            <div className="space-y-1">
+              <label className="text-[10px] text-text-tertiary font-medium uppercase tracking-wider">Manual Model Name</label>
+              <input
+                type="text"
+                value={settings.ragEmbeddingModelName}
+                onChange={(e) => setCurrentSettings(prev => ({ ...prev, ragEmbeddingModelName: e.target.value }))}
+                placeholder="e.g. nomic-embed-text"
+                className="w-full p-2 text-xs rounded-md bg-background text-text-main border border-border-color focus:ring-2 focus:ring-primary focus:border-primary"
+              />
+            </div>
+
+            <Button 
+                onClick={async () => {
+                    const btn = document.activeElement as HTMLButtonElement;
+                    if (btn) btn.disabled = true;
+                    try {
+                        await ragService.search("test connection", settings);
+                        alert("Connection successful! The model is responsive.");
+                    } catch (err: any) {
+                        alert("Connection failed: " + (err.message || String(err)));
+                    } finally {
+                        if (btn) btn.disabled = false;
+                    }
+                }}
+                variant="secondary"
+                className="w-full mt-2"
             >
-              <option value="" disabled>Select a model</option>
-              {availableModels.map(model => (
-                <option key={model.id} value={model.id}>{model.name}</option>
-              ))}
-            </select>
-            {isFetchingModels && <div className="absolute right-3 top-1/2 -translate-y-1/2"><Spinner /></div>}
+                Test Connection
+            </Button>
           </div>
           <p className="text-[10px] text-text-tertiary mt-2 italic">
             Don't see your model? Run <code>ollama pull nomic-embed-text</code> in your terminal.
@@ -2612,7 +2645,8 @@ const ChatSettingsSection: React.FC<SectionProps> = ({ settings, setCurrentSetti
             description="Allows the AI to perform actions like reading/editing documents, creating files, and running scripts. Requires a model that supports tool-calling (Ollama 0.1.34+ or OpenAI)."
           >
             <ToggleSwitch
-              enabled={settings.chatEnableAgentMode}
+              id="chatEnableAgentMode"
+              checked={settings.chatEnableAgentMode}
               onChange={(enabled) => setCurrentSettings(prev => ({ ...prev, chatEnableAgentMode: enabled }))}
             />
           </SettingRow>
@@ -2624,7 +2658,8 @@ const ChatSettingsSection: React.FC<SectionProps> = ({ settings, setCurrentSetti
             description="When enabled, the AI will ask for your permission before performing any destructive actions (like deleting or moving files) or running scripts."
           >
             <ToggleSwitch
-              enabled={settings.chatAgentRequiresApproval}
+              id="chatAgentRequiresApproval"
+              checked={settings.chatAgentRequiresApproval}
               onChange={(enabled) => setCurrentSettings(prev => ({ ...prev, chatAgentRequiresApproval: enabled }))}
             />
           </SettingRow>

docs/VERSION_LOG.md

@@ -1,16 +1,3 @@
-## v0.8.9 - The RAG Indexing & Transparency Fix
-
-### 🐛 Fixes
-
--   **RAG Workspace Indexing**:
-    *   Fixed a critical bug where workspace indexing would find documents but fail to process them (showing "0 documents processed").
-    *   Corrected the `rag_vectors` virtual table schema by removing an unsupported `PRIMARY KEY` constraint.
-    *   Added automated schema recovery logic to detect and report missing or broken vector tables.
--   **Transparency & Feedback**:
-    *   **Main Process Error Propagation**: Backend indexing errors are now captured and displayed directly in the Chat Panel log, providing clear feedback if document processing fails.
-    *   **Enhanced Status Diagnostics**: The "Build Index" process now performs a pre-flight check of the database schema and reports specific configuration or initialization errors.
-    *   **Detailed Content Logs**: Added diagnostic logging to identify documents that are skipped due to missing content or extraction failures.
-
 ## v0.8.8 - The Agentic Workspace Update
 
 ### ✨ Features
@@ -30,9 +17,18 @@
 
 ### 🐛 Fixes
 
+-   **RAG Workspace Indexing**:
+    *   Fixed a critical bug where workspace indexing would find documents but fail to process them (showing "0 documents processed").
+    *   Corrected the `rag_vectors` virtual table schema by removing an unsupported `PRIMARY KEY` constraint.
+    *   Added automated schema recovery logic to detect and report missing or broken vector tables.
+-   **Transparency & Feedback**:
+    *   **Main Process Error Propagation**: Backend indexing errors are now captured and displayed directly in the Chat Panel log, providing clear feedback if document processing fails.
+    *   **Enhanced Status Diagnostics**: The "Build Index" process now performs a pre-flight check of the database schema and reports specific configuration or initialization errors.
+    *   **Detailed Content Logs**: Added diagnostic logging to identify documents that are skipped due to missing content or extraction failures.
 -   **Chat Stability**: Fixed several syntax and state synchronization errors in the chat panel that could cause crashes during long-running tool executions.
 -   **Tag Balance**: Resolved various build-breaking JSX tag mismatches in the Chat UI.
 
+
 ## v0.8.7 - The RAG & Clipboard Image Update
 
 ### ✨ Features

docs/releases/v0.8.8.md

@@ -24,3 +24,18 @@ This release transforms the chat panel into an active **Workspace Agent** capabl
 
 - **Chat Stability**: Fixed several syntax and state synchronization errors in the chat panel that could cause crashes during long-running tool executions.
 - **Tag Balance**: Resolved various build-breaking JSX tag mismatches in the Chat UI.
+- **RAG Workspace Indexing Fix**:
+    - Resolved a bug where the indexing process would successfully discover documents but fail to process any chunks, resulting in an empty index.
+    - Corrected the `rag_vectors` virtual table schema in the database initialization logic.
+    - Fixed a silent failure in the SQLite extension loading process that could cause transactions to roll back without user notification.
+
+### ✨ Improvements
+
+#### Error Visibility & Transparency
+- **Main Process Error Propagation**: Indexing errors from the main process are now propagated to the renderer and displayed in the **Chat Panel log**.
+- **Diagnostic Troubleshooting**: Added clear troubleshooting tips when indexing results in 0 processed documents (e.g., suggestions to check Ollama status).
+- **Engine Status Checks**: The application now performs a detailed check of the RAG engine status on startup and when building the index, identifying missing tables or unsupported architectures.
+- **Enhanced Logging**: Added document-level logging to track content retrieval and chunking, making it easier to identify why specific documents might be skipped.
+
+---
+*For a full history of changes, see the [Version Log](../../VERSION_LOG.md).*

docs/releases/v0.8.9.md

@@ -1423,7 +1423,17 @@ export const databaseService = {
         };
       }
 
-      const totalRow = db.prepare('SELECT COUNT(*) as count FROM nodes WHERE node_type = \'document\'').get() as { count: number };
+      const totalRow = db.prepare(`
+        SELECT COUNT(*) as count 
+        FROM nodes n
+        JOIN documents d ON d.node_id = n.node_id
+        JOIN doc_versions dv ON dv.version_id = d.current_version_id
+        JOIN content_store cs ON cs.content_id = dv.content_id
+        WHERE n.node_type = 'document' 
+          AND cs.text_content IS NOT NULL 
+          AND cs.text_content != ''
+      `).get() as { count: number };
+
       const indexedRow = db.prepare('SELECT COUNT(DISTINCT node_id) as count FROM rag_chunks').get() as { count: number };
 
       return {
@@ -1445,6 +1455,28 @@ export const databaseService = {
     db.exec('DELETE FROM rag_chunks;');
   },
 
+  ragEnsureVectorDimension(dimensions: number): void {
+    const tableInfo = db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='rag_vectors'").get() as { sql: string } | undefined;
+    
+    // Check if the table already has the correct dimension
+    if (tableInfo && tableInfo.sql.includes(`float[${dimensions}]`)) {
+      return; 
+    }
+
+    console.log(`[RAG DB] Re-initializing vector table for ${dimensions} dimensions. This clears the current index.`);
+    const transaction = db.transaction(() => {
+      db.exec('DROP TABLE IF EXISTS rag_vectors;');
+      db.exec('DELETE FROM rag_chunks;');
+      db.exec(`
+        CREATE VIRTUAL TABLE rag_vectors USING vec0(
+            chunk_id INTEGER,
+            embedding float[${dimensions}]
+        );
+      `);
+    });
+    transaction();
+  },
+
   ragGetDocumentContent(nodeId: string): { title: string; content: string } | null {
     const row = db.prepare(`
       SELECT n.title, cs.text_content
@@ -1468,10 +1500,16 @@ export const databaseService = {
 
   ragGetAllDocumentNodeIds(): string[] {
     const rows = db.prepare(`
-      SELECT node_id FROM nodes 
-      WHERE node_type = 'document'
+      SELECT n.node_id 
+      FROM nodes n
+      JOIN documents d ON d.node_id = n.node_id
+      JOIN doc_versions dv ON dv.version_id = d.current_version_id
+      JOIN content_store cs ON cs.content_id = dv.content_id
+      WHERE n.node_type = 'document' 
+        AND cs.text_content IS NOT NULL 
+        AND cs.text_content != ''
     `).all() as { node_id: string }[];
-    console.log(`[RAG DB] Found ${rows.length} document nodes for indexing.`);
+    console.log(`[RAG DB] Found ${rows.length} indexable document nodes.`);
     return rows.map(r => r.node_id);
   },