Implement efficient indexing pipeline

- Remove auto-indexing of mount root on boot
- Add restore routine for previously-added repos
- Batch Ollama embeddings via /api/embed endpoint
- Add embedding_status schema with background backfill
- Implement bounded embedding queue with backpressure
- Batch graph writes (nodes + edges per file)
- Decouple embeddings into background worker

Structural index now usable immediately after parsing.
Semantic embeddings backfill in background and resume on restart.
Bounded CPU/RAM via queue depth limits and batching.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: 2 people

src/cli/index.ts

@@ -101,7 +101,7 @@ program
           })
         : undefined;
 
-      const indexer = new Indexer(storage, graph, workspaceRoot, embeddingQueue, vectorStore);
+      const indexer = new Indexer(storage, graph, workspaceRoot);
       const watcher = new FileWatcher(indexer, { debounceMs: 200 });
 
       // Handle watcher errors gracefully
@@ -198,15 +198,11 @@ program
       const dataDir = resolve(options.dataDir);
       const absolutePath = resolve(repoPath);
 
-      const { loadConfig } = await import('../core/config.js');
       const { SqliteStorageProvider } = await import('../store/sqlite.js');
       const { LanceDBVectorStore } = await import('../store/lance.js');
       const { CodeGraph } = await import('../core/graph.js');
       const { Indexer } = await import('../core/indexer.js');
-      const { createEmbeddingProvider } = await import('../llm/provider.js');
-      const { EmbeddingQueue } = await import('../core/embedding-queue.js');
 
-      const config = loadConfig();
       const dbPath = resolve(dataDir, 'context-simplo.db');
       const lanceDbPath = resolve(dataDir, 'lancedb');
 
@@ -218,21 +214,7 @@ program
 
       const graph = new CodeGraph();
 
-      const embeddingProvider = await createEmbeddingProvider(config.llmProvider.value, {
-        apiKey: config.llmApiKey.value,
-        baseUrl: config.llmBaseUrl.value,
-        model: config.llmEmbeddingModel.value,
-      });
-
-      const embeddingQueue = config.llmProvider.value !== 'none'
-        ? new EmbeddingQueue(embeddingProvider, {
-            concurrency: config.embeddingConcurrency.value,
-            batchSize: config.embeddingBatchSize.value,
-            maxRetries: 3,
-          })
-        : undefined;
-
-      const indexer = new Indexer(storage, graph, process.cwd(), embeddingQueue, vectorStore);
+      const indexer = new Indexer(storage, graph, process.cwd());
 
       const job = await indexer.indexRepository(absolutePath, {
         incremental: options.incremental,

src/core/indexer.ts

@@ -27,7 +27,7 @@
  * Security: Integrates with SecretScrubber before indexing.
  */
 
-import { readdir, stat, readFile } from 'fs/promises';
+import { readdir, stat } from 'fs/promises';
 import { resolve, relative, basename, dirname, join } from 'path';
 import { createHash } from 'crypto';
 import { parseFile, type ParsedFile } from './parser.js';
@@ -41,8 +41,6 @@ import type {
 import { ParseError } from './errors.js';
 import { EventEmitter } from 'events';
 import { ContextIgnore } from '../security/ignore.js';
-import type { EmbeddingQueue } from './embedding-queue.js';
-import type { LanceDBVectorStore } from '../store/lance.js';
 import type { ParsePool } from './parse-pool.js';
 
 export interface IndexerOptions {
@@ -73,8 +71,6 @@ export class Indexer extends EventEmitter {
     public storage: StorageProvider,
     public graph: CodeGraphApi,
     private workspaceRoot: string = '/workspace',
-    private embeddingQueue?: EmbeddingQueue,
-    private vectorStore?: LanceDBVectorStore,
     private memoryGuard?: any,
     private parsePool?: ParsePool
   ) {

src/index.ts

@@ -157,7 +157,7 @@ async function main() {
     console.log('Parse pool disabled (PARSE_WORKER_POOL_SIZE=0)');
   }
 
-  const indexer = new Indexer(storage, graph, workspaceRoot, embeddingQueue, vectorStore, memoryGuard, parsePool);
+  const indexer = new Indexer(storage, graph, workspaceRoot, memoryGuard, parsePool);
   console.log('Indexer ready');
 
   const indexQueue = new IndexQueue({

tests/unit/mcp-handlers.test.ts

@@ -39,12 +39,11 @@ describe('MCP Handlers', () => {
     await storage.initialize();
 
     const graph = new CodeGraph();
-    const indexer = new Indexer({
+    const indexer = new Indexer(
       storage,
       graph,
-      workspaceRoot,
-      maxConcurrency: 1,
-    });
+      workspaceRoot
+    );
 
     const symbolicSearch = new SymbolicSearch(storage);