feat: add local reranker, FTS metadata enrichment, and retrieval audit log

Add Ollama-based local rerank provider for private deployments with
automatic fallback to API reranker. Support any OpenAI-compatible
embedding endpoint via env config. Extend chunks_fts with language and
symbols columns for enriched lexical search. Make RRF fusion weights
env-configurable. Add per-query retrieval audit logging to SQLite.
Add git post-commit diff parser for automatic incremental indexing.
All 478 tests pass (22 new).

Author: codefromkarl

docs/plans/2026-04-14-retrieval-precision-optimization.md

@@ -1,6 +1,11 @@
 /**
  * Embedding 客户端
  *
+ * 支持任何 OpenAI-compatible Embedding API：
+ * - SiliconFlow (默认): EMBEDDINGS_BASE_URL 指向 SiliconFlow 端点
+ * - Ollama 本地: EMBEDDINGS_BASE_URL=http://localhost:11434/v1/embeddings
+ * - vLLM / LocalAI: EMBEDDINGS_BASE_URL 指向对应 /v1/embeddings 端点
+ *
  * 调用 SiliconFlow Embedding API，将文本转换为向量
  * 支持并发控制、批量处理和智能速率限制
  *

src/api/embedding.ts

@@ -0,0 +1,193 @@
+/**
+ * Local Reranker Client — Ollama chat-based reranking
+ *
+ * Uses a local Ollama model to rerank code snippets by relevance.
+ * Falls back to equal scoring if the model output cannot be parsed.
+ */
+
+import { logger } from '../utils/logger.js';
+import type { RerankedDocument, RerankDetailedResult, RerankOptions } from './reranker.js';
+
+export interface LocalRerankerConfig {
+  baseUrl: string;
+  model: string;
+  /** Max tokens for model response (default 512) */
+  maxTokens?: number;
+  /** Request timeout in ms (default 30000) */
+  timeoutMs?: number;
+}
+
+const RERANK_SYSTEM_PROMPT = `You are a code relevance judge. Given a query and a list of code snippets, return ONLY a JSON array of indices ordered by relevance (most relevant first). Example: [2, 0, 1]. No explanation, no markdown, just the array.`;
+
+export class LocalRerankerClient {
+  private config: Required<LocalRerankerConfig>;
+
+  constructor(config: LocalRerankerConfig) {
+    this.config = {
+      maxTokens: 512,
+      timeoutMs: 30000,
+      ...config,
+    };
+  }
+
+  async rerankDetailed(
+    query: string,
+    documents: string[],
+  ): Promise<RerankDetailedResult> {
+    if (documents.length === 0) {
+      return { results: [] };
+    }
+
+    const numberedDocs = documents
+      .map((doc, i) => `[${i}] ${doc}`)
+      .join('\n\n');
+
+    const userPrompt = `Query: ${query}\n\nSnippets:\n${numberedDocs}`;
+
+    const controller = new AbortController();
+    const timeout = setTimeout(
+      () => controller.abort(),
+      this.config.timeoutMs,
+    );
+
+    try {
+      const response = await fetch(`${this.config.baseUrl}/api/chat`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          model: this.config.model,
+          messages: [
+            { role: 'system', content: RERANK_SYSTEM_PROMPT },
+            { role: 'user', content: userPrompt },
+          ],
+          stream: false,
+          options: { num_predict: this.config.maxTokens },
+        }),
+        signal: controller.signal,
+      });
+
+      if (!response.ok) {
+        throw new Error(`Ollama rerank HTTP ${response.status}`);
+      }
+
+      const data = (await response.json()) as {
+        message: { content: string };
+      };
+      const indices = this.parseIndexArray(data.message.content, documents.length);
+
+      if (indices.length === 0) {
+        // Model failed to produce valid ranking — return all with score 0
+        logger.warn('Local rerank produced no valid indices, returning equal scores');
+        return {
+          results: documents.map((text, i) => ({
+            originalIndex: i,
+            score: 0,
+            text,
+          })),
+        };
+      }
+
+      const topN = indices.length;
+      const results: RerankedDocument[] = indices.map(
+        (originalIndex, rank) => ({
+          originalIndex,
+          score: 1 - rank / Math.max(topN, 1),
+          text: documents[originalIndex],
+        }),
+      );
+
+      // Include documents not in the ranked list with score 0
+      const rankedSet = new Set(indices);
+      for (let i = 0; i < documents.length; i++) {
+        if (!rankedSet.has(i)) {
+          results.push({
+            originalIndex: i,
+            score: 0,
+            text: documents[i],
+          });
+        }
+      }
+
+      return { results };
+    } catch (err) {
+      logger.error({ error: err }, 'Local rerank failed, returning equal scores');
+      return {
+        results: documents.map((text, i) => ({
+          originalIndex: i,
+          score: 0,
+          text,
+        })),
+      };
+    } finally {
+      clearTimeout(timeout);
+    }
+  }
+
+  /**
+   * Parse model output as an index array.
+   * Tries direct JSON parse first, then extracts first [...]+ pattern.
+   */
+  /**
+   * Rerank with attached metadata (matches RerankerClient.rerankWithDataDetailed contract)
+   */
+  async rerankWithDataDetailed<T>(
+    query: string,
+    items: T[],
+    textExtractor: (item: T) => string,
+    _options?: RerankOptions,
+  ): Promise<RerankDetailedResult<T>> {
+    if (items.length === 0) {
+      return { results: [] };
+    }
+
+    const texts = items.map(textExtractor);
+    const textResult = await this.rerankDetailed(query, texts);
+
+    return {
+      results: textResult.results.map((result) => ({
+        ...result,
+        data: items[result.originalIndex],
+      })),
+    };
+  }
+
+  private parseIndexArray(content: string, maxIndex: number): number[] {
+    const cleaned = content.trim();
+
+    // Attempt 1: direct parse
+    try {
+      const arr = JSON.parse(cleaned);
+      if (Array.isArray(arr) && arr.every((n) => typeof n === 'number')) {
+        return this.validateIndices(arr as number[], maxIndex);
+      }
+    } catch {
+      // not valid JSON
+    }
+
+    // Attempt 2: extract first [..] block
+    const match = cleaned.match(/\[[\d\s,]+\]/);
+    if (match) {
+      try {
+        const arr = JSON.parse(match[0]) as number[];
+        return this.validateIndices(arr, maxIndex);
+      } catch {
+        // extraction failed
+      }
+    }
+
+    logger.warn({ content: cleaned.slice(0, 200) }, 'Failed to parse rerank output');
+    return [];
+  }
+
+  private validateIndices(indices: number[], maxIndex: number): number[] {
+    const valid = new Set<number>();
+    const result: number[] = [];
+    for (const idx of indices) {
+      if (Number.isInteger(idx) && idx >= 0 && idx < maxIndex && !valid.has(idx)) {
+        valid.add(idx);
+        result.push(idx);
+      }
+    }
+    return result;
+  }
+}

src/api/localReranker.ts

@@ -0,0 +1,44 @@
+/**
+ * Reranker Router — selects API or local reranker based on config
+ */
+
+import { getRerankProvider, getOllamaRerankerConfig } from '../config.js';
+import { LocalRerankerClient } from './localReranker.js';
+import { RerankerClient } from './reranker.js';
+import type { RerankDetailedResult, RerankOptions } from './reranker.js';
+
+export type RerankerBackend = {
+  rerankDetailed(
+    query: string,
+    documents: string[],
+    options?: RerankOptions,
+  ): Promise<RerankDetailedResult>;
+
+  rerankWithDataDetailed<T>(
+    query: string,
+    items: T[],
+    textExtractor: (item: T) => string,
+    options?: RerankOptions,
+  ): Promise<RerankDetailedResult<T>>;
+};
+
+let cachedBackend: RerankerBackend | null = null;
+
+export function getRerankerBackend(): RerankerBackend {
+  if (cachedBackend) return cachedBackend;
+
+  const provider = getRerankProvider();
+  if (provider === 'ollama') {
+    cachedBackend = new LocalRerankerClient(getOllamaRerankerConfig()) as RerankerBackend;
+  } else {
+    cachedBackend = new RerankerClient() as RerankerBackend;
+  }
+  return cachedBackend;
+}
+
+/**
+ * Reset cached backend (for testing)
+ */
+export function resetRerankerBackend(): void {
+  cachedBackend = null;
+}

src/api/rerankerRouter.ts

@@ -82,6 +82,30 @@ export interface RerankerConfig {
   topN: number;
 }
 
+export type RerankProvider = 'api' | 'ollama';
+
+export interface OllamaRerankerConfig {
+  baseUrl: string;
+  model: string;
+  maxTokens: number;
+  timeoutMs: number;
+}
+
+export function getRerankProvider(): RerankProvider {
+  const provider = process.env.RERANK_PROVIDER;
+  if (provider === 'ollama') return 'ollama';
+  return 'api';
+}
+
+export function getOllamaRerankerConfig(): OllamaRerankerConfig {
+  return {
+    baseUrl: process.env.OLLAMA_BASE_URL || 'http://localhost:11434',
+    model: process.env.OLLAMA_RERANK_MODEL || 'qwen2.5-coder:7b',
+    maxTokens: parseInt(process.env.OLLAMA_RERANK_MAX_TOKENS || '512', 10),
+    timeoutMs: parseInt(process.env.OLLAMA_RERANK_TIMEOUT_MS || '30000', 10),
+  };
+}
+
 export interface IndexUpdateStrategyConfig {
   churnThreshold: number;
   costThresholdRatio: number;
@@ -176,7 +200,7 @@ export function getEmbeddingConfig(): EmbeddingConfig {
   const globalMinIntervalMs = parseInt(process.env.EMBEDDINGS_GLOBAL_MIN_INTERVAL_MS || '200', 10);
 
   if (!apiKey) {
-    throw new Error('EMBEDDINGS_API_KEY 环境变量未设置');
+    throw new Error('EMBEDDINGS_API_KEY 环境变量未设置（本地 Ollama 可设置为 "ollama"）');
   }
   if (!baseUrl) {
     throw new Error('EMBEDDINGS_BASE_URL 环境变量未设置');