feat(tier2): Reciprocal Rank Fusion for retrieval (default-on, A/B-confirmed)

RRF fuses BM25 + vector + backlink as per-list RANK positions (Sigma w/(K+rank)),
replacing weighted-sum, in both the SQLite (_searchDb) and PG (search) paths.
Rank-based fusion lets the backlink signal surface a heavily-referenced real
answer above a keyword-stuffed decoy: A/B on a hub-recall corpus had weighted
rank the decoy #1 and the hub #2, while RRF flips them (hub #1).

- config: ZC_RETRIEVAL_FUSION (default rrf; =weighted is the rollback kill-switch),
  RRF_K=60, per-list weights (RRF_W_BM25/VEC/BACKLINK). Backlink list gated on
  W_BACKLINK>0 so that stays the backlink kill-switch in both modes.
- weighted path left byte-identical; full suite 865/868 (3 pre-existing fixtures).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: some-random-agent

src/config.ts

@@ -70,6 +70,22 @@ export const Config = {
   W_BACKLINK:        parseFloat(env["ZC_W_BACKLINK"] ?? "0.08"),
   BACKLINK_LOG_BASE: parseInt(env["ZC_BACKLINK_LOG_BASE"] ?? "10", 10),
 
+  // ── Retrieval fusion (Tier-2 #3) ─────────────────────────────────────────
+  // How the BM25 + vector (+ backlink) candidate lists are combined.
+  //   "weighted" — historical weighted-sum (W_BM25·bm25 + W_COSINE·cosine) plus the
+  //                additive log-damped backlink boost. Byte-identical to v0.31.0.
+  //   "rrf"      — Reciprocal Rank Fusion over per-list RANK positions
+  //                (Σ w/(K+rank)); rank-based, scale-free, robust to score skew.
+  // Default "rrf" (v0.31.x): A/B on a hub-recall corpus showed RRF surfaces the
+  // heavily-referenced real answer above a keyword-stuffed decoy where weighted-sum
+  // fails. Set ZC_RETRIEVAL_FUSION=weighted to roll back. Backlink folds in as a third
+  // RRF list (kept off when W_BACKLINK=0, so that flag remains the backlink kill-switch).
+  RETRIEVAL_FUSION:  (env["ZC_RETRIEVAL_FUSION"] ?? "rrf") as "weighted" | "rrf",
+  RRF_K:             parseInt(env["ZC_RRF_K"] ?? "60", 10),
+  RRF_W_BM25:        parseFloat(env["ZC_RRF_W_BM25"] ?? "1.0"),
+  RRF_W_VEC:         parseFloat(env["ZC_RRF_W_VEC"] ?? "1.0"),
+  RRF_W_BACKLINK:    parseFloat(env["ZC_RRF_W_BACKLINK"] ?? "0.25"),
+
   // ── Fetch / SSRF ──────────────────────────────────────────────────────────
   FETCH_LIMIT:       parseInt(env["ZC_FETCH_LIMIT"] ?? "50", 10),
   FETCH_TIMEOUT_MS:  15_000,

src/knowledge.ts

@@ -403,6 +403,33 @@ function _searchDb(
     } catch { /* table absent on a pre-migration DB — leave map empty */ }
   }
 
+  // Tier-2 #3: Reciprocal Rank Fusion. Precompute per-list RANK positions (1-indexed)
+  // ONLY in RRF mode — the weighted path below is left byte-identical. Backlink is folded
+  // in as a third list, gated on W_BACKLINK>0 so that flag stays the backlink kill-switch.
+  const useRRF = Config.RETRIEVAL_FUSION === "rrf";
+  let bm25RankMap: Map<string, number> | null = null;
+  let cosRankMap:  Map<string, number> | null = null;
+  let blRankMap:   Map<string, number> | null = null;
+  if (useRRF) {
+    const entries = Array.from(candidateMap.entries());
+    // BM25: lower FTS5 rank = more relevant.
+    bm25RankMap = new Map([...entries].sort((a, b) => a[1].rank - b[1].rank).map(([s], i) => [s, i + 1]));
+    // Vector: higher cosine = more relevant (only when a query embedding exists).
+    if (queryVector) {
+      const withCos = entries.map(([s]) => {
+        const v = embeddingMap.get(s);
+        return { s, cos: v ? cosineSimilarity(queryVector, v) : -Infinity };
+      }).sort((a, b) => b.cos - a.cos);
+      cosRankMap = new Map(withCos.map((x, i) => [x.s, i + 1]));
+    }
+    // Backlink: higher weighted_in = stronger hub (only sources with inbound links).
+    if (Config.W_BACKLINK > 0 && backlinkMap.size > 0) {
+      const withBl = entries.map(([s]) => ({ s, w: backlinkMap.get(s) ?? 0 }))
+        .filter((x) => x.w > 0).sort((a, b) => b.w - a.w);
+      blRankMap = new Map(withBl.map((x, i) => [x.s, i + 1]));
+    }
+  }
+
   const ranks     = Array.from(candidateMap.values()).map((r) => r.rank);
   const minRank   = Math.min(...ranks);
   const maxRank   = Math.max(...ranks);
@@ -428,7 +455,21 @@ function _searchDb(
     const blBoost = wIn > 0
       ? Config.W_BACKLINK * (Math.log(1 + wIn) / Math.log(1 + Config.BACKLINK_LOG_BASE))
       : 0;
-    const hybridScore = baseScore + blBoost;
+    // Tier-2 #3: RRF fuses per-list RANK positions (scale-free, robust to score skew);
+    // the weighted path fuses normalized scores + additive boost (byte-identical to v0.31.0).
+    let hybridScore: number;
+    if (useRRF) {
+      const K   = Config.RRF_K;
+      const br  = bm25RankMap!.get(source);
+      const cr  = cosRankMap?.get(source);
+      const blr = blRankMap?.get(source);
+      hybridScore =
+        (br  ? Config.RRF_W_BM25     / (K + br)  : 0) +
+        (cr  ? Config.RRF_W_VEC      / (K + cr)  : 0) +
+        (blr ? Config.RRF_W_BACKLINK / (K + blr) : 0);
+    } else {
+      hybridScore = baseScore + blBoost;
+    }
 
     const firstTerm  = queries[0]?.toLowerCase().split(" ")[0] ?? "";
     const idx        = row.content.toLowerCase().indexOf(firstTerm);

src/store-postgres.ts

@@ -519,16 +519,47 @@ export class PostgresStore implements Store {
         const embMap = new Map(embRes.rows.map(r => [r.source, r.vector]));
         const maxBm25 = Math.max(...bm25Res.rows.map(r => r.rank), 1);
 
-        const scored = bm25Res.rows.map(row => {
-          const bm25Norm = row.rank / maxBm25;
-          let cosScore   = 0;
+        // Compute cosine for every BM25 candidate up front (needed by both fusion modes).
+        const withCos = bm25Res.rows.map(row => {
+          let cosScore = 0;
           const storedVecStr = embMap.get(row.source);
           if (storedVecStr) {
             // Parse pgvector "[x1,x2,...,xN]" string back to Float32Array
             const nums = storedVecStr.slice(1, -1).split(",").map(Number);
             cosScore   = cosineSimilarity(new Float32Array(nums), qEmbed.vector);
           }
-          const hybrid = Config.W_BM25 * bm25Norm + Config.W_COSINE * cosScore + blBoost(row.source);
+          return { row, cosScore };
+        });
+
+        // Tier-2 #3: RRF fuses per-list RANK positions (scale-free); weighted fuses
+        // normalized scores + additive backlink boost (byte-identical to v0.31.0).
+        const useRRF = Config.RETRIEVAL_FUSION === "rrf";
+        // BM25 rank = position in bm25Res.rows (already ORDER BY rank DESC).
+        const bm25RankMap = new Map(bm25Res.rows.map((r, i) => [r.source, i + 1]));
+        let cosRankMap: Map<string, number> | null = null;
+        let blRankMap:  Map<string, number> | null = null;
+        if (useRRF) {
+          cosRankMap = new Map([...withCos].sort((a, b) => b.cosScore - a.cosScore).map((x, i) => [x.row.source, i + 1]));
+          if (Config.W_BACKLINK > 0 && blMap.size > 0) {
+            blRankMap = new Map(bm25Res.rows.map(r => ({ s: r.source, w: blMap.get(r.source) ?? 0 }))
+              .filter(x => x.w > 0).sort((a, b) => b.w - a.w).map((x, i) => [x.s, i + 1]));
+          }
+        }
+
+        const scored = withCos.map(({ row, cosScore }) => {
+          let hybrid: number;
+          if (useRRF) {
+            const K   = Config.RRF_K;
+            const br  = bm25RankMap.get(row.source);
+            const cr  = cosRankMap?.get(row.source);
+            const blr = blRankMap?.get(row.source);
+            hybrid =
+              (br  ? Config.RRF_W_BM25     / (K + br)  : 0) +
+              (cr  ? Config.RRF_W_VEC      / (K + cr)  : 0) +
+              (blr ? Config.RRF_W_BACKLINK / (K + blr) : 0);
+          } else {
+            hybrid = Config.W_BM25 * (row.rank / maxBm25) + Config.W_COSINE * cosScore + blBoost(row.source);
+          }
           return { ...row, vectorScore: cosScore, hybridScore: hybrid };
         });