Replace PQ with IVF-PQ

Author: platypii

README.md

@@ -66,7 +66,7 @@ await writeVectors({
   normalize: true,    // L2-normalize on write; lets search skip sqrt for cosine
   binary: true,       // also write 1-bit-per-dim sign column for binary+rerank search
   clusters: 128,      // k-means clusters for phase-1 pruning (implies binary: true)
-  pq: true,           // optional product-quantized codes for approximate scoring before rerank
+  pq: true,           // optional IVF-PQ index for approximate scoring before rerank
   vectors: myEmbedder(), // any sync or async iterable of { id, vector }
 })
 ```
@@ -181,7 +181,7 @@ Core columns: `id` (STRING), `vector` (`FIXED_LEN_BYTE_ARRAY(4 × dim)`, raw flo
 
 A `cachedAsyncBuffer` deduplicates footer / offset-index byte ranges across all the parallel `parquetRead` calls.
 
-**PQ + rerank path** (`algorithm: 'pq'`, or `auto` when a file has PQ but no binary column): scan compact `vector_pq` codes over the selected cluster ranges, approximate-score candidates with lookup tables built from the query and stored PQ codebooks, then fetch full float32 vectors only for the candidate pool and exact-rerank as above. When `clusters > 0`, PQ uses the same contiguous cluster row ranges as the binary path.
+**IVF-PQ + rerank path** (`algorithm: 'pq'`, or `auto` when a file has PQ but no binary column): rank stored float IVF centroids against the query, scan compact residual `vector_pq` codes over the selected IVF row groups, approximate-score candidates with lookup tables built from the query, IVF centroid, and residual PQ codebooks, then fetch full float32 vectors only for the candidate pool and exact-rerank as above. IVF-PQ uses its own row ordering and should not be combined with binary `clusters`.
 
 For pre-normalized vectors with `metric: 'cosine'`, the search normalizes the query once and scores via dot product to skip the per-candidate sqrt loop.
 
@@ -208,9 +208,13 @@ Key-value metadata:
 | `hypvector.clusters` | number of k-means clusters (0 if not clustered) |
 | `hypvector.centroids` | base64-encoded centroid binary codes (`clusters × dim/8` bytes); present when `clusters > 0` |
 | `hypvector.clusterCounts` | base64-encoded `Uint32Array` of per-cluster row counts; present when `clusters > 0` |
+| `hypvector.pq.mode` | `ivf`; present when `pq: true` |
 | `hypvector.pq.segments` | number of PQ sub-vectors / bytes per code; present when `pq: true` |
 | `hypvector.pq.centroids` | centroids per PQ sub-vector; present when `pq: true` |
-| `hypvector.pq.codebooks` | base64-encoded `Float32Array` codebooks (`pq.centroids × dim` floats); present when `pq: true` |
+| `hypvector.pq.codebooks` | base64-encoded residual `Float32Array` codebooks (`pq.centroids × dim` floats); present when `pq: true` |
+| `hypvector.ivf.clusters` | number of non-empty IVF lists; present when `pq: true` |
+| `hypvector.ivf.centroids` | base64-encoded float IVF centroids (`ivf.clusters × dim` float32 values); present when `pq: true` |
+| `hypvector.ivf.counts` | base64-encoded `Uint32Array` of per-IVF-list row counts; present when `pq: true` |
 
 ### CLI
 

bin/inspect.js

@@ -26,6 +26,7 @@ export async function inspect({ path }) {
   if (meta.hasPq) {
     console.log(`PQ segments: ${meta.pqSegments}`)
     console.log(`PQ centroids: ${meta.pqCentroids}`)
+    console.log(`IVF clusters: ${meta.ivfClusters}`)
   }
   console.log(`Row groups: ${metadata.row_groups.length.toLocaleString()}`)
   console.log(`Raw float32 size: ${rawSize.toLocaleString()} bytes`)

scripts/ablation.js

@@ -7,7 +7,7 @@
  *   A) base               vector + id only  (search must use exact full scan)
  *   B) +binary            adds vector_bin column (binary phase 1 + per-cand phase 2 reads)
  *   C) +cluster           B plus k-means clustering + centroids/counts KV
- *   D) +PQ                C plus vector_pq column + PQ codebooks
+ *   D) IVF-PQ             vector_pq column + IVF centroids + residual PQ codebooks
  *
  * Page size is held at 32 KB for B-D so we isolate the feature contribution
  * from the page-size knob.
@@ -41,7 +41,7 @@ const variants = [
   { name: 'A_base', label: 'A) base (vec only)', opts: { binary: false } },
   { name: 'B_binary', label: 'B) +binary', opts: { binary: true } },
   { name: 'C_cluster', label: 'C) +cluster', opts: { binary: true, clusters: 128 } },
-  { name: 'D_pq', label: 'D) +cluster+PQ', opts: { binary: true, clusters: 128, pq: true }, search: { algorithm: 'pq' } },
+  { name: 'D_ivfpq', label: 'D) IVF-PQ', opts: { pq: true, ivfClusters: 128 }, search: { algorithm: 'pq' } },
 ]
 
 for (const v of variants) {

src/constants.js

@@ -33,9 +33,14 @@ export const defaultClusterIterations = 6
 // file has cluster metadata. Lower = faster but lower recall.
 export const defaultClusterProbeFraction = 0.25
 
-// Default product quantization settings. The initial PQ path stores one
-// code byte per segment, with values in [0, defaultPqCentroids).
+// Default residual product quantization settings. The IVF-PQ path stores
+// one code byte per segment, with values in [0, defaultPqCentroids).
 export const defaultPqSegments = 32
-export const defaultPqCentroids = 16
+export const defaultPqCentroids = 64
 export const defaultPqIterations = 8
 export const defaultPqSampleSize = 4096
+
+// Default IVF coarse quantizer settings for the IVF-PQ path.
+export const defaultIvfClusters = 128
+export const defaultIvfIterations = 6
+export const defaultIvfSampleSize = 4096