Add advanced index types and release tooling

Author: Anupam

.github/dependabot.yml

@@ -0,0 +1,24 @@
+version: 2
+
+updates:
+  - package-ecosystem: "npm"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 5
+
+  - package-ecosystem: "npm"
+    directory: "/test-install"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 2
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/codeql.yml

@@ -0,0 +1,79 @@
+name: CodeQL
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    branches: [ main, develop ]
+  schedule:
+    - cron: '20 4 * * 1'
+
+permissions:
+  actions: read
+  contents: read
+  security-events: write
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - language: javascript-typescript
+            build-mode: none
+          - language: c-cpp
+            build-mode: manual
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        if: matrix.language == 'c-cpp'
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: 'npm'
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v4
+        with:
+          languages: ${{ matrix.language }}
+          build-mode: ${{ matrix.build-mode }}
+
+      - name: Install native dependencies
+        if: matrix.language == 'c-cpp'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y cmake libopenblas-dev libomp-dev build-essential
+          git clone https://github.com/facebookresearch/faiss.git /tmp/faiss
+          cd /tmp/faiss
+          cmake -B build \
+            -DFAISS_ENABLE_GPU=OFF \
+            -DFAISS_ENABLE_PYTHON=OFF \
+            -DBUILD_TESTING=OFF \
+            -DCMAKE_INSTALL_PREFIX=/usr/local \
+            -DCMAKE_CXX_FLAGS="-fopenmp" \
+            -DCMAKE_C_FLAGS="-fopenmp"
+          cmake --build build -j"$(nproc)"
+          sudo cmake --install build
+          sudo ldconfig
+
+      - name: Install npm dependencies
+        if: matrix.language == 'c-cpp'
+        run: npm ci --ignore-scripts
+
+      - name: Build native addon
+        if: matrix.language == 'c-cpp'
+        run: |
+          export CPPFLAGS="-I/usr/local/include"
+          export CXXFLAGS="-I/usr/local/include"
+          export LDFLAGS="-L/usr/local/lib"
+          npm run build
+
+      - name: Analyze
+        uses: github/codeql-action/analyze@v4
+        with:
+          category: /language:${{ matrix.language }}

Doxyfile

@@ -2,7 +2,7 @@
 # Generated by Anupam Maurya
 
 PROJECT_NAME           = "@faiss-node/native"
-PROJECT_NUMBER         = "0.1.2"
+PROJECT_NUMBER         = "0.1.10"
 PROJECT_BRIEF          = "High-performance Node.js native bindings for Facebook FAISS - the fastest vector similarity search library. Supports FLAT_L2, IVF_FLAT, and HNSW index types with async operations, persistence, and batch search."
 PROJECT_LOGO           = 
 OUTPUT_DIRECTORY       = docs

README.md

@@ -12,7 +12,7 @@ High-performance Node.js native bindings for [Facebook FAISS](https://github.com
 
 - 🚀 **Async Operations** - Non-blocking Promise-based API that never blocks the event loop
 - 🔒 **Thread-Safe** - Mutex-protected concurrent operations for production workloads
-- 📦 **Multiple Index Types** - FLAT_L2, FLAT_IP (cosine similarity), IVF_FLAT, and HNSW with optimized defaults
+- 📦 **Multiple Index Types** - FLAT_L2, FLAT_IP, IVF_FLAT, HNSW, PQ, IVF_PQ, IVF_SQ, plus raw FAISS factory strings for advanced pipelines
 - 💾 **Persistence** - Save/load indexes to disk or serialize to buffers
 - ⚡ **High Performance** - Direct C++ bindings with zero-copy data transfer
 - 📚 **TypeScript Support** - Full type definitions included
@@ -111,15 +111,20 @@ const index = new FaissIndex(config);
 ```
 
 **Parameters:**
-- `config.type` (string, optional): Index type - `'FLAT_L2'`, `'FLAT_IP'`, `'IVF_FLAT'`, or `'HNSW'` (default: `'FLAT_L2'`)
+- `config.type` (string, optional): Index type - `'FLAT_L2'`, `'FLAT_IP'`, `'IVF_FLAT'`, `'HNSW'`, `'PQ'`, `'IVF_PQ'`, or `'IVF_SQ'` (default: `'FLAT_L2'`)
+- `config.factory` (string, optional): Raw FAISS factory string for advanced pipelines such as OPQ, PCA, or PCAR
 - `config.dims` (number, required): Vector dimensions (must be positive integer)
-- `config.nlist` (number, optional): Number of clusters for IVF_FLAT (default: 100)
-- `config.nprobe` (number, optional): Clusters to search for IVF_FLAT (default: 10)
+- `config.metric` (string, optional): Distance metric - `'l2'` or `'ip'` for compatible index types and raw factory indexes
+- `config.nlist` (number, optional): Number of clusters for IVF_FLAT, IVF_PQ, or IVF_SQ (default: 100)
+- `config.nprobe` (number, optional): Clusters to search for IVF_FLAT, IVF_PQ, or IVF_SQ (default: 10)
 - `config.M` (number, optional): Connections per node for HNSW (default: 16)
 - `config.efConstruction` (number, optional): HNSW construction parameter (default: 200)
 - `config.efSearch` (number, optional): HNSW search parameter (default: 50)
+- `config.pqSegments` (number, optional): Number of PQ subquantizers for PQ and IVF_PQ
+- `config.pqBits` (number, optional): Bits per PQ code for PQ and IVF_PQ (default: 8)
+- `config.sqType` (string, optional): Scalar quantizer type for IVF_SQ (default: `'SQ8'`)
 
-Use `nlist` and `nprobe` only with `IVF_FLAT`, and use `M`, `efConstruction`, and `efSearch` only with `HNSW`.
+Use `nlist` and `nprobe` only with `IVF_FLAT`, `IVF_PQ`, or `IVF_SQ`. Use `pqSegments` and `pqBits` only with `PQ` or `IVF_PQ`. Use `M`, `efConstruction`, and `efSearch` only with `HNSW`. Use `factory` by itself for advanced FAISS pipelines, because the topology is encoded directly in the factory string.
 
 **Examples:**
 
@@ -141,6 +146,36 @@ const ivfIndex = new FaissIndex({
 });
 await ivfIndex.train(trainingVectors);  // Must train before adding vectors!
 
+// PQ - Memory efficient quantization without IVF
+const pqIndex = new FaissIndex({
+  type: 'PQ',
+  dims: 768,
+  pqSegments: 48,
+  pqBits: 8
+});
+await pqIndex.train(trainingVectors);
+
+// IVF_PQ - IVF coarse quantization plus PQ compression
+const ivfPqIndex = new FaissIndex({
+  type: 'IVF_PQ',
+  dims: 768,
+  nlist: 100,
+  nprobe: 10,
+  pqSegments: 48,
+  pqBits: 8
+});
+await ivfPqIndex.train(trainingVectors);
+
+// IVF_SQ - IVF with scalar quantization
+const ivfSqIndex = new FaissIndex({
+  type: 'IVF_SQ',
+  dims: 768,
+  nlist: 100,
+  nprobe: 10,
+  sqType: 'SQ8'
+});
+await ivfSqIndex.train(trainingVectors);
+
 // HNSW - State-of-the-art approximate search (best for large datasets)
 const hnswIndex = new FaissIndex({ 
   type: 'HNSW', 
@@ -149,6 +184,18 @@ const hnswIndex = new FaissIndex({
   efConstruction: 200, // Construction parameter
   efSearch: 50        // Search parameter (higher = more accurate, slower)
 });
+
+// Advanced factory string - unlock FAISS preprocessing pipelines
+const customIndex = new FaissIndex({
+  dims: 768,
+  factory: 'PCA256,Flat',
+  metric: 'l2'
+});
+await customIndex.train(trainingVectors);
+
+// You can also pass OPQ / PCAR pipelines directly:
+// 'OPQ48_192,IVF100,PQ48'
+// 'PCAR256,IVF100,PQ48'
 ```
 
 ### Methods

src/cpp/faiss_index.cpp

@@ -6,6 +6,11 @@
 #include <faiss/IndexFlat.h>
 #include <faiss/IndexHNSW.h>
 #include <faiss/IndexIVF.h>
+#include <faiss/IndexIVFPQ.h>
+#include <faiss/IndexPQ.h>
+#include <faiss/IndexPreTransform.h>
+#include <faiss/IndexScalarQuantizer.h>
+#include <faiss/VectorTransform.h>
 #include <faiss/index_factory.h>
 #include <faiss/index_io.h>
 #include <faiss/impl/io.h>
@@ -19,8 +24,95 @@
 #include "faiss_index.h"
 #include <stdexcept>
 
-FaissIndexWrapper::FaissIndexWrapper(int dims, const std::string& indexDescription, int metric) 
-    : dims_(dims), disposed_(false) {
+namespace {
+
+std::string MetricToString(faiss::MetricType metric) {
+    return metric == faiss::METRIC_INNER_PRODUCT ? "ip" : "l2";
+}
+
+std::string InferTransformLabel(const faiss::IndexPreTransform* pretransform) {
+    if (pretransform == nullptr) {
+        return "";
+    }
+
+    for (const auto* transform : pretransform->chain) {
+        if (dynamic_cast<const faiss::OPQMatrix*>(transform) != nullptr) {
+            return "OPQ";
+        }
+
+        const auto* pca = dynamic_cast<const faiss::PCAMatrix*>(transform);
+        if (pca != nullptr) {
+            return pca->random_rotation ? "PCAR" : "PCA";
+        }
+    }
+
+    return "PRETRANSFORM";
+}
+
+std::string InferIndexType(const faiss::Index* index) {
+    if (index == nullptr) {
+        return "UNKNOWN";
+    }
+
+    const auto* pretransform = dynamic_cast<const faiss::IndexPreTransform*>(index);
+    if (pretransform != nullptr) {
+        const std::string transformLabel = InferTransformLabel(pretransform);
+        const std::string innerLabel = InferIndexType(pretransform->index);
+
+        if (transformLabel.empty()) {
+            return innerLabel;
+        }
+
+        if (innerLabel.empty() || innerLabel == "UNKNOWN" || innerLabel == "CUSTOM") {
+            return transformLabel;
+        }
+
+        return transformLabel + "_" + innerLabel;
+    }
+
+    if (dynamic_cast<const faiss::IndexIVFPQ*>(index) != nullptr) {
+        return "IVF_PQ";
+    }
+
+    if (dynamic_cast<const faiss::IndexIVFScalarQuantizer*>(index) != nullptr) {
+        return "IVF_SQ";
+    }
+
+    if (dynamic_cast<const faiss::IndexPQ*>(index) != nullptr) {
+        return "PQ";
+    }
+
+    if (dynamic_cast<const faiss::IndexScalarQuantizer*>(index) != nullptr) {
+        return "SQ";
+    }
+
+    if (dynamic_cast<const faiss::IndexHNSW*>(index) != nullptr) {
+        return "HNSW";
+    }
+
+    if (dynamic_cast<const faiss::IndexIVF*>(index) != nullptr) {
+        return "IVF_FLAT";
+    }
+
+    if (dynamic_cast<const faiss::IndexFlat*>(index) != nullptr) {
+        return index->metric_type == faiss::METRIC_INNER_PRODUCT ? "FLAT_IP" : "FLAT_L2";
+    }
+
+    return "CUSTOM";
+}
+
+}  // namespace
+
+FaissIndexWrapper::FaissIndexWrapper(
+        int dims,
+        const std::string& indexDescription,
+        int metric,
+        const std::string& typeLabel,
+        const std::string& factoryDescription)
+    : dims_(dims),
+      disposed_(false),
+      type_label_(typeLabel),
+      factory_description_(factoryDescription.empty() ? indexDescription : factoryDescription) {
     if (dims <= 0) {
         throw std::invalid_argument("Dimensions must be positive");
     }
@@ -29,6 +121,10 @@ FaissIndexWrapper::FaissIndexWrapper(int dims, const std::string& indexDescripti
     // Examples: "Flat" -> IndexFlatL2, "IVF100,Flat" -> IndexIVFFlat, "HNSW32" -> IndexHNSW
     faiss::MetricType metricType = static_cast<faiss::MetricType>(metric);
     index_ = std::unique_ptr<faiss::Index>(faiss::index_factory(dims, indexDescription.c_str(), metricType));
+
+    if (type_label_.empty()) {
+        type_label_ = InferIndexType(index_.get());
+    }
 }
 
 FaissIndexWrapper::FaissIndexWrapper(int dims) 
@@ -189,20 +285,23 @@ std::string FaissIndexWrapper::GetIndexType() const {
     if (disposed_) {
         return "UNKNOWN";
     }
+    return type_label_.empty() ? InferIndexType(index_.get()) : type_label_;
+}
 
-    if (dynamic_cast<faiss::IndexHNSW*>(index_.get()) != nullptr) {
-        return "HNSW";
-    }
-
-    if (dynamic_cast<faiss::IndexIVF*>(index_.get()) != nullptr) {
-        return "IVF_FLAT";
+std::string FaissIndexWrapper::GetFactoryDescription() const {
+    std::lock_guard<std::mutex> lock(mutex_);
+    if (disposed_) {
+        return "";
     }
+    return factory_description_;
+}
 
-    if (dynamic_cast<faiss::IndexFlat*>(index_.get()) != nullptr) {
-        return index_->metric_type == faiss::METRIC_INNER_PRODUCT ? "FLAT_IP" : "FLAT_L2";
+std::string FaissIndexWrapper::GetMetricName() const {
+    std::lock_guard<std::mutex> lock(mutex_);
+    if (disposed_) {
+        return "l2";
     }
-
-    return "UNKNOWN";
+    return MetricToString(index_->metric_type);
 }
 
 void FaissIndexWrapper::Dispose() {
@@ -244,6 +343,8 @@ std::unique_ptr<FaissIndexWrapper> FaissIndexWrapper::Load(const std::string& fi
         auto wrapper = std::make_unique<FaissIndexWrapper>(loaded_index->d);
         wrapper->index_.reset(loaded_index);
         wrapper->dims_ = loaded_index->d;
+        wrapper->type_label_ = InferIndexType(loaded_index);
+        wrapper->factory_description_.clear();
 
         return wrapper;
     } catch (const std::exception& e) {
@@ -287,6 +388,8 @@ std::unique_ptr<FaissIndexWrapper> FaissIndexWrapper::FromBuffer(const uint8_t*
         auto wrapper = std::make_unique<FaissIndexWrapper>(loaded_index->d);
         wrapper->index_.reset(loaded_index);
         wrapper->dims_ = loaded_index->d;
+        wrapper->type_label_ = InferIndexType(loaded_index);
+        wrapper->factory_description_.clear();
 
         return wrapper;
     } catch (const std::exception& e) {