upgrade faiss 1.14.1+, commit cb69d7c5665241d1e84f476105ffb6fd6c7c192c

thirdparty/faiss/.github/actions/build_cmake/action.yml

@@ -40,7 +40,7 @@ runs:
       uses: conda-incubator/setup-miniconda@v3
       with:
         python-version: '3.12'
-        miniforge-version: latest # ensures conda-forge channel is used.
+        miniforge-version: latest
         channels: conda-forge
         conda-remove-defaults: 'true'
         # Set to aarch64 if we're on arm64 because there's no miniforge ARM64 package, just aarch64.
@@ -50,18 +50,15 @@ runs:
       if: inputs.metal != 'ON'
       shell: bash
       run: |
-        # initialize Conda
-        conda config --set solver libmamba
         # Ensure starting packages are from conda-forge.
         conda list --show-channel-urls
-        conda install -y -q "conda<=25.07"
         echo "$CONDA/bin" >> $GITHUB_PATH
 
         conda install -y -q python=3.12 cmake=3.30.4 make=4.2 swig=4.0 "numpy>=2.0,<3.0" scipy=1.16 pytest=7.4 gflags=2.2 setuptools
 
         # install base packages for ARM64
         if [ "${{ runner.arch }}" = "ARM64" ]; then
-          conda install -y -q -c conda-forge openblas=0.3.29 gxx_linux-aarch64=14.2 sysroot_linux-aarch64=2.17
+          conda install -y -q -c conda-forge openblas=0.3.33 gxx_linux-aarch64=14.2 sysroot_linux-aarch64=2.17
         fi
 
         # install base packages for X86_64
@@ -79,7 +76,11 @@ runs:
           conda install -y -q cuda-libraries-dev=12.6 cuda-nvcc=12.6 cuda-nvtx=12.6 cuda-cupti=12.6 cuda-cudart-dev=12.6 gxx_linux-64=12.4 -c "nvidia/label/cuda-12.6"
         # and CUDA from cuVS channel for cuVS builds
         elif [ "${{ inputs.cuvs }}" = "ON" ]; then
-          conda install -y -q libcuvs=26.02 'cuda-version=12.9' cuda-toolkit=12.9 sysroot_linux-64=2.34 -c rapidsai -c rapidsai-nightly -c conda-forge
+          conda install -y -q libcuvs=26.02 'cuda-version=12.9' sysroot_linux-64=2.34 -c rapidsai -c rapidsai-nightly -c conda-forge
+          # Clean index cache to prevent sqlite3 "database is locked" errors
+          # from conda-libmamba-solver's shards cache between consecutive installs.
+          # See: https://github.com/conda/conda-libmamba-solver/issues/667
+          conda clean --index-cache 2>/dev/null || true
         fi
 
         # install SVS runtime for SVS builds
@@ -241,6 +242,7 @@ runs:
       shell: bash
       run: |
         conda list --show-channel-urls
+        conda install -y -q "pip<26"
         pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm6.1
     - name: Python tests (CPU only)
       if: inputs.gpu == 'OFF' && inputs.metal != 'ON'

thirdparty/faiss/.github/actions/build_conda/action.yml

@@ -30,7 +30,7 @@ runs:
       uses: conda-incubator/setup-miniconda@v3
       with:
         python-version: '3.12'
-        miniforge-version: latest # ensures conda-forge channel is used.
+        miniforge-version: latest
         channels: conda-forge
         conda-remove-defaults: 'true'
         activate-environment: 'base'
@@ -81,7 +81,7 @@ runs:
       working-directory: conda
       run: |
         conda list --show-channel-urls
-        conda build faiss-gpu --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
+        conda build faiss-gpu --python 3.12 --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
             -c pytorch -c nvidia/label/cuda-${{ inputs.cuda }} -c nvidia
     - name: Conda build (GPU) w/ anaconda upload
       if: inputs.label != '' && inputs.cuda != '' && inputs.cuvs == ''
@@ -99,7 +99,7 @@ runs:
       working-directory: conda
       run: |
         conda list --show-channel-urls
-        conda build faiss-gpu-cuvs --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
+        conda build faiss-gpu-cuvs --python 3.12 --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
             -c pytorch -c rapidsai -c rapidsai-nightly -c conda-forge -c nvidia
     - name: Conda build (GPU w/ cuVS) w/ anaconda upload
       if: inputs.label != '' && inputs.cuda != '' && inputs.cuvs != ''

thirdparty/faiss/.github/workflows/build-pull-request.yml

@@ -302,6 +302,109 @@ jobs:
         uses: ./.github/actions/build_cmake
         with:
           svs: ON
+  linux-riscv64-DD-cmake:
+    name: Linux riscv64 Dynamic Dispatch cross-compile (cmake)
+    needs: linux-x86_64-cmake
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up riscv64 cross-compilation environment
+        run: |
+          sudo dpkg --add-architecture riscv64
+
+          # Restrict existing apt sources to amd64/i386 so that apt does not
+          # try (and fail) to fetch riscv64 packages from the main Ubuntu
+          # mirror, which does not carry riscv64.
+          if [ -f /etc/apt/sources.list.d/ubuntu.sources ]; then
+            # Ubuntu 24.04+ deb822 format: insert Architectures field.
+            sudo sed -i '/^Types: deb$/a Architectures: amd64 i386' \
+              /etc/apt/sources.list.d/ubuntu.sources
+          fi
+          if grep -q '^deb http' /etc/apt/sources.list 2>/dev/null; then
+            # Ubuntu 22.04 traditional .list format.
+            sudo sed -i 's|^deb http|deb [arch=amd64,i386] http|g' \
+              /etc/apt/sources.list
+          fi
+
+          # Add Ubuntu Ports as the riscv64 package source.
+          CODENAME=$(. /etc/os-release && echo "$VERSION_CODENAME")
+          echo "deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports ${CODENAME} main restricted universe" \
+            | sudo tee    /etc/apt/sources.list.d/riscv64-ports.list
+          echo "deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports ${CODENAME}-updates main restricted universe" \
+            | sudo tee -a /etc/apt/sources.list.d/riscv64-ports.list
+
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq \
+            cmake \
+            gcc-riscv64-linux-gnu \
+            g++-riscv64-linux-gnu \
+            libopenblas-dev:riscv64 \
+            libgomp1:riscv64 \
+            qemu-user-static
+
+          # Bridge Ubuntu's multiarch library directory into the sysroot so
+          # that CMake's find_library() (ONLY mode, root=/usr/riscv64-linux-gnu)
+          # can reach packages installed as :riscv64.
+          # CMake searches ${root}/usr/lib/<multiarch-tuple>/ thanks to the
+          # compiler's -print-multiarch output (riscv64-linux-gnu).
+          sudo mkdir -p /usr/riscv64-linux-gnu/usr/lib
+          sudo ln -sfn /usr/lib/riscv64-linux-gnu \
+            /usr/riscv64-linux-gnu/usr/lib/riscv64-linux-gnu
+
+      - name: Setup ccache
+        uses: hendrikmuhs/ccache-action@v1
+        with:
+          key: linux-riscv64-dd
+          max-size: 2G
+          update-package-index: true
+
+      - name: Configure (cmake)
+        run: |
+          cmake -B build \
+            -DCMAKE_TOOLCHAIN_FILE=cmake/toolchains/riscv64-linux-gnu.cmake \
+            -DBUILD_TESTING=ON \
+            -DBUILD_SHARED_LIBS=ON \
+            -DFAISS_ENABLE_GPU=OFF \
+            -DFAISS_OPT_LEVEL=dd \
+            -DFAISS_ENABLE_PYTHON=OFF \
+            -DFAISS_ENABLE_C_API=OFF \
+            -DBLA_VENDOR=OpenBLAS \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+            -DCMAKE_BUILD_RPATH=/usr/lib/riscv64-linux-gnu \
+            .
+
+      - name: Build faiss_test (includes test_factory_tools)
+        run: cmake --build build --target faiss_test -j$(nproc)
+
+      - name: Verify binary with ldd (via qemu)
+        run: |
+          file build/tests/faiss_test
+          file build/tests/faiss_test | grep -q "RISC-V" || \
+            { echo "ERROR: not a RISC-V binary"; exit 1; }
+
+          # Expose riscv64 libs inside the qemu sysroot so the riscv64 ld.so
+          # launched by qemu can resolve NEEDED entries.
+          sudo mkdir -p /usr/riscv64-linux-gnu/usr/lib
+          sudo ln -sfn /usr/lib/riscv64-linux-gnu \
+            /usr/riscv64-linux-gnu/usr/lib/riscv64-linux-gnu
+
+          # Invoke the riscv64 dynamic linker under qemu with --list, which
+          # is exactly what ldd does on native hardware.
+          RISCV64_LD=$(find /usr/riscv64-linux-gnu/lib \
+            -name "ld-linux-riscv64-*.so.1" | head -1)
+          qemu-riscv64-static -L /usr/riscv64-linux-gnu \
+            "$RISCV64_LD" --list build/tests/faiss_test
+
+      - name: Run test_simd_levels (via QEMU)
+        run: |
+          qemu-riscv64-static -L /usr/riscv64-linux-gnu \
+            build/tests/faiss_test \
+            --gtest_filter="SIMDConfig.*:SIMDLevel.*:CompileOptions.*"
+
   index-io-backward-compatibility:
     needs: linux-x86_64-cmake
     name: Index serialization backward compatibility

thirdparty/faiss/.github/workflows/index-io-backward-compatibility.yml

@@ -52,7 +52,7 @@ jobs:
           eval "$(conda shell.bash hook)"
           conda create -n faiss_conda_read -y python=3.12
           conda activate faiss_conda_read
-          conda install -y -c pytorch -c conda-forge faiss-cpu=1.14.1
+          conda install -y -c pytorch -c conda-forge faiss-cpu=1.14.1 "mkl>=2024.2.2,<2026"
           conda list
 
       - name: Run Conda reader (read Faiss index and verify)
@@ -89,7 +89,7 @@ jobs:
           eval "$(conda shell.bash hook)"
           conda create -n faiss_conda_write -y python=3.12
           conda activate faiss_conda_write
-          conda install -y -c pytorch -c conda-forge faiss-cpu=1.14.1
+          conda install -y -c pytorch -c conda-forge faiss-cpu=1.14.1 "mkl>=2024.2.2,<2026"
           conda list
 
       - name: Create shared data directory

thirdparty/faiss/CMakeLists.txt

@@ -127,7 +127,7 @@ endif()
 include(CTest)
 if(BUILD_TESTING)
   add_subdirectory(tests)
-  if(NOT WIN32)
+  if(NOT WIN32 AND NOT CMAKE_CROSSCOMPILING)
     add_subdirectory(perf_tests)
   endif()
   if(FAISS_ENABLE_GPU)

thirdparty/faiss/benchs/bench_fw/optimize.py

@@ -275,7 +275,7 @@ def optimize_codec(
             pareto_metric=ParetoMetric.TIME_SPACE,
         )
         results = [
-            factory[r] for r in set(v["factory"] for _, _, _, k, v in filtered)
+            factory[r] for r in {v["factory"] for _, _, _, k, v in filtered}
         ]
         return results
 

thirdparty/faiss/benchs/bench_super_kmeans.py

@@ -0,0 +1,112 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Benchmark `faiss.SuperKMeans` against vanilla `faiss.Clustering` on the same
+input. Inputs may be a .fvecs file or a synthetic Gaussian mixture.
+
+Examples:
+    # Synthetic data
+    python bench_super_kmeans.py --n 10000 --d 128 --k 64 --niter 10
+
+    # Real .fvecs (e.g., SIFT base)
+    python bench_super_kmeans.py --fvecs /path/to/sift_base.fvecs --k 1024 \\
+        --niter 10
+"""
+
+import argparse
+import sys
+import time
+
+import faiss
+import numpy as np
+from datasets import fvecs_read
+
+
+def gaussian_mixture(n, d, k, seed):
+    """k centers in [-1, 1]^d, ~n/k samples per center from N(center, 0.1)."""
+    rng = np.random.RandomState(seed)
+    centers = rng.uniform(-1.0, 1.0, size=(k, d)).astype("float32")
+    cluster = np.arange(n) % k
+    noise = rng.normal(0.0, 0.1, size=(n, d)).astype("float32")
+    return centers[cluster] + noise
+
+
+def run_vanilla(x, d, k, niter, seed):
+    quant = faiss.IndexFlatL2(d)
+    cl = faiss.Clustering(d, k)
+    cl.seed = seed
+    cl.niter = niter
+    cl.verbose = False
+    t0 = time.perf_counter()
+    cl.train(x, quant)
+    dt = time.perf_counter() - t0
+    final_obj = cl.iteration_stats.at(cl.iteration_stats.size() - 1).obj
+    return dt, final_obj, cl.iteration_stats.size()
+
+
+def run_super(x, d, k, niter, seed):
+    p = faiss.SuperKMeansParameters()
+    p.seed = seed
+    p.niter = niter
+    p.verbose = False
+    sc = faiss.SuperKMeans(d, k, p)
+    t0 = time.perf_counter()
+    sc.train(x)
+    dt = time.perf_counter() - t0
+    final_obj = sc.iteration_stats.at(sc.iteration_stats.size() - 1).obj
+    iter_sum = sum(
+        sc.iteration_stats.at(i).time for i in range(sc.iteration_stats.size())
+    )
+    setup = dt - iter_sum
+    rates = faiss.vector_to_array(sc.gemm_pruning_rates)
+    return dt, final_obj, sc.iteration_stats.size(), iter_sum, setup, rates
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__)
+    src = ap.add_mutually_exclusive_group()
+    src.add_argument("--fvecs", help="path to a .fvecs file")
+    ap.add_argument("--n", type=int, default=10000, help="synthetic n")
+    ap.add_argument("--d", type=int, default=128, help="synthetic d")
+    ap.add_argument("--k", type=int, default=64, help="number of centroids")
+    ap.add_argument("--niter", type=int, default=10, help="iterations")
+    ap.add_argument("--seed", type=int, default=42, help="RNG seed")
+    args = ap.parse_args()
+
+    if args.fvecs:
+        x = fvecs_read(args.fvecs)
+        n, d = x.shape
+        print("=== bench_super_kmeans ===")
+        print(
+            f"fvecs={args.fvecs} n={n} d={d} k={args.k} "
+            f"niter={args.niter} seed={args.seed}"
+        )
+    else:
+        n, d = args.n, args.d
+        x = gaussian_mixture(n, d, args.k, args.seed)
+        print("=== bench_super_kmeans ===")
+        print(
+            f"n={n} d={d} k={args.k} niter={args.niter} seed={args.seed}"
+        )
+
+    print("\n--- faiss.Clustering ---")
+    v_dt, v_obj, v_iters = run_vanilla(x, d, args.k, args.niter, args.seed)
+    print(f"  wall_time={v_dt:.3f}s final_obj={v_obj:g} iters={v_iters}")
+
+    print("\n--- faiss.SuperKMeans ---")
+    s_dt, s_obj, s_iters, iter_sum, setup, rates = run_super(
+        x, d, args.k, args.niter, args.seed
+    )
+    print(
+        f"  wall_time={s_dt:.3f}s final_obj={s_obj:g} iters={s_iters} "
+        f"iter_sum={iter_sum:.3f}s setup={setup:.3f}s"
+    )
+    print("  gemm_pruning_rates:", " ".join(f"{r:.3f}" for r in rates))
+    print()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

thirdparty/faiss/cmake/toolchains/riscv64-linux-gnu.cmake

@@ -0,0 +1,33 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+# Cross-compilation toolchain for RISC-V 64-bit (lp64d ABI) on Ubuntu/Debian.
+#
+# Requires these packages installed on the build host:
+#   gcc-riscv64-linux-gnu  g++-riscv64-linux-gnu
+#
+# Target libraries (e.g. libopenblas-dev:riscv64) are installed via apt
+# multiarch to /usr/lib/riscv64-linux-gnu/.  CMake's ONLY find-root mode
+# searches ${CMAKE_FIND_ROOT_PATH}/usr/lib/riscv64-linux-gnu/ (via the
+# compiler's multiarch tuple), so the CI script creates a symlink:
+#   /usr/riscv64-linux-gnu/usr/lib/riscv64-linux-gnu
+#     -> /usr/lib/riscv64-linux-gnu
+# before invoking cmake.
+
+set(CMAKE_SYSTEM_NAME Linux)
+set(CMAKE_SYSTEM_PROCESSOR riscv64)
+
+set(CMAKE_C_COMPILER   riscv64-linux-gnu-gcc)
+set(CMAKE_CXX_COMPILER riscv64-linux-gnu-g++)
+
+# Cross-compiler sysroot provided by gcc-riscv64-linux-gnu.
+set(CMAKE_FIND_ROOT_PATH /usr/riscv64-linux-gnu)
+
+# Never look for host-side tools (cmake, python, …) inside the sysroot.
+set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
+# Look for target libraries/headers/packages only inside the sysroot.
+set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
+set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
+set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)