upgrade faiss to 1.14.1+, commit 967eda65dec2515ccd18c5abe464b485f8a883d0 (#1618)

Signed-off-by: Alexandr Guzhva <alexanderguzhva@gmail.com>

Author: alexanderguzhva

cmake/libs/libfaiss.cmake

@@ -40,14 +40,14 @@ knowhere_file_glob(
 # AVX512 vanilla Faiss dynamic dispatch related files. Baseline
 # sq-avx512.cpp is replaced by a knowhere-local prelude file that declares
 # a fast DCTemplate specialization for QT_4bit_uniform + L2 and then
-# textually #includes the baseline sq-avx512.cpp — see
+# textually #includes the baseline sq-avx512.cpp \u2014 see
 # cppcontrib/knowhere/impl/sq-avx512-fastpath.cpp for the full design note.
 knowhere_file_glob(
   GLOB
   FAISS_DD_AVX512_SRCS
   thirdparty/faiss/faiss/impl/fast_scan/impl-avx512.cpp
   thirdparty/faiss/faiss/impl/hnsw/avx512.cpp
-  thirdparty/faiss/faiss/impl/pq_code_distance/pq_code_distance-avx512.cpp
+  thirdparty/faiss/faiss/impl/pq_code_distance/avx512.cpp
   thirdparty/faiss/faiss/impl/binary_hamming/avx512.cpp
   thirdparty/faiss/faiss/cppcontrib/knowhere/impl/sq-avx512-fastpath.cpp
   thirdparty/faiss/faiss/utils/distances_fused/avx512.cpp
@@ -85,7 +85,7 @@ knowhere_file_glob(
   thirdparty/faiss/faiss/impl/approx_topk/avx2.cpp
   thirdparty/faiss/faiss/impl/fast_scan/impl-avx2.cpp
   thirdparty/faiss/faiss/impl/hnsw/avx2.cpp
-  thirdparty/faiss/faiss/impl/pq_code_distance/pq_code_distance-avx2.cpp
+  thirdparty/faiss/faiss/impl/pq_code_distance/avx2.cpp
   thirdparty/faiss/faiss/impl/binary_hamming/avx2.cpp
   thirdparty/faiss/faiss/cppcontrib/knowhere/impl/sq-avx2-fastpath.cpp
   thirdparty/faiss/faiss/utils/distances_fused/simdlib_based.cpp
@@ -138,6 +138,7 @@ knowhere_file_glob(
   thirdparty/faiss/faiss/impl/approx_topk/neon.cpp
   thirdparty/faiss/faiss/impl/fast_scan/impl-neon.cpp
   thirdparty/faiss/faiss/impl/binary_hamming/neon.cpp
+  thirdparty/faiss/faiss/impl/pq_code_distance/neon.cpp
   thirdparty/faiss/faiss/cppcontrib/knowhere/impl/sq-neon-fastpath.cpp
   thirdparty/faiss/faiss/utils/distances_fused/simdlib_based_neon.cpp
   thirdparty/faiss/faiss/utils/hamming_distance/hamming_neon.cpp
@@ -186,6 +187,7 @@ knowhere_file_glob(
 knowhere_file_glob(
   GLOB
   FAISS_DD_RVV_SRCS
+  thirdparty/faiss/faiss/impl/pq_code_distance/rvv.cpp
   thirdparty/faiss/faiss/impl/scalar_quantizer/sq-rvv.cpp
   thirdparty/faiss/faiss/impl/binary_hamming/rvv.cpp
   thirdparty/faiss/faiss/utils/simd_impl/distances_rvv.cpp

thirdparty/faiss/.github/actions/build_conda/action.yml

@@ -33,6 +33,7 @@ runs:
         miniforge-version: latest # ensures conda-forge channel is used.
         channels: conda-forge
         conda-remove-defaults: 'true'
+        activate-environment: 'base'
         # Set to runner.arch=aarch64 if we're on arm64 because
         # there's no miniforge ARM64 package, just aarch64.
         # They are the same thing, just named differently.
@@ -43,7 +44,7 @@ runs:
       run: |
         # Ensure starting packages are from conda-forge.
         conda list --show-channel-urls
-        conda install -y -q "conda!=24.11.0,<=25.07"
+        conda install -y -q "conda!=24.11.0,<25.7"
         conda install -y -q "conda-build=25.3.1" "liblief=0.14.1"
     - name: Enable anaconda uploads
       if: inputs.label != ''

thirdparty/faiss/.github/workflows/build-pip.yml

@@ -48,10 +48,11 @@ jobs:
       id-token: write
       contents: read
     steps:
-      - name: Download all artifacts
+      - name: Download wheel artifacts
         uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e  # v4.1.7
         with:
           path: dist
+          pattern: wheels-*
           merge-multiple: true
 
       - name: Publish to PyPI
@@ -69,14 +70,16 @@ jobs:
       id-token: write
       contents: read
     steps:
-      - name: Download all artifacts
+      - name: Download wheel artifacts
         uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e  # v4.1.7
         with:
           path: dist
+          pattern: wheels-*
           merge-multiple: true
 
       - name: Publish to TestPyPI
         uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc  # release/v1
         with:
           repository-url: https://test.pypi.org/legacy/
           skip-existing: true
+          verbose: true

thirdparty/faiss/.github/workflows/build-pull-request.yml

@@ -276,6 +276,21 @@ jobs:
           fetch-tags: true
       - name: Build and Package (conda)
         uses: ./.github/actions/build_conda
+  linux-x86_64-GPU-conda:
+    name: Linux x86_64 GPU (conda, CUDA 12.6)
+    runs-on: 4-core-ubuntu-gpu-t4
+    env:
+      CUDA_ARCHS: "70-real;72-real;75-real;80;86-real"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+      - name: Build and Package (conda)
+        uses: ./.github/actions/build_conda
+        with:
+          cuda: "12.6"
   linux-x86_64-svs:
     name: Linux x86_64 w/ SVS (cmake)
     needs: linux-x86_64-cmake

thirdparty/faiss/conda/faiss/meta.yaml

@@ -156,7 +156,8 @@ outputs:
       requires:
         - numpy >=2.0,<3.0
         - scipy
-        - pytorch-cpu >=2.7
+        - pytorch-cpu >=2.7  # [not win]
+        - pytorch >=2.7      # [win]
         {% if PY_VER == '3.10' or PY_VER == '3.11' %}
         - mkl >=2024.2.2  # [x86_64]
         - python_abi <3.12

thirdparty/faiss/faiss/CMakeLists.txt

@@ -11,7 +11,7 @@
 set(FAISS_SIMD_AVX2_SRC
   impl/fast_scan/impl-avx2.cpp
   impl/hnsw/avx2.cpp
-  impl/pq_code_distance/pq_code_distance-avx2.cpp
+  impl/pq_code_distance/avx2.cpp
   impl/scalar_quantizer/sq-avx2.cpp
   impl/approx_topk/avx2.cpp
   impl/binary_hamming/avx2.cpp
@@ -25,7 +25,7 @@ set(FAISS_SIMD_AVX2_SRC
 set(FAISS_SIMD_AVX512_SRC
   impl/fast_scan/impl-avx512.cpp
   impl/hnsw/avx512.cpp
-  impl/pq_code_distance/pq_code_distance-avx512.cpp
+  impl/pq_code_distance/avx512.cpp
   impl/scalar_quantizer/sq-avx512.cpp
   impl/binary_hamming/avx512.cpp
   utils/simd_impl/distances_avx512.cpp
@@ -39,6 +39,7 @@ set(FAISS_SIMD_NEON_SRC
   impl/scalar_quantizer/sq-neon.cpp
   impl/approx_topk/neon.cpp
   impl/binary_hamming/neon.cpp
+  impl/pq_code_distance/neon.cpp
   utils/simd_impl/distances_aarch64.cpp
   utils/hamming_distance/hamming_neon.cpp
   utils/simd_impl/partitioning_neon.cpp
@@ -50,6 +51,7 @@ set(FAISS_SIMD_SVE_SRC
   utils/simd_impl/distances_arm_sve.cpp
 )
 set(FAISS_SIMD_RVV_SRC
+  impl/pq_code_distance/rvv.cpp
   impl/scalar_quantizer/sq-rvv.cpp
   impl/binary_hamming/rvv.cpp
   utils/simd_impl/distances_rvv.cpp
@@ -137,6 +139,7 @@ set(FAISS_SRC
   impl/PolysemousTraining.cpp
   impl/ProductQuantizer.cpp
   impl/pq_code_distance/pq_code_distance-generic.cpp
+  impl/pq_code_distance/IVFPQ_QueryTables.cpp
   impl/AdditiveQuantizer.cpp
   impl/RaBitQuantizer.cpp
   impl/RaBitQuantizerMultiBit.cpp
@@ -315,8 +318,13 @@ set(FAISS_HEADERS
   impl/simd_dispatch.h
   impl/fast_scan/simd_result_handlers.h
   impl/zerocopy_io.h
-  utils/pq_code_distance.h
   impl/pq_code_distance/pq_code_distance-inl.h
+  impl/pq_code_distance/pq_code_distance-avx2.h
+  impl/pq_code_distance/pq_code_distance-avx512.h
+  impl/pq_code_distance/pq_code_distance-generic.h
+  impl/pq_code_distance/IVFPQ_QueryTables.h
+  impl/pq_code_distance/IVFPQScanner_impl.h
+  impl/pq_code_distance/PQDistanceComputer_impl.h
   invlists/BlockInvertedLists.h
   invlists/DirectMap.h
   invlists/InvertedLists.h