Upgrade faiss to 1.41.1+, from e5114c23 to ef28c6be (#1558)

Signed-off-by: Alexandr Guzhva <alexanderguzhva@gmail.com>

Author: alexanderguzhva

cmake/libs/libfaiss.cmake

@@ -9,6 +9,7 @@ knowhere_file_glob(
   thirdparty/faiss/faiss/*.cpp
   thirdparty/faiss/faiss/impl/*.cpp
   thirdparty/faiss/faiss/impl/fast_scan/*.cpp
+  thirdparty/faiss/faiss/impl/hnsw/*.cpp
   thirdparty/faiss/faiss/impl/pq_code_distance/*.cpp
   thirdparty/faiss/faiss/impl/scalar_quantizer/*.cpp
   thirdparty/faiss/faiss/invlists/*.cpp
@@ -32,11 +33,12 @@ knowhere_file_glob(
   GLOB
   FAISS_DD_AVX512_SRCS
   thirdparty/faiss/faiss/impl/fast_scan/impl-avx512.cpp
+  thirdparty/faiss/faiss/impl/hnsw/avx512.cpp
   thirdparty/faiss/faiss/impl/pq_code_distance/pq_code_distance-avx512.cpp
   thirdparty/faiss/faiss/impl/scalar_quantizer/sq-avx512.cpp
-  # # temporarily disabled
-  # thirdparty/faiss/faiss/utils/distances_fused/avx512.cpp
+  thirdparty/faiss/faiss/utils/distances_fused/avx512.cpp
   thirdparty/faiss/faiss/utils/simd_impl/distances_avx512.cpp
+  thirdparty/faiss/faiss/utils/simd_impl/rabitq_avx512.cpp
 )
 # combine files
 list(APPEND FAISS_AVX512_SRCS ${FAISS_DD_AVX512_SRCS})
@@ -56,9 +58,12 @@ knowhere_file_glob(
   FAISS_DD_AVX2_SRCS
   thirdparty/faiss/faiss/impl/approx_topk/avx2.cpp
   thirdparty/faiss/faiss/impl/fast_scan/impl-avx2.cpp
+  thirdparty/faiss/faiss/impl/hnsw/avx2.cpp
   thirdparty/faiss/faiss/impl/pq_code_distance/pq_code_distance-avx2.cpp
   thirdparty/faiss/faiss/impl/scalar_quantizer/sq-avx2.cpp
+  thirdparty/faiss/faiss/utils/distances_fused/simdlib_based.cpp
   thirdparty/faiss/faiss/utils/simd_impl/distances_avx2.cpp
+  thirdparty/faiss/faiss/utils/simd_impl/rabitq_avx2.cpp
 )
 # combine files
 list(APPEND FAISS_AVX2_SRCS ${FAISS_DD_AVX2_SRCS})
@@ -97,7 +102,9 @@ knowhere_file_glob(
   thirdparty/faiss/faiss/impl/approx_topk/neon.cpp
   thirdparty/faiss/faiss/impl/fast_scan/impl-neon.cpp
   thirdparty/faiss/faiss/impl/scalar_quantizer/sq-neon.cpp
+  thirdparty/faiss/faiss/utils/distances_fused/simdlib_based_neon.cpp
   thirdparty/faiss/faiss/utils/simd_impl/distances_aarch64.cpp
+  thirdparty/faiss/faiss/utils/simd_impl/rabitq_neon.cpp
 )
 # combine files
 list(APPEND FAISS_NEON_SRCS ${FAISS_DD_NEON_SRCS})

thirdparty/faiss/benchs/avx512_result_handlers/bench_avx512_result_handler.cpp

@@ -5,6 +5,8 @@
  * LICENSE file in the root directory of this source tree.
  */
 
+#if defined(__x86_64__)
+
 #include "faiss_avx512_result_handler.h"
 
 #include <faiss/IndexIVF.h>
@@ -125,3 +127,5 @@ int main() {
 
     return 0;
 }
+
+#endif // defined(__x86_64__)

thirdparty/faiss/benchs/avx512_result_handlers/faiss_avx512_result_handler.h

@@ -7,6 +7,8 @@
 
 #pragma once
 
+#if defined(__x86_64__)
+
 #include <faiss/impl/FaissAssert.h>
 #include <faiss/impl/ResultHandler.h>
 #include <immintrin.h>
@@ -145,3 +147,5 @@ struct ReservoirResultHandlerAVX512 : ResultHandler {
 };
 
 } // namespace faiss
+
+#endif // defined(__x86_64__)

thirdparty/faiss/benchs/avx512_result_handlers/partition.cpp

@@ -5,12 +5,13 @@
  * LICENSE file in the root directory of this source tree.
  */
 
+#if defined(__x86_64__)
+
 #include "partition.h"
 
 #include <algorithm>
 #include <cassert>
 #include <cstdint>
-#include <cstdio>
 #include <cstring>
 #include <limits>
 #include <numeric>
@@ -601,3 +602,5 @@ void argsort(size_t n, float* val, int32_t* idx) {
     argsort(left_size, val, idx);
     argsort(n - left_size, val + left_size, idx + left_size);
 }
+
+#endif // defined(__x86_64__)

thirdparty/faiss/benchs/avx512_result_handlers/partition.h

@@ -7,6 +7,8 @@
 
 #pragma once
 
+#if defined(__x86_64__)
+
 #include <cstddef>
 #include <cstdint>
 
@@ -38,3 +40,5 @@ void argpartition(size_t n, float* val, int32_t* idx, size_t k);
  * @param idx   Array of indices (modified in place, reordered with values)
  */
 void argsort(size_t n, float* val, int32_t* idx);
+
+#endif // defined(__x86_64__)

thirdparty/faiss/benchs/avx512_result_handlers/test_bitonic.cpp

@@ -4,6 +4,9 @@
  * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  */
+
+#if defined(__x86_64__)
+
 #include <immintrin.h>
 #include <algorithm>
 #include <cassert>
@@ -109,3 +112,5 @@ int main() {
 
     return 0;
 }
+
+#endif // defined(__x86_64__)

thirdparty/faiss/benchs/avx512_result_handlers/test_partition.cpp

@@ -5,6 +5,8 @@
  * LICENSE file in the root directory of this source tree.
  */
 
+#if defined(__x86_64__)
+
 #include <gtest/gtest.h>
 
 #include "partition.h"
@@ -170,3 +172,5 @@ TEST(ArgpartitionTest, PartitionsCorrectly) {
         }
     }
 }
+
+#endif // defined(__x86_64__)

thirdparty/faiss/benchs/bench_cppcontrib_sa_decode.cpp

@@ -12,7 +12,6 @@
 #include <iostream>
 #include <memory>
 #include <random>
-#include <thread>
 #include <tuple>
 #include <vector>
 
@@ -32,7 +31,7 @@ std::tuple<std::shared_ptr<faiss::Index>, std::vector<uint8_t>> trainDataset(
         const uint64_t d,
         const std::string& description) {
     //
-    omp_set_num_threads(std::thread::hardware_concurrency());
+    omp_set_num_threads(omp_get_max_threads());
 
     // train an index
     auto index = std::shared_ptr<faiss::Index>(

thirdparty/faiss/benchs/bench_hnsw_flat_panorama.py

@@ -26,17 +26,22 @@
 
 def eval_recall(index, efSearch_val, xq, gt, k):
     """Evaluate recall and QPS for a given efSearch value."""
+    nq = len(xq)
+    faiss.cvar.indexPanorama_stats.reset()
     t0 = time.time()
     _, I = index.search(xq, k=k)
     t = time.time() - t0
-    speed = t * 1000 / len(xq)
+    speed = t * 1000 / nq
     qps = 1000 / speed
 
-    corrects = (gt == I).sum()
-    recall = corrects / (len(xq) * k)
+    recall = np.mean(
+        [len(set(gt[i]) & set(I[i])) / k for i in range(nq)],
+    )
+    ratio_dims_scanned = faiss.cvar.indexPanorama_stats.ratio_dims_scanned
     print(
         f"\tefSearch {efSearch_val:3d}, Recall@{k}: "
-        f"{recall:.6f}, speed: {speed:.6f} ms/query, QPS: {qps:.2f}"
+        f"{recall:.6f}, speed: {speed:.6f} ms/query, QPS: {qps:.2f}, "
+        f"dims scanned: {ratio_dims_scanned * 100:.1f}%"
     )
 
     return recall, qps

thirdparty/faiss/c_api/utils/distances_c.h

@@ -74,11 +74,11 @@ void faiss_fvec_norms_L2sqr(float* norms, const float* x, size_t d, size_t nx);
 /// L2-renormalize a set of vector. Nothing done if the vector is 0-normed
 void faiss_fvec_renorm_L2(size_t d, size_t nx, float* x);
 
-/// Setter of threshold value on nx above which we switch to BLAS to compute
+/// Setter of threshold value on nx * d above which we switch to BLAS to compute
 /// distances
 void faiss_set_distance_compute_blas_threshold(int value);
 
-/// Getter of threshold value on nx above which we switch to BLAS to compute
+/// Getter of threshold value on nx * d above which we switch to BLAS to compute
 /// distances
 int faiss_get_distance_compute_blas_threshold();