mostly get rid of faiss fork

Signed-off-by: Alexandr Guzhva <alexanderguzhva@gmail.com>

Author: alexanderguzhva

benchmark/hdf5/benchmark_float.cpp

@@ -629,6 +629,7 @@ TEST_F(Benchmark_float, TEST_IVF_RABITQ) {
     for (auto nlist : NLISTs_) {
         for (auto qb : {0, 4, 6, 8}) {
             conf[knowhere::indexparam::NLIST] = nlist;
+            conf[knowhere::indexparam::RABITQ_BITS] = 1;
             conf[knowhere::indexparam::RABITQ_QUERY_BITS] = qb;
             std::vector<int32_t> params = {nlist, qb};
 
@@ -638,3 +639,22 @@ TEST_F(Benchmark_float, TEST_IVF_RABITQ) {
         }
     }
 }
+
+TEST_F(Benchmark_float, TEST_IVF_RABITQ_MULTIBIT) {
+    index_type_ = knowhere::IndexEnum::INDEX_FAISS_IVFRABITQ;
+
+    std::string index_file_name;
+    knowhere::Json conf = cfg_;
+    conf.erase(knowhere::indexparam::RABITQ_QUERY_BITS);
+    for (auto nlist : NLISTs_) {
+        for (auto rbq_bits : {2, 3, 4}) {
+            conf[knowhere::indexparam::NLIST] = nlist;
+            conf[knowhere::indexparam::RABITQ_BITS] = rbq_bits;
+            std::vector<int32_t> params = {nlist, rbq_bits};
+
+            TEST_INDEX(ivf, knowhere::fp32, params);
+            TEST_INDEX(ivf, knowhere::fp16, params);
+            TEST_INDEX(ivf, knowhere::bf16, params);
+        }
+    }
+}

cmake/libs/libdiskann.cmake

@@ -36,12 +36,15 @@ if (WITH_CUVS)
 endif()
 
 add_library(diskann STATIC ${DISKANN_SOURCES})
-set_target_properties(diskann PROPERTIES CXX_STANDARD 17)
+# folly v2026 headers require C++20 (consteval/constinit); DiskANN includes
+# milvus-common's thread_pool.h which pulls in folly/executors.
+set_target_properties(diskann PROPERTIES CXX_STANDARD 20 CXX_STANDARD_REQUIRED ON)
 target_link_libraries(
   diskann
   PUBLIC ${AIO_LIBRARIES}
          ${DISKANN_BOOST_PROGRAM_OPTIONS_LIB}
          nlohmann_json::nlohmann_json
+         milvus-common
          Folly::folly
          fmt::fmt
          prometheus-cpp::core

cmake/libs/libfaiss.cmake

@@ -48,10 +48,13 @@ knowhere_file_glob(
   thirdparty/faiss/faiss/impl/fast_scan/impl-avx512.cpp
   thirdparty/faiss/faiss/impl/hnsw/avx512.cpp
   thirdparty/faiss/faiss/impl/pq_code_distance/pq_code_distance-avx512.cpp
+  thirdparty/faiss/faiss/impl/binary_hamming/avx512.cpp
   thirdparty/faiss/faiss/cppcontrib/knowhere/impl/sq-avx512-fastpath.cpp
   thirdparty/faiss/faiss/utils/distances_fused/avx512.cpp
+  thirdparty/faiss/faiss/utils/hamming_distance/hamming_avx512.cpp
   thirdparty/faiss/faiss/utils/simd_impl/distances_avx512.cpp
   thirdparty/faiss/faiss/utils/simd_impl/rabitq_avx512.cpp
+  thirdparty/faiss/faiss/utils/simd_impl/super_kmeans_kernels_avx512.cpp
 )
 # Baseline sq-avx512.cpp is pulled in textually by the prelude file, not
 # compiled directly. Remove it from the generic list so it is not picked
@@ -83,11 +86,14 @@ knowhere_file_glob(
   thirdparty/faiss/faiss/impl/fast_scan/impl-avx2.cpp
   thirdparty/faiss/faiss/impl/hnsw/avx2.cpp
   thirdparty/faiss/faiss/impl/pq_code_distance/pq_code_distance-avx2.cpp
+  thirdparty/faiss/faiss/impl/binary_hamming/avx2.cpp
   thirdparty/faiss/faiss/cppcontrib/knowhere/impl/sq-avx2-fastpath.cpp
   thirdparty/faiss/faiss/utils/distances_fused/simdlib_based.cpp
+  thirdparty/faiss/faiss/utils/hamming_distance/hamming_avx2.cpp
   thirdparty/faiss/faiss/utils/simd_impl/distances_avx2.cpp
   thirdparty/faiss/faiss/utils/simd_impl/partitioning_avx2.cpp
   thirdparty/faiss/faiss/utils/simd_impl/rabitq_avx2.cpp
+  thirdparty/faiss/faiss/utils/simd_impl/super_kmeans_kernels_avx2.cpp
 )
 knowhere_file_glob(
   GLOB
@@ -111,7 +117,6 @@ knowhere_file_glob(
   thirdparty/faiss/faiss/IndexPQFastScan.cpp
   thirdparty/faiss/faiss/IndexIVFFastScan.cpp
   thirdparty/faiss/faiss/IndexIVFPQFastScan.cpp
-  thirdparty/faiss/faiss/cppcontrib/knowhere/IndexIVFFastScan.cpp
   thirdparty/faiss/faiss/cppcontrib/knowhere/IndexIVFPQFastScan.cpp
   thirdparty/faiss/faiss/cppcontrib/knowhere/IVFFastScanIteratorWorkspace.cpp
 )
@@ -132,8 +137,10 @@ knowhere_file_glob(
   FAISS_DD_NEON_SRCS
   thirdparty/faiss/faiss/impl/approx_topk/neon.cpp
   thirdparty/faiss/faiss/impl/fast_scan/impl-neon.cpp
+  thirdparty/faiss/faiss/impl/binary_hamming/neon.cpp
   thirdparty/faiss/faiss/cppcontrib/knowhere/impl/sq-neon-fastpath.cpp
   thirdparty/faiss/faiss/utils/distances_fused/simdlib_based_neon.cpp
+  thirdparty/faiss/faiss/utils/hamming_distance/hamming_neon.cpp
   thirdparty/faiss/faiss/utils/simd_impl/distances_aarch64.cpp
   thirdparty/faiss/faiss/utils/simd_impl/partitioning_neon.cpp
   thirdparty/faiss/faiss/utils/simd_impl/rabitq_neon.cpp
@@ -175,13 +182,18 @@ knowhere_file_glob(
   FAISS_RVV_SRCS
   thirdparty/faiss/faiss/cppcontrib/knowhere/impl/*rvv.cpp
 )
-# # RVE vanilla Faiss dynamic dispatch related files are not there yet
-# knowhere_file_glob(
-#   GLOB
-#   FAISS_DD_RVV_SRCS
-# )
-# # combine files
-# list(APPEND FAISS_RVV_SRCS ${FAISS_DD_RVV_SRCS})
+# RVV vanilla Faiss dynamic dispatch related files
+knowhere_file_glob(
+  GLOB
+  FAISS_DD_RVV_SRCS
+  thirdparty/faiss/faiss/impl/scalar_quantizer/sq-rvv.cpp
+  thirdparty/faiss/faiss/impl/binary_hamming/rvv.cpp
+  thirdparty/faiss/faiss/utils/simd_impl/distances_rvv.cpp
+  thirdparty/faiss/faiss/utils/hamming_distance/hamming_rvv.cpp
+  thirdparty/faiss/faiss/utils/simd_impl/rabitq_rvv.cpp
+)
+# combine files
+list(APPEND FAISS_RVV_SRCS ${FAISS_DD_RVV_SRCS})
 # remove platform files from general files
 list(REMOVE_ITEM FAISS_SRCS ${FAISS_RVV_SRCS})
 
@@ -322,20 +334,14 @@ if(__PPC64)
 endif()
 
 
-if(LINUX)
-  set(BLA_VENDOR OpenBLAS)
-endif()
-
 if(APPLE)
   set(BLA_VENDOR Apple)
-endif()
-
-if(CMAKE_SYSTEM_NAME STREQUAL "Android" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
-  find_package(OpenBLAS REQUIRED)
-  set(BLAS_LIBRARIES OpenBLAS::OpenBLAS)
-else()
   find_package(LAPACK REQUIRED)
   find_package(BLAS REQUIRED)
+else()
+  find_package(OpenBLAS CONFIG REQUIRED)
+  set(BLAS_LIBRARIES OpenBLAS::OpenBLAS)
+  set(LAPACK_LIBRARIES OpenBLAS::OpenBLAS)
 endif()
 
 find_package(xxHash REQUIRED)
@@ -489,7 +495,7 @@ if(__RISCV64)
   add_dependencies(faiss knowhere_utils)
   target_link_libraries(faiss PUBLIC OpenMP::OpenMP_CXX ${BLAS_LIBRARIES}
                                      ${LAPACK_LIBRARIES} knowhere_utils)
-  target_compile_definitions(faiss PRIVATE FINTEGER=int)
+  target_compile_definitions(faiss PRIVATE FINTEGER=int COMPILE_SIMD_RISCV_RVV)
 endif()
 
 # generate `faiss` library for PPC64

cmake/libs/libmilvus-common.cmake

@@ -1,6 +1,6 @@
 
 set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES "")
-set( MILVUS-COMMON-VERSION 2eb54ae )
+set( MILVUS-COMMON-VERSION 28a2b48f5ab5e14280cfd5d6bf9b34a9cb9a6fd7 )
 set( GIT_REPOSITORY  "https://github.com/zilliztech/milvus-common.git" )
 
 message(STATUS "milvus-common repo: ${GIT_REPOSITORY}")
@@ -16,6 +16,15 @@ find_package(fmt REQUIRED)
 find_package(glog REQUIRED)
 find_package(nlohmann_json REQUIRED)
 find_package(folly REQUIRED)
+if(APPLE)
+    set(BLA_VENDOR Apple)
+    find_package(LAPACK REQUIRED)
+    find_package(BLAS REQUIRED)
+    set(MILVUS_COMMON_BLAS_LIBS ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES})
+else()
+    find_package(OpenBLAS CONFIG REQUIRED)
+    set(MILVUS_COMMON_BLAS_LIBS OpenBLAS::OpenBLAS)
+endif()
 find_package(prometheus-cpp REQUIRED)
 
 FetchContent_Declare(
@@ -32,11 +41,19 @@ if ( NOT milvus-common_POPULATED )
     add_subdirectory( ${milvus-common_SOURCE_DIR}
                       ${milvus-common_BINARY_DIR} )
 
+    # milvus-common's CMakeLists hardcodes CMAKE_CXX_STANDARD=17, but folly v2026
+    # headers require C++20 (consteval/constinit). Override on the target.
+    set_target_properties(milvus-common PROPERTIES
+        CXX_STANDARD 20
+        CXX_STANDARD_REQUIRED ON)
+
     if(NOT APPLE)
         target_link_libraries(milvus-common PUBLIC atomic)
     endif()
 endif()
 
+target_link_libraries(milvus-common PUBLIC ${MILVUS_COMMON_BLAS_LIBS})
+
 set( MILVUS_COMMON_INCLUDE_DIR ${milvus-common_SOURCE_DIR}/include
      CACHE INTERNAL "Path to milvus-common include directory" )
 

conanfile.py

@@ -90,6 +90,12 @@ def _minimum_compilers_version(self):
             "apple-clang": "13",
         }
 
+    @property
+    def _openblas_dynamic_arches(self):
+        # Keep OpenBLAS portable across supported CPU families without pinning a
+        # concrete TARGET such as SANDYBRIDGE.
+        return ["x86", "x86_64", "armv8", "ppc64", "ppc64le"]
+
     def config_options(self):
         if self.settings.os == "Windows":
             self.options.rm_safe("fPIC")
@@ -101,6 +107,8 @@ def config_options(self):
     def configure(self):
         if self.options.shared:
             self.options.rm_safe("fPIC")
+        if self.settings.os in ["Linux", "Android"] and str(self.settings.arch) in self._openblas_dynamic_arches:
+            self.options["openblas"].dynamic_arch = True
 
     def requirements(self):
         self.requires("abseil/20250127.0#481edcc75deb0efb16500f511f0f0a1c")
@@ -114,18 +122,21 @@ def requirements(self):
         self.requires("double-conversion/3.3.0#640e35791a4bac95b0545e2f54b7aceb")
         self.requires("xz_utils/5.4.5#fc4e36861e0a47ecd4a40a00e6d29ac8")
         self.requires("protobuf/5.27.0@milvus/dev#42f031a96d21c230a6e05bcac4bdd633", force=True, override=True)
-        self.requires("lz4/1.9.4#7f0b5851453198536c14354ee30ca9ae", force=True, override=True)
+        self.requires("lz4/1.10.0#982d9b673900f665a1da109e09c17cab", force=True, override=True)
         if self.settings.os == "Linux":
             self.requires("liburing/2.8", force=True, override=True)
         self.requires("fmt/11.2.0#eb98daa559c7c59d591f4720dde4cd5c", force=True, override=True)
         self.requires("libevent/2.1.12#95065aaefcd58d3956d6dfbfc5631d97")
         self.requires("grpc/1.67.1@milvus/dev#efeaa484b59bffaa579004d5e82ec4fd")
-        self.requires("folly/2024.08.12.00@milvus/dev#f9b2bdf162c0ec47cb4e5404097b340d")
+        self.requires("folly/2026.04.20.00@milvus/dev#06852bea5b6449f0c4eb0df002b5779c")
+        self.requires("fast_float/8.0.0@milvus/dev#c7802833c74c5a86ffed70e4af1a795e")
         self.requires("libcurl/8.10.1#a3113369c86086b0e84231844e7ed0a9", force=True, override=True)
         self.requires("simde/0.8.2#5e1edfd5cba92f25d79bf6ef4616b972")
         self.requires("xxhash/0.8.3#caa6d0af1b951c247922e38fbcebdbe6")
+        if self.settings.os == "Linux":
+            self.requires("openblas/0.3.30")
         if self.settings.os == "Android":
-            self.requires("openblas/0.3.27")
+            self.requires("openblas/0.3.30")
         if not self.options.with_light:
             self.requires("opentelemetry-cpp/1.23.0@milvus/dev#11bc565ec6e82910ae8f7471da756720")
         if self.settings.os not in ["Macos", "Android"]:

include/knowhere/comp/index_param.h

@@ -228,6 +228,7 @@ constexpr const char* DROP_RATIO_BUILD = "drop_ratio_build";
 constexpr const char* DROP_RATIO_SEARCH = "drop_ratio_search";
 
 // RaBitQ Params
+constexpr const char* RABITQ_BITS = "rbq_bits";
 constexpr const char* RABITQ_QUERY_BITS = "rbq_bits_query";
 
 // minhash meta Params

include/knowhere/index/index_node.h

@@ -47,11 +47,7 @@ class ThreadPool;
 #endif
 
 namespace faiss {
-namespace cppcontrib {
-namespace knowhere {
 class IndexFlat;
-}
-}  // namespace cppcontrib
 }  // namespace faiss
 
 namespace knowhere {
@@ -644,7 +640,11 @@ class IndexNode : public Object {
     // into different representations for ANN search. Since the base index holds encoded
     // vectors (not raw), this IndexFlat stores original vectors for exact distance
     // computation during MaxSim reranking.
-    std::shared_ptr<faiss::cppcontrib::knowhere::IndexFlat> emb_list_raw_index_;
+    // Baseline type so that the same shared_ptr can hold either the knowhere
+    // Jaccard-aware IndexFlat subclass (fresh build path, if ever needed) or
+    // a plain ::faiss::IndexFlat{,IP,L2} restored by the deserialization
+    // factory in cppcontrib/knowhere/impl/index_read.cpp.
+    std::shared_ptr<::faiss::IndexFlat> emb_list_raw_index_;
 
 #if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT)
     struct PrometheusMetrics {

scripts/install_deps.sh

@@ -57,7 +57,6 @@ if [[ "${OS}" == "Linux" ]]; then
             libdouble-conversion-dev   # folly dependency
             libevent-dev               # gRPC dependency
             libgflags-dev              # folly / glog dependency
-            libopenblas-openmp-dev     # faiss BLAS operations
             libomp-dev                 # OpenMP parallelization
             python3 python3-pip
             clang-tidy-14              # CI: static analysis (analyzer CI)
@@ -80,7 +79,6 @@ if [[ "${OS}" == "Linux" ]]; then
             double-conversion-devel  # folly dependency
             libevent-devel           # gRPC dependency
             gflags-devel             # folly / glog dependency
-            openblas-devel           # faiss BLAS operations
             libomp-devel             # OpenMP parallelization
             python3 python3-pip
             clang-tools-extra        # CI: static analysis (provides clang-tidy)

src/index/data_view_dense_index/refine_computer.h

@@ -139,7 +139,11 @@ struct QuantRefine {
     static constexpr size_t list_num = 1;
     static constexpr size_t segment_size = 48;
     faiss::ScalarQuantizer* quantizer = nullptr;
-    faiss::cppcontrib::knowhere::InvertedLists* storage = nullptr;
+    // Path-D step 10.10: declared as the concrete
+    // ConcurrentArrayInvertedLists* (previously the base InvertedLists*)
+    // because this class uses the segment-sliced get_codes(list_no, offset)
+    // accessor which is no longer a virtual on the base.
+    faiss::cppcontrib::knowhere::ConcurrentArrayInvertedLists* storage = nullptr;
     faiss::MetricType metric_type;
     DataFormatEnum origin_data_type;
     RefineType refine_type;

src/index/hnsw/faiss_hnsw.cc

@@ -706,8 +706,11 @@ get_index_data_format(const faiss::Index* index) {
     }
 
     // is it flat?
-    // note: IndexFlatCosine preserves the original data, no cosine norm is applied
-    auto index_flat = dynamic_cast<const faiss::cppcontrib::knowhere::IndexFlat*>(index);
+    // note: IndexFlatCosine preserves the original data, no cosine norm is applied.
+    // Cast to baseline ::faiss::IndexFlat so this catches both the knowhere
+    // Jaccard-aware subclass (fresh build path) and baseline variants produced
+    // by the IxFI/IxF2 deserialization factory.
+    auto index_flat = dynamic_cast<const ::faiss::IndexFlat*>(index);
     if (index_flat != nullptr) {
         return DataFormatEnum::fp32;
     }
@@ -1433,7 +1436,7 @@ class BaseFaissRegularIndexHNSWNode : public BaseFaissRegularIndexNode {
 
                     // perform the search
                     if (is_refined) {
-                        faiss::cppcontrib::knowhere::IndexRefineSearchParameters refine_params;
+                        faiss::IndexRefineSearchParameters refine_params;
                         refine_params.k_factor = hnsw_cfg.refine_k.value_or(1);
                         // a refine procedure itself does not need to care about filtering
                         refine_params.sel = nullptr;
@@ -1727,7 +1730,7 @@ class BaseFaissRegularIndexHNSWNode : public BaseFaissRegularIndexNode {
 
                         // perform the search
                         if (is_refined) {
-                            faiss::cppcontrib::knowhere::IndexRefineSearchParameters refine_params;
+                            faiss::IndexRefineSearchParameters refine_params;
                             refine_params.k_factor = hnsw_cfg.refine_k.value_or(1);
                             // a refine procedure itself does not need to care about filtering
                             refine_params.sel = nullptr;