Update build scripts (#1619)

* update build scripts

Signed-off-by: Alexandr Guzhva <alexanderguzhva@gmail.com>

* fix clang-tidy

Signed-off-by: Alexandr Guzhva <alexanderguzhva@gmail.com>

* continue fixing clang-tidy

Signed-off-by: Alexandr Guzhva <alexanderguzhva@gmail.com>

* more fixes

Signed-off-by: Alexandr Guzhva <alexanderguzhva@gmail.com>

* more fixes

Signed-off-by: Alexandr Guzhva <alexanderguzhva@gmail.com>

* more cleanup

Signed-off-by: Alexandr Guzhva <alexanderguzhva@gmail.com>

* more fixes

Signed-off-by: Alexandr Guzhva <alexanderguzhva@gmail.com>

* more cleanup

Signed-off-by: Alexandr Guzhva <alexanderguzhva@gmail.com>

* more formatting

Signed-off-by: Alexandr Guzhva <alexanderguzhva@gmail.com>

---------

Signed-off-by: Alexandr Guzhva <alexanderguzhva@gmail.com>

Author: alexanderguzhva

.clang-tidy

@@ -26,7 +26,8 @@ Checks: >
     performance-trivially-destructible, performance-inefficient-vector-operation,
     performance-move-const-arg, performance-move-constructor-init,
     performance-noexcept-move-constructor, performance-no-automatic-move,
-    performance-type-promotion-in-math-fn
+    performance-type-promotion-in-math-fn, -clang-analyzer-optin.core.EnumCastOutOfRange,
+    -modernize-use-ranges
 
 WarningsAsErrors: '*'
 HeaderFilterRegex: '^(?:.*\/knowhere\/include\/.+\.h)|(?:.*\/knowhere\/src\/.+\.h)$'

.claude/skills/building/SKILL.md

@@ -10,7 +10,7 @@ description: Use when building knowhere from source, configuring build options (
 ```bash
 # Ubuntu/Debian
 sudo apt install build-essential libopenblas-openmp-dev libaio-dev python3-dev python3-pip
-pip3 install conan==2.25.1 --user
+pip3 install conan==2.28.1 --user
 conan profile detect --force
 conan remote add default-conan-local2 https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local2
 export PATH=$PATH:$HOME/.local/bin

.github/workflows/analyzer.yaml

@@ -39,7 +39,7 @@ jobs:
       - name: Build & Analyzer
         run: |
           make WITH_UT=True \
-          && find src -type f | grep -E "\.cc$" | xargs /usr/bin/run-clang-tidy-14.py -quiet -p=./build/Release
+          && find src -type f | grep -E "\.cc$" | xargs /usr/bin/run-clang-tidy-22.py -quiet -p=./build/Release
       - name: Save Cache
         uses: ./.github/actions/cache-save
         with:

.github/workflows/release-python.yml

@@ -20,8 +20,6 @@ jobs:
         include:
           - os: ubuntu-22.04
             arch: x86_64
-          # - os: ubuntu-20.04
-          #   arch: aarch64
           # - os: macos-11
           #   arch: x86_64
           # - os: macos-11
@@ -37,20 +35,21 @@ jobs:
 
       - uses: actions/checkout@v3
 
-      - uses: pypa/cibuildwheel@v2.12.1
+      - uses: pypa/cibuildwheel@v3.4.1
         env:
-          CIBW_BUILD: "cp38-* cp39-* cp310-* cp311*"
-          CIBW_SKIP: "*musllinux*" # faiss-cpu fails to build on musllinux
+          CIBW_BUILD: "cp310-* cp311-* cp312-* cp313-* cp314-*"
+          CIBW_SKIP: "*musllinux* cp31?t-*" # faiss-cpu fails to build on musllinux; skip free-threaded CPython.
           CIBW_ARCHS: ${{ matrix.arch }}
+          CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28"
+          CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux_2_28"
           CIBW_BEFORE_ALL_LINUX: "bash scripts/install_deps.sh && make"
-          # Use portable wheel build script instead of plain setup.py
-          CIBW_BEFORE_BUILD: "pip3 install pytest numpy faiss-cpu bfloat16 auditwheel"
+          CIBW_BEFORE_BUILD: "python -m pip install wheel pytest 'numpy>=2,<3' ml-dtypes faiss-cpu auditwheel"
           # Custom build command using our portable wheel builder
           CIBW_BUILD_FRONTEND: "build"
           # Note: cibuildwheel will use our build_portable_wheel.sh via setup.py
           # The script automatically handles auditwheel repair
-          CIBW_TEST_REQUIRES: "pytest numpy<2 faiss-cpu bfloat16"
-          CIBW_TEST_COMMAND: "pytest {project}/tests/python/test_index_with_random.py"
+          CIBW_TEST_REQUIRES: "wheel pytest numpy>=2,<3 ml-dtypes faiss-cpu"
+          CIBW_TEST_COMMAND: "python -m pytest {project}/tests/python/test_index_with_random.py"
           CIBW_ENVIRONMENT: "SETUPTOOLS_SCM_PRETEND_VERSION=${{ github.event.inputs.tag }}"
         with:
           package-dir: python
@@ -61,7 +60,7 @@ jobs:
 
   upload:
     needs: [build_wheels]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/download-artifact@v3
         with:

.github/workflows/ut.yaml

@@ -103,17 +103,15 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: "3.8"
+          python-version: "3.10"
       - name: Install Dependency
         run: |
           bash scripts/install_deps.sh
-          pip3 install pytest faiss-cpu
+          python -m pip install wheel pytest 'numpy>=2,<3' ml-dtypes faiss-cpu
       - name: Install GCC 12
         # GCC 11 (ubuntu-22.04 default) has incomplete C++20 support:
         # std::partial_ordering, std::strong_ordering, std::weak_ordering
         # from <compare> are not available, causing abseil build failures.
-        # Installed after install_deps.sh so bfloat16 builds with GCC 11
-        # (bfloat16 fails on GCC 12 due to missing #include <memory>).
         run: |
           sudo apt-get update
           sudo apt-get install -y gcc-12 g++-12
@@ -139,4 +137,4 @@ jobs:
           cd python && pip3 install dist/*-manylinux*.whl
       - name: Test
         run: |
-          pytest tests/python/test_index_with_random.py
+          python -m pytest tests/python/test_index_with_random.py

.gitignore

@@ -55,6 +55,7 @@ coverage/
 
 # virtualenv
 venv/
+**/.venv/
 **/dist/
 **/knowhere.egg-info/
 **/knowhere/knowhere_wrap.cpp

Makefile

@@ -31,6 +31,10 @@ CARDINAL_VERSION_FORCE_CHECKOUT ?=
 WITH_DEBUG ?=
 CONAN_PROFILE ?=
 
+# CMake 4.x removed compatibility with cmake_minimum_required versions below
+# 3.5. Export this so Conan-built third-party packages inherit it too.
+export CMAKE_POLICY_VERSION_MINIMUM ?= 3.5
+
 # Prevent build-flag variables from leaking into the environment of child
 # processes (conan / cmake).  Without this, GNU Make exports command-line
 # variables such as WITH_ASAN to every sub-process, which causes the custom
@@ -104,7 +108,7 @@ ifneq ($(CARDINAL_VERSION_FORCE_CHECKOUT),)
 endif
 
 ifdef CONAN_PROFILE
-    CONAN_SETTINGS += -pr $(CONAN_PROFILE)
+    CONAN_SETTINGS += -pr:h $(CONAN_PROFILE) -pr:b $(CONAN_PROFILE)
 endif
 
 .PHONY: build test \
@@ -173,7 +177,7 @@ help: ## Show available targets
 	@echo "  WITH_CARDINAL=True Enable Cardinal build"
 	@echo "  CARDINAL_VERSION_FORCE_CHECKOUT=True Force Cardinal checkout to configured version"
 	@echo "  WITH_DEBUG=True    Debug build (default: Release)"
-	@echo "  CONAN_PROFILE=<p>  Use a custom Conan profile (e.g. clang, gcc-15)"
+	@echo "  CONAN_PROFILE=<p>  Use one custom Conan profile for host and build"
 	@echo "  LIBCXX=<lib>       Override compiler.libcxx (auto-detected from OS)"
 	@echo ""
 	@echo "Examples:"
@@ -183,5 +187,5 @@ help: ## Show available targets
 	@echo "  make WITH_GPU=True WITH_UT=True   # GPU UT"
 	@echo "  make WITH_DEBUG=True WITH_UT=True # CPU debug + UT"
 	@echo "  make LIBCXX=libc++                # override compiler.libcxx"
-	@echo "  make CONAN_PROFILE=gcc15          # CPU with custom profile"
+	@echo "  make CONAN_PROFILE=/path/to/profile # CPU with custom profile"
 	@echo ""

ci/docker/builder/cpu/ubuntu20.04/Dockerfile

@@ -33,7 +33,7 @@ RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3 \
 # install knowhere dependancies
 RUN apt update \
     && apt install -y libopenblas-openmp-dev libcurl4-openssl-dev libaio-dev libevent-dev lcov \
-    && pip3 install conan==2.25.1 \
+    && pip3 install conan==2.28.1 \
     && conan profile detect --force \
     && conan remote add default-conan-local2 https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local2 \
     && export PATH=$PATH:$HOME/.local/bin

ci/docker/builder/cpu/ubuntu22.04/Dockerfile

@@ -6,7 +6,6 @@ ENV CMAKE_TAR="cmake-3.28.5-linux-x86_64.tar.gz"
 ENV CCACHE_VERSION="v4.9.1"
 ENV CCACHE_DIR="ccache-4.9.1-linux-x86_64"
 ENV CCACHE_TAR="ccache-4.9.1-linux-x86_64.tar.xz"
-ENV BFLOAT16_WHL="bfloat16-1.4.0-cp311-cp311-linux_x86_64.whl"
 
 RUN apt update \
     && apt install -y ca-certificates apt-transport-https software-properties-common lsb-release \
@@ -26,7 +25,7 @@ RUN apt update \
     && apt remove --purge -y \
     && rm -rf /var/lib/apt/lists/*
 
-# install cmake, ccache and bfloat16
+# install cmake, ccache and Python BF16 dtype support
 RUN cd /tmp \
     && wget https://github.com/Kitware/CMake/releases/download/${CMAKE_VERSION}/${CMAKE_TAR} \
     && tar --strip-components=1 -xz -C /usr/local -f ${CMAKE_TAR} \
@@ -35,14 +34,12 @@ RUN cd /tmp \
     && tar -xf ${CCACHE_TAR} \
     && cp ${CCACHE_DIR}/ccache /usr/local/bin \
     && rm -f ${CCACHE_TAR} \
-    && wget https://github.com/zilliztech/knowhere/releases/download/v2.3.1/${BFLOAT16_WHL} \
-    && pip3 install ${BFLOAT16_WHL} \
-    && rm -f ${BFLOAT16_WHL}
+    && pip3 install 'numpy>=2,<3' ml-dtypes
 
 # install knowhere dependancies
 RUN apt update \
     && apt install -y libopenblas-openmp-dev libcurl4-openssl-dev libaio-dev libevent-dev lcov \
-    && pip3 install conan==2.25.1 \
+    && pip3 install conan==2.28.1 \
     && conan profile detect --force \
     && conan remote add default-conan-local2 https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local2
 # clone knowhere repo and build to update .conan2

ci/docker/builder/cpu/ubuntu22.04/arm64/Dockerfile

@@ -5,7 +5,6 @@ ENV CMAKE_VERSION="v3.30.5"
 ENV CMAKE_TAR="cmake-3.30.5-linux-aarch64.tar.gz"
 ENV CCACHE_DIR="ccache-4.9.1"
 ENV CCACHE_SRC_TAR="v4.9.1.tar.gz"
-ENV BFLOAT16_WHL="bfloat16-1.4.0-cp311-cp311-linux_aarch64.whl"
 
 RUN apt update \
     && apt install -y ca-certificates apt-transport-https software-properties-common lsb-release \
@@ -25,7 +24,7 @@ RUN apt update \
     && apt remove --purge -y \
     && rm -rf /var/lib/apt/lists/*
 
-# install cmake, ccache and bfloat16
+# install cmake, ccache and Python BF16 dtype support
 RUN cd /tmp \
     && wget https://github.com/Kitware/CMake/releases/download/${CMAKE_VERSION}/${CMAKE_TAR} \
     && tar --strip-components=1 -xz -C /usr/local -f ${CMAKE_TAR} \
@@ -37,14 +36,12 @@ RUN cd /tmp \
     && cmake -D CMAKE_BUILD_TYPE=Release .. \
     && make -j3 && make install \
     && cd ../.. && rm -rf ${CCACHE_DIR} ${CCACHE_SRC_TAR} \
-    && wget https://github.com/zilliztech/knowhere/releases/download/v2.3.1/${BFLOAT16_WHL} \
-    && pip3 install ${BFLOAT16_WHL} \
-    && rm -f ${BFLOAT16_WHL}
+    && pip3 install 'numpy>=2,<3' ml-dtypes
 
 # install knowhere dependancies
 RUN apt update \
     && apt install -y libopenblas-openmp-dev libcurl4-openssl-dev libaio-dev libevent-dev lcov \
-    && pip3 install conan==2.25.1 \
+    && pip3 install conan==2.28.1 \
     && conan profile detect --force \
     && conan remote add default-conan-local2 https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local2
 # clone knowhere repo and build to update .conan2