From be9ed45b4f134d5921d80b3d3ee910aeda92dbb8 Mon Sep 17 00:00:00 2001 From: Sergey Nikolaev Date: Fri, 10 Apr 2026 01:06:02 +0700 Subject: [PATCH 1/5] ci: include embeddings runtime in build artifacts --- .github/workflows/build_template.yml | 4 ++-- .github/workflows/test.yml | 2 +- .github/workflows/test_template.yml | 13 ------------- .github/workflows/win_test_template.yml | 14 -------------- 4 files changed, 3 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build_template.yml b/.github/workflows/build_template.yml index 64003b95..982b4328 100644 --- a/.github/workflows/build_template.yml +++ b/.github/workflows/build_template.yml @@ -49,7 +49,7 @@ on: artifact_list: required: false type: string - default: "build/columnar/lib_manticore_columnar_avx2.so build/columnar/lib_manticore_columnar_avx512.so build/columnar/lib_manticore_columnar.so build/secondary/lib_manticore_secondary_avx2.so build/secondary/lib_manticore_secondary_avx512.so build/secondary/lib_manticore_secondary.so build/knn/lib_manticore_knn.so build/knn/lib_manticore_knn_avx2.so build/knn/lib_manticore_knn_avx512.so build/_deps/manticore-build/src/searchd build/_deps/manticore-build/src/indexer build/_deps/manticore-build/src/indextool build/_deps/manticore-build/config/*.c build/_deps/manticore-build/config/*.h" + default: "build/columnar/lib_manticore_columnar_avx2.so build/columnar/lib_manticore_columnar_avx512.so build/columnar/lib_manticore_columnar.so build/secondary/lib_manticore_secondary_avx2.so build/secondary/lib_manticore_secondary_avx512.so build/secondary/lib_manticore_secondary.so build/knn/lib_manticore_knn.so build/knn/lib_manticore_knn_avx2.so build/knn/lib_manticore_knn_avx512.so build/share/manticore/modules/lib_manticore_knn_embeddings.so build/share/manticore/modules/lib_manticore_knn_embeddings.dylib build/share/modules/lib_manticore_knn_embeddings.dll build/_deps/manticore-build/src/searchd build/_deps/manticore-build/src/indexer build/_deps/manticore-build/src/indextool build/_deps/manticore-build/config/*.c build/_deps/manticore-build/config/*.h" HOMEBREW_PREFIX: required: false type: string @@ -213,4 +213,4 @@ jobs: uses: manticoresoftware/upload_artifact_with_retries@v4 with: name: ${{ steps.artifact_name.outputs.name }} - path: ${{ inputs.artifact_list }} \ No newline at end of file + path: ${{ inputs.artifact_list }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 40ec13b5..39f92059 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -148,7 +148,7 @@ jobs: CTEST_CMAKE_GENERATOR: "Ninja Multi-Config" CTEST_CONFIGURATION_TYPE: Debug cache_key: build_windows_x64 - artifact_list: "build/columnar/Debug/lib_manticore_columnar.dll build/secondary/Debug/lib_manticore_secondary.dll build/knn/Debug/lib_manticore_knn.dll build/columnar/Debug/lib_manticore_columnar_avx2.dll build/secondary/Debug/lib_manticore_secondary_avx2.dll build/knn/Debug/lib_manticore_knn_avx2.dll build/columnar/Debug/lib_manticore_columnar_avx512.dll build/secondary/Debug/lib_manticore_secondary_avx512.dll build/knn/Debug/lib_manticore_knn_avx512.dll build/_deps/manticore-build/src/Debug/indexer.exe build/_deps/manticore-build/src/Debug/searchd.exe" # + artifact_list: "build/columnar/Debug/lib_manticore_columnar.dll build/secondary/Debug/lib_manticore_secondary.dll build/knn/Debug/lib_manticore_knn.dll build/columnar/Debug/lib_manticore_columnar_avx2.dll build/secondary/Debug/lib_manticore_secondary_avx2.dll build/knn/Debug/lib_manticore_knn_avx2.dll build/columnar/Debug/lib_manticore_columnar_avx512.dll build/secondary/Debug/lib_manticore_secondary_avx512.dll build/knn/Debug/lib_manticore_knn_avx512.dll build/share/modules/lib_manticore_knn_embeddings.dll build/_deps/manticore-build/src/Debug/indexer.exe build/_deps/manticore-build/src/Debug/searchd.exe" # MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }} embeddings_artifact: embeddings_windows_x86_64 diff --git a/.github/workflows/test_template.yml b/.github/workflows/test_template.yml index 9fa5ef4b..b9a45018 100644 --- a/.github/workflows/test_template.yml +++ b/.github/workflows/test_template.yml @@ -104,19 +104,6 @@ jobs: name: ${{ steps.build_artifact.outputs.name }} path: . - - name: Download embeddings lib - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: embeddings_linux_x86_64 - path: ./embeddings-lib/ - - - name: Initialization of embeddings lib - run: | - mkdir -p embeddings/target/release - mv ./embeddings-lib/build/* embeddings/target/release/ - rm -fr ./embeddings-lib - - name: Check out main cache before building uses: actions/cache@v4 with: diff --git a/.github/workflows/win_test_template.yml b/.github/workflows/win_test_template.yml index 91499c34..8ca42dc5 100644 --- a/.github/workflows/win_test_template.yml +++ b/.github/workflows/win_test_template.yml @@ -146,20 +146,6 @@ jobs: php-version: '8.2' extensions: mysqli - - name: Download embeddings lib - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: embeddings_windows_x86_64 - path: ./embeddings-lib/ - - - name: Initialization of embeddings lib - run: | - mkdir embeddings\target - mkdir embeddings\target\release - mv .\embeddings-lib\build\* embeddings\target\release\ - rm -r ./embeddings-lib - - name: Replace standard libs to AVX2 if: ${{ inputs.USE_AVX2 == true }} run: | From 7c377a96a23fba904bced79ac4c9bf7c11293434 Mon Sep 17 00:00:00 2001 From: Sergey Nikolaev Date: Fri, 10 Apr 2026 08:47:36 +0700 Subject: [PATCH 2/5] Fix embeddings runtime path in CI tests --- .github/workflows/build_template.yml | 2 +- .github/workflows/test_template.yml | 16 ++++++++++++++++ .github/workflows/win_test_template.yml | 15 +++++++++++++++ testing.cmake | 3 ++- 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_template.yml b/.github/workflows/build_template.yml index 982b4328..e27289b8 100644 --- a/.github/workflows/build_template.yml +++ b/.github/workflows/build_template.yml @@ -49,7 +49,7 @@ on: artifact_list: required: false type: string - default: "build/columnar/lib_manticore_columnar_avx2.so build/columnar/lib_manticore_columnar_avx512.so build/columnar/lib_manticore_columnar.so build/secondary/lib_manticore_secondary_avx2.so build/secondary/lib_manticore_secondary_avx512.so build/secondary/lib_manticore_secondary.so build/knn/lib_manticore_knn.so build/knn/lib_manticore_knn_avx2.so build/knn/lib_manticore_knn_avx512.so build/share/manticore/modules/lib_manticore_knn_embeddings.so build/share/manticore/modules/lib_manticore_knn_embeddings.dylib build/share/modules/lib_manticore_knn_embeddings.dll build/_deps/manticore-build/src/searchd build/_deps/manticore-build/src/indexer build/_deps/manticore-build/src/indextool build/_deps/manticore-build/config/*.c build/_deps/manticore-build/config/*.h" + default: "build/columnar/lib_manticore_columnar_avx2.so build/columnar/lib_manticore_columnar_avx512.so build/columnar/lib_manticore_columnar.so build/secondary/lib_manticore_secondary_avx2.so build/secondary/lib_manticore_secondary_avx512.so build/secondary/lib_manticore_secondary.so build/knn/lib_manticore_knn.so build/knn/lib_manticore_knn_avx2.so build/knn/lib_manticore_knn_avx512.so build/_deps/manticore-build/src/searchd build/_deps/manticore-build/src/indexer build/_deps/manticore-build/src/indextool build/_deps/manticore-build/config/*.c build/_deps/manticore-build/config/*.h" HOMEBREW_PREFIX: required: false type: string diff --git a/.github/workflows/test_template.yml b/.github/workflows/test_template.yml index b9a45018..803597ab 100644 --- a/.github/workflows/test_template.yml +++ b/.github/workflows/test_template.yml @@ -21,6 +21,10 @@ on: required: false type: string default: "" + embeddings_artifact: + required: false + type: string + default: embeddings_linux_x86_64 timeout: required: false type: number @@ -104,6 +108,18 @@ jobs: name: ${{ steps.build_artifact.outputs.name }} path: . + - name: Download embeddings lib + uses: manticoresoftware/download_artifact_with_retries@v4 + with: + name: ${{ inputs.embeddings_artifact }} + path: ./embeddings-lib/ + + - name: Prepare embeddings runtime path + run: | + mkdir -p embeddings/target/release + mv ./embeddings-lib/build/* embeddings/target/release/ + rm -fr ./embeddings-lib + - name: Check out main cache before building uses: actions/cache@v4 with: diff --git a/.github/workflows/win_test_template.yml b/.github/workflows/win_test_template.yml index 8ca42dc5..6ee25221 100644 --- a/.github/workflows/win_test_template.yml +++ b/.github/workflows/win_test_template.yml @@ -14,6 +14,10 @@ on: artifact_name: required: true type: string + embeddings_artifact: + required: false + type: string + default: embeddings_windows_x86_64 MANTICORE_LOCATOR: required: false type: string @@ -90,6 +94,17 @@ jobs: with: name: build_windows_Debug_x64 path: . + - name: Download embeddings lib + uses: manticoresoftware/download_artifact_with_retries@v4 + with: + name: ${{ inputs.embeddings_artifact }} + path: ./embeddings-lib/ + - name: Prepare embeddings runtime path + shell: bash + run: | + mkdir -p embeddings/target/release + mv ./embeddings-lib/build/* embeddings/target/release/ + rm -fr ./embeddings-lib - name: Cache Windows bundle id: bundle_cache uses: actions/cache@v4 diff --git a/testing.cmake b/testing.cmake index 5d560624..6f568785 100644 --- a/testing.cmake +++ b/testing.cmake @@ -35,6 +35,7 @@ function ( special_ubertest_properties test ) set_property ( TEST "${test}" APPEND PROPERTY ENVIRONMENT "LIB_MANTICORE_COLUMNAR=$" ) set_property ( TEST "${test}" APPEND PROPERTY ENVIRONMENT "LIB_MANTICORE_SECONDARY=$" ) set_property ( TEST "${test}" APPEND PROPERTY ENVIRONMENT "LIB_MANTICORE_KNN=$" ) + set_property ( TEST "${test}" APPEND PROPERTY ENVIRONMENT "LIB_MANTICORE_KNN_EMBEDDINGS=$" ) endfunction () # this will switch off pure manticore-specific tests: google, api, keyword consistency and benches (we don't need them here) @@ -64,4 +65,4 @@ include ( "${columnar_BINARY_DIR}/manticore-get.cmake" ) # add manticore sources to the tree. All testing will be done on manticore side; necessary additional tests/properties will # be set by cb functions defined above. -FetchContent_MakeAvailable ( manticore ) \ No newline at end of file +FetchContent_MakeAvailable ( manticore ) From 7328806fcc745fe0e9a5f139c6a78e19a37232ff Mon Sep 17 00:00:00 2001 From: Sergey Nikolaev Date: Fri, 10 Apr 2026 09:17:59 +0700 Subject: [PATCH 3/5] Fix embeddings target visibility in CI --- .github/workflows/test.yml | 2 +- knn/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 39f92059..40ec13b5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -148,7 +148,7 @@ jobs: CTEST_CMAKE_GENERATOR: "Ninja Multi-Config" CTEST_CONFIGURATION_TYPE: Debug cache_key: build_windows_x64 - artifact_list: "build/columnar/Debug/lib_manticore_columnar.dll build/secondary/Debug/lib_manticore_secondary.dll build/knn/Debug/lib_manticore_knn.dll build/columnar/Debug/lib_manticore_columnar_avx2.dll build/secondary/Debug/lib_manticore_secondary_avx2.dll build/knn/Debug/lib_manticore_knn_avx2.dll build/columnar/Debug/lib_manticore_columnar_avx512.dll build/secondary/Debug/lib_manticore_secondary_avx512.dll build/knn/Debug/lib_manticore_knn_avx512.dll build/share/modules/lib_manticore_knn_embeddings.dll build/_deps/manticore-build/src/Debug/indexer.exe build/_deps/manticore-build/src/Debug/searchd.exe" # + artifact_list: "build/columnar/Debug/lib_manticore_columnar.dll build/secondary/Debug/lib_manticore_secondary.dll build/knn/Debug/lib_manticore_knn.dll build/columnar/Debug/lib_manticore_columnar_avx2.dll build/secondary/Debug/lib_manticore_secondary_avx2.dll build/knn/Debug/lib_manticore_knn_avx2.dll build/columnar/Debug/lib_manticore_columnar_avx512.dll build/secondary/Debug/lib_manticore_secondary_avx512.dll build/knn/Debug/lib_manticore_knn_avx512.dll build/_deps/manticore-build/src/Debug/indexer.exe build/_deps/manticore-build/src/Debug/searchd.exe" # MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }} embeddings_artifact: embeddings_windows_x86_64 diff --git a/knn/CMakeLists.txt b/knn/CMakeLists.txt index a093ae18..b002b2c9 100644 --- a/knn/CMakeLists.txt +++ b/knn/CMakeLists.txt @@ -50,7 +50,7 @@ if (PACK AND NOT EMBEDDINGS_LIB) endif() # Add embeddings library as a component to be included in packages -add_library ( embeddings MODULE IMPORTED ) +add_library ( embeddings MODULE IMPORTED GLOBAL ) set_target_properties ( embeddings PROPERTIES IMPORTED_LOCATION "${EMBEDDINGS_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/embeddings/") if ( NOT EXTERNAL_LIB ) # install only once, not from external_project build From 3f504091dc2164d873d93a01d5d9c2560e46b91e Mon Sep 17 00:00:00 2001 From: Sergey Nikolaev Date: Fri, 10 Apr 2026 10:30:17 +0700 Subject: [PATCH 4/5] Build Linux embeddings in main CI build --- .github/workflows/build_template.yml | 31 +++++++++++++++++++++++---- .github/workflows/nightly_no_avx2.yml | 18 +++------------- .github/workflows/test.yml | 10 +++++---- .github/workflows/test_template.yml | 16 -------------- cmake/citest.cmake | 5 +++++ 5 files changed, 41 insertions(+), 39 deletions(-) diff --git a/.github/workflows/build_template.yml b/.github/workflows/build_template.yml index e27289b8..c7cb91e1 100644 --- a/.github/workflows/build_template.yml +++ b/.github/workflows/build_template.yml @@ -44,8 +44,13 @@ on: type: string default: boost_nov22 embeddings_artifact: - required: true + required: false type: string + default: "" + build_embeddings_locally: + required: false + type: boolean + default: false artifact_list: required: false type: string @@ -124,6 +129,7 @@ jobs: cat CMakeLists.txt - name: Download embeddings lib + if: inputs.build_embeddings_locally != true uses: manticoresoftware/download_artifact_with_retries@v3 continue-on-error: true with: @@ -134,9 +140,20 @@ jobs: run: | # without adding the safe.directory cmake fails to do git log -1 --date=short --format=%cd bash /sysroot/root/entry_point.sh git config --global --add safe.directory /__w/columnar/columnar - mkdir -p embeddings/target/release - mv ./embeddings-lib/build/* embeddings/target/release/ - rm -fr ./embeddings-lib + if [ "${{ inputs.build_embeddings_locally }}" = "true" ]; then + apt-get update -y + apt-get install -y curl ca-certificates pkg-config protobuf-compiler build-essential libssl-dev + curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain stable + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + else + mkdir -p embeddings/target/release + mv ./embeddings-lib/build/* embeddings/target/release/ + rm -fr ./embeddings-lib + fi + + - name: Enable local embeddings build + if: inputs.build_embeddings_locally == true + run: echo "BUILD_EMBEDDINGS_LOCALLY=1" >> "$GITHUB_ENV" - name: Check out main cache before building uses: actions/cache@v4 @@ -159,6 +176,12 @@ jobs: - name: 🚀 Build run: ${{ inputs.cmake_command }} + - name: Sync locally built embeddings runtime + if: success() && inputs.build_embeddings_locally == true + run: | + mkdir -p embeddings/target/release + cp build/knn/embeddings/release/lib_manticore_knn_embeddings.* embeddings/target/release/ + - name: Move AVX2/AVX-512-optimized libraries if: success() && inputs.skip_avx2_move != true run: | diff --git a/.github/workflows/nightly_no_avx2.yml b/.github/workflows/nightly_no_avx2.yml index e712b421..48f4e0ff 100644 --- a/.github/workflows/nightly_no_avx2.yml +++ b/.github/workflows/nightly_no_avx2.yml @@ -78,26 +78,14 @@ jobs: echo "Set manticore_locator: (empty)" fi - embedding_linux_pack: - needs: commit_info - uses: ./.github/workflows/embedding_build_template.yml - strategy: - fail-fast: false - matrix: - distr: [linux] - arch: [x86_64] - name: ${{ matrix.distr }} ${{ matrix.arch }} embedding packing - with: - distr: ${{ matrix.distr }} - arch: ${{ matrix.arch }} - build_linux_release: name: Linux release build - needs: [embedding_linux_pack, check_branch] + needs: [check_branch] uses: ./.github/workflows/build_template.yml with: MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }} - embeddings_artifact: embeddings_linux_x86_64 + artifact_list: "build/columnar/lib_manticore_columnar_avx2.so build/columnar/lib_manticore_columnar_avx512.so build/columnar/lib_manticore_columnar.so build/secondary/lib_manticore_secondary_avx2.so build/secondary/lib_manticore_secondary_avx512.so build/secondary/lib_manticore_secondary.so build/knn/lib_manticore_knn.so build/knn/lib_manticore_knn_avx2.so build/knn/lib_manticore_knn_avx512.so embeddings/target/release/lib_manticore_knn_embeddings.so build/_deps/manticore-build/src/searchd build/_deps/manticore-build/src/indexer build/_deps/manticore-build/src/indextool build/_deps/manticore-build/config/*.c build/_deps/manticore-build/config/*.h" + build_embeddings_locally: true test_linux_release: name: Linux release mode tests diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 40ec13b5..6ab02301 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -123,20 +123,22 @@ jobs: build_linux_release: name: Linux release build - needs: [embedding_linux_pack, check_branch] + needs: [check_branch] uses: ./.github/workflows/build_template.yml with: MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }} - embeddings_artifact: embeddings_linux_x86_64 + artifact_list: "build/columnar/lib_manticore_columnar_avx2.so build/columnar/lib_manticore_columnar_avx512.so build/columnar/lib_manticore_columnar.so build/secondary/lib_manticore_secondary_avx2.so build/secondary/lib_manticore_secondary_avx512.so build/secondary/lib_manticore_secondary.so build/knn/lib_manticore_knn.so build/knn/lib_manticore_knn_avx2.so build/knn/lib_manticore_knn_avx512.so embeddings/target/release/lib_manticore_knn_embeddings.so build/_deps/manticore-build/src/searchd build/_deps/manticore-build/src/indexer build/_deps/manticore-build/src/indextool build/_deps/manticore-build/config/*.c build/_deps/manticore-build/config/*.h" + build_embeddings_locally: true build_linux_debug: name: Linux debug build - needs: [embedding_linux_pack, check_branch] + needs: [check_branch] uses: ./.github/workflows/build_template.yml with: CTEST_CONFIGURATION_TYPE: "Debug" MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }} - embeddings_artifact: embeddings_linux_x86_64 + artifact_list: "build/columnar/lib_manticore_columnar_avx2.so build/columnar/lib_manticore_columnar_avx512.so build/columnar/lib_manticore_columnar.so build/secondary/lib_manticore_secondary_avx2.so build/secondary/lib_manticore_secondary_avx512.so build/secondary/lib_manticore_secondary.so build/knn/lib_manticore_knn.so build/knn/lib_manticore_knn_avx2.so build/knn/lib_manticore_knn_avx512.so embeddings/target/release/lib_manticore_knn_embeddings.so build/_deps/manticore-build/src/searchd build/_deps/manticore-build/src/indexer build/_deps/manticore-build/src/indextool build/_deps/manticore-build/config/*.c build/_deps/manticore-build/config/*.h" + build_embeddings_locally: true build_windows: name: Windows x64 build diff --git a/.github/workflows/test_template.yml b/.github/workflows/test_template.yml index 803597ab..b9a45018 100644 --- a/.github/workflows/test_template.yml +++ b/.github/workflows/test_template.yml @@ -21,10 +21,6 @@ on: required: false type: string default: "" - embeddings_artifact: - required: false - type: string - default: embeddings_linux_x86_64 timeout: required: false type: number @@ -108,18 +104,6 @@ jobs: name: ${{ steps.build_artifact.outputs.name }} path: . - - name: Download embeddings lib - uses: manticoresoftware/download_artifact_with_retries@v4 - with: - name: ${{ inputs.embeddings_artifact }} - path: ./embeddings-lib/ - - - name: Prepare embeddings runtime path - run: | - mkdir -p embeddings/target/release - mv ./embeddings-lib/build/* embeddings/target/release/ - rm -fr ./embeddings-lib - - name: Check out main cache before building uses: actions/cache@v4 with: diff --git a/cmake/citest.cmake b/cmake/citest.cmake index e1a84dea..401e1ca6 100644 --- a/cmake/citest.cmake +++ b/cmake/citest.cmake @@ -14,6 +14,7 @@ set ( SEARCHD_CLI_EXTRA "$ENV{SEARCHD_CLI_EXTRA}" ) set ( WITH_COVERAGE "$ENV{WITH_COVERAGE}" ) set ( NO_TESTS "$ENV{NO_TESTS}" ) set ( NO_BUILD "$ENV{NO_BUILD}" ) +set ( BUILD_EMBEDDINGS_LOCALLY "$ENV{BUILD_EMBEDDINGS_LOCALLY}" ) set_property ( GLOBAL PROPERTY Label P$ENV{CI_PIPELINE_ID} J$ENV{CI_JOB_ID} ) # how may times try the test before it is considered failed @@ -49,6 +50,10 @@ if (SEARCHD_CLI_EXTRA) list ( APPEND CONFIG_OPTIONS "SEARCHD_CLI_EXTRA=${SEARCHD_CLI_EXTRA}" ) endif () +if (BUILD_EMBEDDINGS_LOCALLY) + list ( APPEND CONFIG_OPTIONS "BUILD_EMBEDDINGS_LOCALLY=ON" ) +endif () + set ( CTEST_START_WITH_EMPTY_BINARY_DIRECTORY TRUE ) #ctest_empty_binary_directory(${CTEST_BINARY_DIRECTORY}) From e171530bebd8c277d06faa81f0be55ed706661af Mon Sep 17 00:00:00 2001 From: Sergey Nikolaev Date: Fri, 10 Apr 2026 11:48:18 +0700 Subject: [PATCH 5/5] Use compatible ONNX runtime in Linux CI builds --- .github/workflows/build_template.yml | 24 ++++++++++++++++++- .../workflows/embedding_build_template.yml | 14 ++++++++--- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_template.yml b/.github/workflows/build_template.yml index c7cb91e1..3526c5f2 100644 --- a/.github/workflows/build_template.yml +++ b/.github/workflows/build_template.yml @@ -142,9 +142,31 @@ jobs: git config --global --add safe.directory /__w/columnar/columnar if [ "${{ inputs.build_embeddings_locally }}" = "true" ]; then apt-get update -y - apt-get install -y curl ca-certificates pkg-config protobuf-compiler build-essential libssl-dev + apt-get install -y curl ca-certificates pkg-config protobuf-compiler build-essential libssl-dev unzip curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain stable echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + + ort_version="1.24.2" + case "${{ inputs.arch }}" in + x86_64) + ort_asset="onnxruntime-linux-x64-static_lib-${ort_version}-glibc2_17.zip" + ort_dir="/tmp/ort/onnxruntime-linux-x64-static_lib-${ort_version}-glibc2_17" + ;; + aarch64) + ort_asset="onnxruntime-linux-aarch64-static_lib-${ort_version}-glibc2_17.zip" + ort_dir="/tmp/ort/onnxruntime-linux-aarch64-static_lib-${ort_version}-glibc2_17" + ;; + *) + echo "Unsupported arch for local embeddings build: ${{ inputs.arch }}" + exit 1 + ;; + esac + + rm -rf /tmp/ort + mkdir -p /tmp/ort + curl -fsSL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${ort_version}/${ort_asset}" -o /tmp/ort/ort.zip + unzip -q /tmp/ort/ort.zip -d /tmp/ort + echo "ORT_LIB_LOCATION=${ort_dir}/lib" >> "$GITHUB_ENV" else mkdir -p embeddings/target/release mv ./embeddings-lib/build/* embeddings/target/release/ diff --git a/.github/workflows/embedding_build_template.yml b/.github/workflows/embedding_build_template.yml index 023853db..b4ce9dae 100644 --- a/.github/workflows/embedding_build_template.yml +++ b/.github/workflows/embedding_build_template.yml @@ -233,10 +233,12 @@ jobs: # Set Docker image based on architecture if [[ "${{ inputs.arch }}" == "aarch64" ]]; then docker_image="ghcr.io/manticoresoftware/rust-min-libc:aarch64-rust1.94.1-glibc2.27-openssl1.1.1k" - extra="" + ort_asset="onnxruntime-linux-aarch64-static_lib-1.24.2-glibc2_17.zip" + ort_dir="onnxruntime-linux-aarch64-static_lib-1.24.2-glibc2_17" else docker_image="ghcr.io/manticoresoftware/rust-min-libc:amd64-rust1.94.1-glibc2.27-openssl1.1.1k" - extra="" + ort_asset="onnxruntime-linux-x64-static_lib-1.24.2-glibc2_17.zip" + ort_dir="onnxruntime-linux-x64-static_lib-1.24.2-glibc2_17" fi docker run --rm \ @@ -250,7 +252,13 @@ jobs: -e GIT_TIMESTAMP_ID="${{ steps.git_meta.outputs.timestamp }}" \ --entrypoint bash \ "${docker_image}" \ - -c "apt-get update -y && apt-get install -y gcc g++ protobuf-compiler && ${extra} cargo build ${{ steps.params.outputs.args }}" + -c "set -euo pipefail && \ + apt-get update -y && apt-get install -y gcc g++ protobuf-compiler curl unzip && \ + rm -rf /tmp/ort && mkdir -p /tmp/ort && \ + curl -fsSL https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.24.2/${ort_asset} -o /tmp/ort/ort.zip && \ + unzip -q /tmp/ort/ort.zip -d /tmp/ort && \ + export ORT_LIB_LOCATION=/tmp/ort/${ort_dir}/lib && \ + cargo build ${{ steps.params.outputs.args }}" sudo chown -hR $(id -u):$(id -g) ${{ steps.vars.outputs.lib_dir }} sudo chown -hR $(id -u):$(id -g) .cargo-cache