diff --git a/.github/workflows/build_template.yml b/.github/workflows/build_template.yml deleted file mode 100644 index 64003b95..00000000 --- a/.github/workflows/build_template.yml +++ /dev/null @@ -1,216 +0,0 @@ -name: Build Template - -on: - workflow_call: - inputs: - arch: - required: false - type: string - default: x86_64 - DISTR: - required: false - type: string - default: jammy - CTEST_CMAKE_GENERATOR: - required: false - type: string - default: Ninja - CTEST_CONFIGURATION_TYPE: - required: false - type: string - default: RelWithDebInfo - version: - required: false - type: string - default: "" - package_version: - required: false - type: string - default: "" - rpm_package_version: - required: false - type: string - default: "" - cmake_command: - required: false - type: string - default: "ctest -VV -S cmake/citest.cmake --no-compress-output" - cache_key: - required: false - type: string - default: build_linux_x86_64 - boost: - required: false - type: string - default: boost_nov22 - embeddings_artifact: - required: true - type: string - artifact_list: - required: false - type: string - default: "build/columnar/lib_manticore_columnar_avx2.so build/columnar/lib_manticore_columnar_avx512.so build/columnar/lib_manticore_columnar.so build/secondary/lib_manticore_secondary_avx2.so build/secondary/lib_manticore_secondary_avx512.so build/secondary/lib_manticore_secondary.so build/knn/lib_manticore_knn.so build/knn/lib_manticore_knn_avx2.so build/knn/lib_manticore_knn_avx512.so build/_deps/manticore-build/src/searchd build/_deps/manticore-build/src/indexer build/_deps/manticore-build/src/indextool build/_deps/manticore-build/config/*.c build/_deps/manticore-build/config/*.h" - HOMEBREW_PREFIX: - required: false - type: string - default: "" - MANTICORE_LOCATOR: - required: false - type: string - default: "" - skip_avx2_move: - required: false - type: boolean - default: false - artifact_name: - required: false - type: string - default: "" - -jobs: - build: - runs-on: ubuntu-22.04 - defaults: - run: - shell: bash - timeout-minutes: 30 - container: - image: manticoresearch/external_toolchain:clang16_cmake3263 - env: - CACHEB: "../cache" - VERBOSE: 1 - DIAGNOSTIC: 1 - NO_TESTS: 1 - DISTR: ${{ inputs.DISTR }} - boost: ${{ inputs.boost }} - sysroot: roots_nov22 - arch: ${{ inputs.arch }} - CTEST_CMAKE_GENERATOR: ${{ inputs.CTEST_CMAKE_GENERATOR }} - CTEST_CONFIGURATION_TYPE: ${{ inputs.CTEST_CONFIGURATION_TYPE }} - SYSROOT_URL: https://repo.manticoresearch.com/repository/sysroots - HOMEBREW_PREFIX: ${{ inputs.HOMEBREW_PREFIX }} - MANTICORE_LOCATOR: ${{ inputs.MANTICORE_LOCATOR }} - PACKAGE_VERSION: ${{ inputs.package_version }} - RPM_PACKAGE_VERSION: ${{ inputs.rpm_package_version }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - token: ${{ secrets.GITHUB_TOKEN }} - set-safe-directory: true - - - name: Print Manticore Locator - run: | - echo "Using MANTICORE_LOCATOR: '${{ inputs.MANTICORE_LOCATOR }}'" - if [[ -n "${{ inputs.MANTICORE_LOCATOR }}" ]]; then - echo "Manticore locator is set and will be used for building" - else - echo "Manticore locator is empty, using default manticore sources" - fi - - - name: Determine branch name for cache - id: branch - run: | - if [ "${{ github.event_name }}" = "pull_request" ]; then - echo "branch_name=${{ github.head_ref }}" >> $GITHUB_OUTPUT - else - echo "branch_name=${{ github.ref_name }}" >> $GITHUB_OUTPUT - fi - - - name: Patch version - if: inputs.version != '' - run: | - sed -i "s/0\.0\.0/${{ inputs.version }}/g" CMakeLists.txt - cat CMakeLists.txt - - - name: Download embeddings lib - uses: manticoresoftware/download_artifact_with_retries@v3 - continue-on-error: true - with: - name: ${{ inputs.embeddings_artifact }} - path: ./embeddings-lib/ - - - name: Initialization - run: | # without adding the safe.directory cmake fails to do git log -1 --date=short --format=%cd - bash /sysroot/root/entry_point.sh - git config --global --add safe.directory /__w/columnar/columnar - mkdir -p embeddings/target/release - mv ./embeddings-lib/build/* embeddings/target/release/ - rm -fr ./embeddings-lib - - - name: Check out main cache before building - uses: actions/cache@v4 - with: - path: cache - enableCrossOsArchive: true - key: ${{ inputs.cache_key }}_${{ steps.branch.outputs.branch_name }} - restore-keys: | - ${{ inputs.cache_key }}_master - - - name: Check out deps cache before building - uses: actions/cache@v4 - with: - path: build/_deps/cache - enableCrossOsArchive: true - key: ${{ inputs.cache_key }}_deps_${{ steps.branch.outputs.branch_name }} - restore-keys: | - ${{ inputs.cache_key }}_deps_master - - - name: 🚀 Build - run: ${{ inputs.cmake_command }} - - - name: Move AVX2/AVX-512-optimized libraries - if: success() && inputs.skip_avx2_move != true - run: | - if [ -d build/avx2/share/manticore/modules ]; then - echo "Move linux based AVX2 libs" - mv build/avx2/share/manticore/modules/lib_manticore_columnar_avx2.so build/columnar/ - mv build/avx2/share/manticore/modules/lib_manticore_secondary_avx2.so build/secondary/ - mv build/avx2/share/manticore/modules/lib_manticore_knn_avx2.so build/knn/ - ls build/columnar/ - ls build/secondary/ - ls build/knn/ - fi - if [ -d build/avx512/share/manticore/modules ]; then - echo "Move linux based AVX-512 libs" - mv build/avx512/share/manticore/modules/lib_manticore_columnar_avx512.so build/columnar/ - mv build/avx512/share/manticore/modules/lib_manticore_secondary_avx512.so build/secondary/ - mv build/avx512/share/manticore/modules/lib_manticore_knn_avx512.so build/knn/ - ls build/columnar/ - ls build/secondary/ - ls build/knn/ - fi - if [ -d build/avx2/share/modules ]; then - echo "Move windows based AVX2 libs" - mv build/avx2/share/modules/lib_manticore_columnar_avx2.dll build/columnar/Debug/ - mv build/avx2/share/modules/lib_manticore_secondary_avx2.dll build/secondary/Debug/ - mv build/avx2/share/modules/lib_manticore_knn_avx2.dll build/knn/Debug/ - ls build/columnar/Debug/ - ls build/secondary/Debug/ - ls build/knn/Debug/ - fi - if [ -d build/avx512/share/modules ]; then - echo "Move windows based AVX-512 libs" - mv build/avx512/share/modules/lib_manticore_columnar_avx512.dll build/columnar/Debug/ - mv build/avx512/share/modules/lib_manticore_secondary_avx512.dll build/secondary/Debug/ - mv build/avx512/share/modules/lib_manticore_knn_avx512.dll build/knn/Debug/ - ls build/columnar/Debug/ - ls build/secondary/Debug/ - ls build/knn/Debug/ - fi - - - name: Set artifact name - id: artifact_name - run: | - if [ -n "${{ inputs.artifact_name }}" ]; then - echo "name=${{ inputs.artifact_name }}" >> $GITHUB_OUTPUT - else - echo "name=build_${{ inputs.DISTR }}_${{ inputs.CTEST_CONFIGURATION_TYPE }}_${{ inputs.arch }}" >> $GITHUB_OUTPUT - fi - - - name: Upload build artifacts - if: success() - uses: manticoresoftware/upload_artifact_with_retries@v4 - with: - name: ${{ steps.artifact_name.outputs.name }} - path: ${{ inputs.artifact_list }} \ No newline at end of file diff --git a/.github/workflows/clt_tests.yml b/.github/workflows/clt_tests.yml deleted file mode 100644 index 5f6dfcb2..00000000 --- a/.github/workflows/clt_tests.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: CLT tests -on: - workflow_call: - inputs: - docker_image: - required: true - type: string - description: "Docker image to use for tests" - artifact_name: - required: false - type: string - description: "Name of the docker image artifact" - repository: - required: false - type: string - description: "Repository to checkout" - ref: - required: false - type: string - description: "Ref to checkout" - continue_on_error: - required: false - type: boolean - description: "Continue on error" - default: false - secrets: - OPENAI_API_KEY: - required: true - description: "OpenAI API key for CLT tests" - VOYAGE_API_KEY: - required: true - description: "Voyage API key for CLT tests" - JINA_API_KEY: - required: true - description: "Jina API key for CLT tests" - -jobs: - clt: - name: CLT - runs-on: ubuntu-22.04 - timeout-minutes: 30 - continue-on-error: ${{ inputs.continue_on_error }} - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }} - JINA_API_KEY: ${{ secrets.JINA_API_KEY }} - strategy: - fail-fast: false - matrix: - test-suite: - - name: MCL - test_prefix: test/clt-tests/mcl/ - - steps: - - uses: manticoresoftware/clt@0.7.3 - with: - artifact: ${{ inputs.artifact_name }} - image: ${{ inputs.docker_image }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - test_prefix: ${{ matrix.test-suite.test_prefix }} - comment_mode: failures - run_args: "-e OPENAI_API_KEY -e VOYAGE_API_KEY -e JINA_API_KEY" - ui_host: "https://clt.manticoresearch.com" diff --git a/.github/workflows/embedding_build_template.yml b/.github/workflows/embedding_build_template.yml deleted file mode 100644 index 1e1c2dc3..00000000 --- a/.github/workflows/embedding_build_template.yml +++ /dev/null @@ -1,496 +0,0 @@ -name: Embeddings Build Template - -on: - workflow_call: - inputs: - distr: - required: true - type: string - arch: - required: true - type: string - setup: - required: false - type: string - default: "" - -jobs: - # Quality check job - runs tests and clippy before build - quality-check: - runs-on: ubuntu-24.04 - defaults: - run: - shell: bash - timeout-minutes: 15 - steps: - - uses: actions/checkout@v4 - - - name: Setup Rust toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - components: clippy, rustfmt - override: true - - - name: Install protoc - uses: arduino/setup-protoc@v3 - with: - version: '29.3' - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Cache cargo registry - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - key: cargo-registry-${{ runner.os }}-${{ hashFiles('embeddings/Cargo.lock') }} - restore-keys: cargo-registry-${{ runner.os }}- - - - name: Cache cargo build target - uses: actions/cache@v4 - with: - path: ./embeddings/target - key: cargo-target-${{ runner.os }}-${{ hashFiles('embeddings/Cargo.lock') }}-${{ hashFiles('embeddings/src/**/*.rs') }} - restore-keys: | - cargo-target-${{ runner.os }}-${{ hashFiles('embeddings/Cargo.lock') }}- - cargo-target-${{ runner.os }}- - - - name: Cache ML test models - uses: actions/cache@v4 - with: - path: ./embeddings/.cache/manticore - key: ml-models-${{ runner.os }}-${{ hashFiles('embeddings/src/model/local_test.rs') }} - restore-keys: ml-models-${{ runner.os }}- - - - name: Run Clippy (strict mode) - run: cargo clippy --lib --manifest-path ./embeddings/Cargo.toml -- -D warnings - - - name: Check code formatting - run: cargo fmt --check --manifest-path ./embeddings/Cargo.toml - - - name: Run unit tests - run: cargo test --lib --verbose --manifest-path ./embeddings/Cargo.toml - env: - RUST_BACKTRACE: 1 - - - name: Run OpenAI integration tests (if API key available) - run: | - cd ./embeddings - if [ -n "${{ secrets.OPENAI_API_KEY }}" ]; then - echo "✅ Running OpenAI integration tests..." - cargo test --lib openai_test::tests --verbose - else - echo "⏭️ Skipping OpenAI tests - OPENAI_API_KEY not configured" - fi - - if [ -n "${{ secrets.VOYAGE_API_KEY }}" ]; then - echo "✅ Running Voyager integration tests..." - cargo test --lib voyager_test::tests --verbose - else - echo "⏭️ Skipping Voyager tests - VOYAGE_API_KEY not configured" - fi - - if [ -n "${{ secrets.JINA_API_KEY }}" ]; then - echo "✅ Running Jina integration tests..." - cargo test --lib jina_test::tests --verbose - else - echo "⏭️ Skipping Jina tests - JINA_API_KEY not configured" - fi - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }} - JINA_API_KEY: ${{ secrets.JINA_API_KEY }} - RUST_BACKTRACE: 1 - - - name: Clean build target bloat - if: always() - run: | - rm -rf ./embeddings/target/debug/incremental - rm -rf ./embeddings/target/debug/.fingerprint - find ./embeddings/target/debug -name "*.d" -delete 2>/dev/null || true - - - name: Quality check summary - run: | - echo "✅ Quality Check Summary:" - echo "- Clippy: Passed (zero warnings enforced)" - echo "- Format: Checked and compliant" - echo "- Tests: All unit tests passed" - echo "- OpenAI: $([ -n "${{ secrets.OPENAI_API_KEY }}" ] && echo "Integration tested" || echo "Skipped (no API key)")" - echo "- Voyage: $([ -n "${{ secrets.VOYAGE_API_KEY }}" ] && echo "Integration tested" || echo "Skipped (no API key)")" - echo "- Jina: $([ -n "${{ secrets.JINA_API_KEY }}" ] && echo "Integration tested" || echo "Skipped (no API key)")" - - build: - needs: quality-check # Only run build after quality check passes - runs-on: ${{ - (inputs.distr == 'linux' && inputs.arch == 'aarch64' && 'ubuntu-24.04-arm') || - (inputs.distr == 'linux' && 'ubuntu-24.04') || - (inputs.distr == 'macos' && 'macos-14') || - (inputs.distr == 'windows' && 'windows-2022') || - 'ubuntu-24.04' }} - defaults: - run: - shell: bash - timeout-minutes: 30 - steps: - - name: Setup - if: ${{ inputs.setup }} - run: | - ${{ inputs.setup }} - - - name: Check distribution type - id: vars - run: | - case ${{ inputs.distr }} in - linux) - target="unknown-linux-gnu" - ;; - macos) - target="apple-darwin" - ;; - windows) - target="pc-windows-msvc" - ;; - *) - echo "Unsupported distribution type: ${{ inputs.distr }}" - exit 1 - ;; - esac - target="${{ inputs.arch }}-${target}" - echo "target=${target}" >> $GITHUB_OUTPUT - - lib_dir="./embeddings/target/${target}/release" - echo "lib_dir=${lib_dir}" >> $GITHUB_OUTPUT - - - uses: actions/checkout@v4 - - - name: Setup Rust toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - target: ${{ steps.vars.outputs.target }} - override: true - - - name: Install protoc - if: ${{ inputs.distr != 'linux' }} - uses: arduino/setup-protoc@v3 - with: - version: '29.3' - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract git metadata - id: git_meta - run: | - git config --global --add safe.directory "$(pwd)" - - commit="$(git log -1 --format=%h)" - timestamp="$(git log -1 --date=format-local:%y%m%d%H --format=%cd)" - - if [[ -z "${commit}" || -z "${timestamp}" ]]; then - echo "Failed to extract git metadata for embeddings versioning" - exit 1 - fi - - echo "commit=${commit}" >> "$GITHUB_OUTPUT" - echo "timestamp=${timestamp}" >> "$GITHUB_OUTPUT" - - - name: Prepare build args - id: params - run: | - # Enable platform-specific BLAS acceleration: - # - macOS: Apple Accelerate framework (always available) - # - Linux/Windows x86_64: Intel MKL (downloaded at build time) - # - Linux aarch64: no extra BLAS (gemm crate uses NEON via runtime detection) - features="" - if [[ "${{ inputs.distr }}" == "macos" ]]; then - features="--features accelerate" - elif [[ "${{ inputs.arch }}" == "x86_64" ]]; then - features="--features mkl" - fi - - args=( - "--target" - "${{ steps.vars.outputs.target }}" - "--lib" - "--release" - "--manifest-path" - "./embeddings/Cargo.toml" - ${features} - ) - echo "args=${args[@]}" >> $GITHUB_OUTPUT - - - name: Cache cargo registry for Docker build - if: ${{ inputs.distr == 'linux' }} - uses: actions/cache@v4 - with: - path: | - .cargo-cache/registry - .cargo-cache/git - key: emb-build-cargo-${{ runner.os }}-${{ inputs.arch }}-${{ hashFiles('embeddings/Cargo.lock') }} - restore-keys: emb-build-cargo-${{ runner.os }}-${{ inputs.arch }}- - - - name: Build for Linux - if: ${{ inputs.distr == 'linux' }} - run: | - # Set Docker image based on architecture - # Download glibc2_17-compatible ORT static lib (avoids __isoc23_strtoll from pyke.io builds) - ORT_VERSION="1.24.2" - if [[ "${{ inputs.arch }}" == "aarch64" ]]; then - docker_image="ghcr.io/manticoresoftware/rust-min-libc:aarch64-rust1.94.1-glibc2.27-openssl1.1.1k" - ort_asset="onnxruntime-linux-aarch64-static_lib-${ORT_VERSION}-glibc2_17" - else - docker_image="ghcr.io/manticoresoftware/rust-min-libc:amd64-rust1.94.1-glibc2.27-openssl1.1.1k" - ort_asset="onnxruntime-linux-x64-static_lib-${ORT_VERSION}-glibc2_17" - fi - - curl -sL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${ORT_VERSION}/${ort_asset}.zip" -o /tmp/ort.zip - unzip -q /tmp/ort.zip -d /tmp/ort - - docker run --rm \ - -v $(pwd):/src \ - -v $(pwd)/.cargo-cache:/cargo-cache \ - -v /tmp/ort/${ort_asset}/lib:/ort-lib \ - -w /src \ - -u root \ - -e CARGO_HOME=/cargo-cache \ - -e ORT_LIB_PATH=/ort-lib \ - -e MKLROOT=/opt/intel/oneapi/mkl/latest \ - -e GIT_COMMIT_ID="${{ steps.git_meta.outputs.commit }}" \ - -e GIT_TIMESTAMP_ID="${{ steps.git_meta.outputs.timestamp }}" \ - --entrypoint bash \ - "${docker_image}" \ - -c "cargo build ${{ steps.params.outputs.args }}" - sudo chown -hR $(id -u):$(id -g) ${{ steps.vars.outputs.lib_dir }} - sudo chown -hR $(id -u):$(id -g) .cargo-cache - - - - name: Download ONNX Runtime static library - if: ${{ inputs.distr == 'macos' || inputs.distr == 'windows' }} - shell: bash - run: | - ort_version="1.24.2" - if [[ "${{ inputs.distr }}" == "macos" ]]; then - if [[ "${{ inputs.arch }}" == "aarch64" ]]; then - ort_name="onnxruntime-osx-arm64-static_lib-${ort_version}" - else - ort_name="onnxruntime-osx-x86_64-static_lib-${ort_version}" - fi - curl -sL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${ort_version}/${ort_name}.zip" -o /tmp/ort.zip - unzip -q /tmp/ort.zip -d /tmp/ort - echo "ORT_LIB_LOCATION=/tmp/ort/${ort_name}/lib" >> $GITHUB_ENV - else - if [[ "${{ inputs.arch }}" == "aarch64" ]]; then - ort_name="onnxruntime-win-arm64-static_lib-MD-Release-${ort_version}" - else - ort_name="onnxruntime-win-x64-static_lib-MD-Release-${ort_version}" - fi - curl -sL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${ort_version}/${ort_name}.tar.bz2" -o ort.tar.bz2 - mkdir -p ort-lib - tar xjf ort.tar.bz2 -C ort-lib - echo "ORT_LIB_LOCATION=${{ github.workspace }}/ort-lib/${ort_name}/lib" >> $GITHUB_ENV - fi - - - name: Build native - if: ${{ inputs.distr == 'windows' || inputs.distr == 'macos' }} - run: | - rustup target add ${{ steps.vars.outputs.target }} - cargo build ${{ steps.params.outputs.args }} - env: - GIT_COMMIT_ID: ${{ steps.git_meta.outputs.commit }} - GIT_TIMESTAMP_ID: ${{ steps.git_meta.outputs.timestamp }} - - - run: | - mkdir build - lib_dir="${{ steps.vars.outputs.lib_dir }}" - ls -la $lib_dir/* - find $lib_dir -type f \( -name "libmanticore*.so" -o -name "libmanticore*.dylib" -o -name "manticore*.dll" \) -exec cp {} build/ \; - - # Rename libmanticore*.so to lib_manticore*.so - for file in build/libmanticore*.so; do - if [ -f "$file" ]; then - base=$(basename "$file") - newbase=$(echo "$base" | sed 's/^libmanticore/lib_manticore/') - mv "$file" "build/$newbase" - echo "Renamed $file to build/$newbase" - fi - done - - # Rename libmanticore*.dylib to lib_manticore*.dylib - for file in build/libmanticore*.dylib; do - if [ -f "$file" ]; then - base=$(basename "$file") - newbase=$(echo "$base" | sed 's/^libmanticore/lib_manticore/') - mv "$file" "build/$newbase" - echo "Renamed $file to build/$newbase" - fi - done - - # Rename manticore*.dll to lib_manticore*.dll - for file in build/manticore*.dll; do - if [ -f "$file" ]; then - base=$(basename "$file") - newbase="lib_$base" - mv "$file" "build/$newbase" - echo "Renamed $file to build/$newbase" - fi - done - - if [[ "${{ inputs.distr }}" == "windows" && "${{ inputs.arch }}" != "aarch64" ]]; then - echo "=== Package Windows runtime DLL dependencies ===" - openmp_redist_version="2025.3.3.31" - openmp_redist_dir=".openmp-redist" - for dll in libiomp5md.dll; do - dep=$(find ./embeddings/target "$HOME/.cargo" -type f -iname "$dll" 2>/dev/null | head -1) - if [[ -z "$dep" ]]; then - echo "$dll was not found in the Cargo build tree; downloading Intel OpenMP redist ${openmp_redist_version}" - rm -rf "$openmp_redist_dir" - mkdir -p "$openmp_redist_dir/pkg" - curl -fsSL \ - "https://api.nuget.org/v3-flatcontainer/intelopenmp.redist.win/${openmp_redist_version}/intelopenmp.redist.win.${openmp_redist_version}.nupkg" \ - -o "$openmp_redist_dir/intelopenmp.redist.win.zip" - powershell -NoProfile -NonInteractive -Command \ - "Expand-Archive -LiteralPath '$(cygpath -w "$openmp_redist_dir/intelopenmp.redist.win.zip")' -DestinationPath '$(cygpath -w "$openmp_redist_dir/pkg")' -Force" - dep=$(find "$openmp_redist_dir/pkg" -type f -iname "$dll" | head -1) - fi - - if [[ -z "$dep" ]]; then - echo "Required Windows runtime dependency was not found after redist download: $dll" - find ./embeddings/target "$HOME/.cargo" "$openmp_redist_dir" -type f -iname "*.dll" 2>/dev/null | sort || true - exit 1 - fi - - cp "$dep" build/ - echo "Packaged $dll from $dep" - done - elif [[ "${{ inputs.distr }}" == "windows" ]]; then - echo "Skipping Intel OpenMP runtime packaging for Windows ${{ inputs.arch }}; Intel OpenMP redist currently ships win-x64 DLLs only" - fi - - - name: Verify Linux library compatibility - if: ${{ inputs.distr == 'linux' }} - run: | - lib=$(find build -name "lib_manticore_knn_embeddings.so" | head -1) - if [ -z "$lib" ]; then - echo "Embeddings library not found in build/, skipping check" - exit 0 - fi - - echo "=== Checking $lib ===" - failed=0 - - # Check GLIBC — must not exceed 2.27 - max_glibc=$(objdump -T "$lib" | grep -oE 'GLIBC_[0-9.]+' | sort -u -t. -k1,1 -k2,2n | tail -1) - echo "Max GLIBC: $max_glibc" - glibc_minor=$(echo "$max_glibc" | grep -oE '[0-9]+$') - if [ -n "$glibc_minor" ] && [ "$glibc_minor" -gt 27 ]; then - echo "FAIL: GLIBC requirement $max_glibc exceeds 2.27" - failed=1 - fi - - # Check GLIBCXX — must have none (libstdc++ should be static) - glibcxx=$(objdump -T "$lib" | grep -c 'GLIBCXX' || true) - echo "GLIBCXX dynamic symbols: $glibcxx" - if [ "$glibcxx" -gt 0 ]; then - echo "FAIL: Found GLIBCXX dynamic symbols — libstdc++ not statically linked" - objdump -T "$lib" | grep 'GLIBCXX' | head -5 - failed=1 - fi - - # Check __isoc23 — must have none - isoc23=$(nm -D "$lib" 2>/dev/null | grep -c '__isoc23' || true) - echo "__isoc23 symbols: $isoc23" - if [ "$isoc23" -gt 0 ]; then - echo "FAIL: Found __isoc23 symbols — ORT prebuilt has wrong glibc target" - failed=1 - fi - - if [ "$failed" -eq 1 ]; then - echo "" - echo "Library compatibility check FAILED" - exit 1 - fi - echo "All checks passed" - - - name: Verify Windows embeddings DLL loading - if: ${{ inputs.distr == 'windows' }} - shell: powershell - run: | - $ErrorActionPreference = "Stop" - $dll = "build\lib_manticore_knn_embeddings.dll" - - if (-not (Test-Path $dll)) { - throw "Embeddings DLL was not found at $dll" - } - - if ("${{ inputs.arch }}" -ne "aarch64" -and -not (Test-Path "build\libiomp5md.dll")) { - throw "Required runtime DLL was not packaged: build\libiomp5md.dll" - } - - Write-Host "=== Packaged embeddings files ===" - Get-ChildItem -Force build | Format-List Name,Length,FullName - - Write-Host "=== dumpbin /dependents ===" - $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" - $vsInstall = $null - if (Test-Path $vswhere) { - $vsInstall = & $vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath - } - if (-not $vsInstall) { - throw "Visual Studio installation was not found by vswhere" - } - - $vcvars = Join-Path $vsInstall "VC\Auxiliary\Build\vcvars64.bat" - if (-not (Test-Path $vcvars)) { - throw "vcvars64.bat was not found at $vcvars" - } - - cmd /c "`"$vcvars`" >nul && dumpbin /dependents `"$dll`"" - if ($LASTEXITCODE -ne 0) { - throw "dumpbin /dependents failed" - } - - cmd /c "`"$vcvars`" >nul && dumpbin /exports `"$dll`" | findstr GetLibFuncs" - if ($LASTEXITCODE -ne 0) { - throw "GetLibFuncs export was not found" - } - - if ("${{ inputs.arch }}" -eq "aarch64") { - Write-Host "Skipping direct LoadLibraryW probe for Windows aarch64; GitHub runner is x64 and cannot load an ARM64 DLL" - exit 0 - } - - Write-Host "=== Direct LoadLibraryW probe ===" - $buildDir = (Resolve-Path "build").Path - $env:PATH = "$buildDir;$env:PATH" - - Add-Type -TypeDefinition @' - using System; - using System.Runtime.InteropServices; - - public static class NativeDllProbe - { - [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] - public static extern IntPtr LoadLibraryW(string lpFileName); - } - '@ - - $fullDll = (Resolve-Path $dll).Path - $handle = [NativeDllProbe]::LoadLibraryW($fullDll) - if ($handle -eq [IntPtr]::Zero) { - $code = [Runtime.InteropServices.Marshal]::GetLastWin32Error() - $message = (New-Object ComponentModel.Win32Exception($code)).Message - throw "LoadLibraryW($fullDll) failed: $code $message" - } - - Write-Host "LoadLibraryW($fullDll) succeeded: $handle" - - - name: Install current Bash on macOS - if: inputs.distr == 'macos' - run: brew install bash - - - name: Upload build artifacts - if: success() - uses: manticoresoftware/upload_artifact_with_retries@v4 - with: - name: embeddings_${{ inputs.distr }}_${{ inputs.arch }} - path: ./build/* diff --git a/.github/workflows/nightly_no_avx2.yml b/.github/workflows/nightly_no_avx2.yml deleted file mode 100644 index e712b421..00000000 --- a/.github/workflows/nightly_no_avx2.yml +++ /dev/null @@ -1,226 +0,0 @@ -name: Nightly no AVX2 tests -run-name: 🌙 Nightly no AVX2 tests ${{ github.sha }} - -on: - schedule: - - cron: '00 20 * * *' - push: - branches: - - '**' - paths: - - '.github/workflows/nightly_no_avx2.yml' - - '.github/workflows/build_template.yml' - - '.github/workflows/test_template.yml' - - -# cancels the previous workflow run when a new one appears in the same branch (e.g. master or a PR's branch) -concurrency: - group: nightly_no_avx2_${{ github.ref }} - cancel-in-progress: true - -jobs: - commit_info: - name: Commit info - runs-on: ubuntu-22.04 - steps: - - run: | - echo "# Automated Tests of commit ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY - [[ "${{ github.event.schedule }}" == "00 20 * * *" ]] && echo "* **Nightly test, testing master's head with the daemon's master head**" >> $GITHUB_STEP_SUMMARY - echo "* Commit URL: [${{ github.sha }}](/${{ github.repository }}/commit/${{ github.sha }})" >> $GITHUB_STEP_SUMMARY - echo "* Initiated by: [@${{ github.actor }}](https://github.com/${{ github.actor }})" >> $GITHUB_STEP_SUMMARY - echo "* Ref: ${{ github.ref_type }} \"${{ github.ref_name }}\"" >> $GITHUB_STEP_SUMMARY - echo "* Attempt: ${{ github.run_attempt }}" >> $GITHUB_STEP_SUMMARY - - check_branch: - name: Check branch existence - runs-on: ubuntu-22.04 - outputs: - manticore_locator: ${{ steps.set_locator.outputs.manticore_locator }} - steps: - - name: Check if branch exists in manticoresoftware/manticoresearch - id: check_branch - if: github.ref_name != 'master' - run: | - # Extract the actual branch name for pull requests - if [[ "${{ github.event_name }}" == "pull_request" ]]; then - BRANCH_NAME="${{ github.event.pull_request.head.ref }}" - echo "Pull request detected, using head ref: $BRANCH_NAME" - else - BRANCH_NAME="${{ github.ref_name }}" - echo "Direct push detected, using ref name: $BRANCH_NAME" - fi - - echo "Checking if branch '$BRANCH_NAME' exists in manticoresoftware/manticoresearch..." - HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" https://api.github.com/repos/manticoresoftware/manticoresearch/branches/$BRANCH_NAME) - echo "GitHub API response status: $HTTP_STATUS" - - if [ "$HTTP_STATUS" -eq "200" ]; then - echo "✅ Branch '$BRANCH_NAME' EXISTS in manticoresoftware/manticoresearch" - echo "branch_exists=true" >> $GITHUB_OUTPUT - echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT - else - echo "❌ Branch '$BRANCH_NAME' does NOT exist in manticoresoftware/manticoresearch (HTTP $HTTP_STATUS)" - echo "branch_exists=false" >> $GITHUB_OUTPUT - echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT - fi - - - name: Set Manticore Locator - id: set_locator - run: | - if [[ "${{ github.event.schedule }}" == "00 20 * * *" ]]; then - echo "manticore_locator=GIT_REPOSITORY https://github.com/manticoresoftware/manticoresearch.git GIT_TAG master GIT_SUBMODULES \"\"" >> $GITHUB_OUTPUT - echo "Set manticore_locator: GIT_REPOSITORY https://github.com/manticoresoftware/manticoresearch.git GIT_TAG master" - elif [[ "${{ github.ref_name }}" != "master" && "${{ steps.check_branch.outputs.branch_exists }}" == "true" ]]; then - echo "manticore_locator=GIT_REPOSITORY https://github.com/manticoresoftware/manticoresearch.git GIT_TAG ${{ steps.check_branch.outputs.branch_name }} GIT_SUBMODULES \"\"" >> $GITHUB_OUTPUT - echo "Set manticore_locator: GIT_REPOSITORY https://github.com/manticoresoftware/manticoresearch.git GIT_TAG ${{ steps.check_branch.outputs.branch_name }}" - else - echo "manticore_locator=" >> $GITHUB_OUTPUT - echo "Set manticore_locator: (empty)" - fi - - embedding_linux_pack: - needs: commit_info - uses: ./.github/workflows/embedding_build_template.yml - strategy: - fail-fast: false - matrix: - distr: [linux] - arch: [x86_64] - name: ${{ matrix.distr }} ${{ matrix.arch }} embedding packing - with: - distr: ${{ matrix.distr }} - arch: ${{ matrix.arch }} - - build_linux_release: - name: Linux release build - needs: [embedding_linux_pack, check_branch] - uses: ./.github/workflows/build_template.yml - with: - MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }} - embeddings_artifact: embeddings_linux_x86_64 - - test_linux_release: - name: Linux release mode tests - needs: [build_linux_release, check_branch] - uses: ./.github/workflows/test_template.yml - strategy: - fail-fast: false - matrix: - name: [1_54,55_108,109_162,163_216,217_270,271_324,325_378,379_and_on] - include: - - name: 1_54 - start: 1 - end: 54 - - name: 55_108 - start: 55 - end: 108 - - name: 109_162 - start: 109 - end: 162 - - name: 163_216 - start: 163 - end: 216 - - name: 217_270 - start: 217 - end: 270 - - name: 271_324 - start: 271 - end: 324 - - name: 325_378 - start: 325 - end: 378 - - name: 379_and_on - start: 379 - end: 999999 - with: - CTEST_CONFIGURATION_TYPE: "RelWithDebInfo" - CTEST_START: ${{ matrix.start }} - CTEST_END: ${{ matrix.end }} - MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }} - artifact_name: no_avx2_release_test_${{ matrix.name }} - xml_command: "cd build; cp -r Testing/2*/Test.xml .; xsltproc -o junit_tests_${{ matrix.name }}.xml ../misc/junit/ctest2junit.xsl Test.xml" - timeout: 10 - USE_AVX2: false - - release_tests_report: - name: release mode tests summary and report - needs: test_linux_release - runs-on: ubuntu-22.04 - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: Download test report artifact 1_54 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: no_avx2_release_test_1_54 - path: . - - name: Download test report artifact 55_108 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: no_avx2_release_test_55_108 - path: . - - name: Download test report artifact 109_162 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: no_avx2_release_test_109_162 - path: . - - name: Download test report artifact 163_216 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: no_avx2_release_test_163_216 - path: . - - name: Download test report artifact 217_270 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: no_avx2_release_test_217_270 - path: . - - name: Download test report artifact 271_324 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: no_avx2_release_test_271_324 - path: . - - name: Download test report artifact 325_378 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: no_avx2_release_test_325_378 - path: . - - name: Download test report artifact 379_and_on - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: no_avx2_release_test_379_and_on - path: . - - name: Publish test results - uses: EnricoMi/publish-unit-test-result-action@v2 - continue-on-error: true - with: - check_name: Linux release test results - files: build/junit*.xml - compare_to_earlier_commit: false - - name: Per-test results - run: | - for file in build/status*; do echo -n "$file: "; cat "$file"; done - grep -o "success" build/status* | wc -l | awk '{if ($1==8) exit 0; else {print "Found only "$1" successful runs out of 8"; exit 1}}' - shell: bash - - name: Delete unneded per-shard artifacts - if: always() - uses: geekyeggo/delete-artifact@v5 - with: - name: no_avx2_release_test_* - failOnError: false - - name: Upload combined artifacts - if: always() - continue-on-error: true - uses: manticoresoftware/upload_artifact_with_retries@v4 - with: - name: no_avx2_release_test_resuls - path: build diff --git a/.github/workflows/pack_publish.yml b/.github/workflows/pack_publish.yml deleted file mode 100644 index 3ee42499..00000000 --- a/.github/workflows/pack_publish.yml +++ /dev/null @@ -1,363 +0,0 @@ -name: 📦 Pack and publish -run-name: 📦 Pack and publish ${{ github.sha }} - -#on: workflow_call -on: - workflow_run: - workflows: [ 🔬 Test ] - types: [ completed ] - branches: [ master ] - pull_request: - branches: [ master ] - types: [opened, synchronize, reopened, labeled, unlabeled] - push: - branches: - - columnar-* - tags: - - 'release*' - - 'pack_publish' - -# cancels the previous workflow run when a new one appears in the same branch (e.g. master or a PR's branch) -concurrency: - group: pack_${{ github.ref }} - cancel-in-progress: true - -jobs: - - pack: - name: OK to pack? - runs-on: ubuntu-22.04 - outputs: - version: ${{ steps.semver-tagger.outputs.version }} - version_full: ${{ steps.semver-tagger.outputs.version_full }} - version_rpm: ${{ steps.semver-tagger.outputs.version_rpm }} - target: ${{ steps.semver-tagger.outputs.target }} - should_continue: ${{ steps.check-should-continue.outputs.should_continue }} - if: | - (github.event_name == 'pull_request' && (contains(github.event.pull_request.labels.*.name, 'pack') || contains(github.event.pull_request.labels.*.name, 'publish'))) - || ( github.event_name == 'push' && ( startsWith( github.ref, 'refs/heads/columnar-' ) || contains( github.ref, 'refs/tags/pack_publish' ) ) ) - || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' && github.ref == 'refs/heads/master' && github.event.workflow_run.event != 'schedule') - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - - name: Update version - id: semver-tagger - uses: manticoresoftware/semver-tagger-action@main - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - conventional_commits_authors: alexey@manticoresearch.com - ignore_patterns: '\.md$|^test/|^manual/|\.clt|\.github|\.patterns|\.yml|\.gitignore' - debug: true - - name: Check if we should continue packing - id: check-should-continue - run: | - # Continue if version was updated, if we have the "pack" label on PR, or if target is "release" - if [[ "${{ steps.semver-tagger.outputs.version_updated }}" == "true" ]]; then - echo "Continuing because version was updated" - echo "should_continue=true" >> $GITHUB_OUTPUT - elif [[ "${{ steps.semver-tagger.outputs.target }}" == "release" ]]; then - echo "Continuing because target is release" - echo "should_continue=true" >> $GITHUB_OUTPUT - elif [[ "${{ github.event_name }}" == "pull_request" && "${{ contains(github.event.pull_request.labels.*.name, 'pack') }}" == "true" ]]; then - echo "Continuing because PR has 'pack' label" - echo "should_continue=true" >> $GITHUB_OUTPUT - elif [[ "${{ contains(github.ref, 'refs/tags/pack_publish') }}" == "true" ]]; then - echo "Continuing because commit has 'pack_publish' tag" - echo "should_continue=true" >> $GITHUB_OUTPUT - else - echo "Skipping packing because version wasn't updated, target is not release, and there's no 'pack' label or tag" - echo "should_continue=false" >> $GITHUB_OUTPUT - fi - - run: | - echo "# Packing and publishing all for commit ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY - echo "* Commit URL: [${{ github.sha }}](/${{ github.repository }}/commit/${{ github.sha }})" >> $GITHUB_STEP_SUMMARY - echo "* Initiated by: [@${{ github.actor }}](https://github.com/${{ github.actor }})" >> $GITHUB_STEP_SUMMARY - echo "* Ref: ${{ github.ref_type }} \"${{ github.ref_name }}\"" >> $GITHUB_STEP_SUMMARY - echo "* Attempt: ${{ github.run_attempt }}" >> $GITHUB_STEP_SUMMARY - echo "* Version updated: ${{ steps.semver-tagger.outputs.version_updated }}" >> $GITHUB_STEP_SUMMARY - echo "* Should continue packing: ${{ steps.check-should-continue.outputs.should_continue }}" >> $GITHUB_STEP_SUMMARY - - check_branch: - name: Check branch existence - runs-on: ubuntu-22.04 - outputs: - manticore_locator: ${{ steps.set_locator.outputs.manticore_locator }} - steps: - - name: Check if branch exists in manticoresoftware/manticoresearch - id: check_branch - if: github.ref_name != 'master' - run: | - # Extract the actual branch name for pull requests - if [[ "${{ github.event_name }}" == "pull_request" ]]; then - BRANCH_NAME="${{ github.event.pull_request.head.ref }}" - else - BRANCH_NAME="${{ github.ref_name }}" - fi - - HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" https://api.github.com/repos/manticoresoftware/manticoresearch/branches/$BRANCH_NAME) - if [ "$HTTP_STATUS" -eq "200" ]; then - echo "branch_exists=true" >> $GITHUB_OUTPUT - echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT - else - echo "branch_exists=false" >> $GITHUB_OUTPUT - echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT - fi - - - name: Set Manticore Locator - id: set_locator - run: | - if [[ "${{ github.event.schedule }}" == "00 20 * * *" ]]; then - echo "manticore_locator=GIT_REPOSITORY https://github.com/manticoresoftware/manticoresearch.git GIT_TAG master GIT_SUBMODULES \"\"" >> $GITHUB_OUTPUT - elif [[ "${{ github.ref_name }}" != "master" && "${{ steps.check_branch.outputs.branch_exists }}" == "true" ]]; then - echo "manticore_locator=GIT_REPOSITORY https://github.com/manticoresoftware/manticoresearch.git GIT_TAG ${{ steps.check_branch.outputs.branch_name }} GIT_SUBMODULES \"\"" >> $GITHUB_OUTPUT - else - echo "manticore_locator=" >> $GITHUB_OUTPUT - fi - - embedding_builds: - needs: pack - uses: ./.github/workflows/embedding_build_template.yml - strategy: - fail-fast: false - matrix: - include: - # Linux builds - - distr: linux - arch: x86_64 - - distr: linux - arch: aarch64 - # macOS builds - - distr: macos - arch: x86_64 - - distr: macos - arch: aarch64 - # Windows builds - - distr: windows - arch: x86_64 - - distr: windows - arch: aarch64 - name: ${{ matrix.distr }} ${{ matrix.arch }} embedding packing - with: - distr: ${{ matrix.distr }} - arch: ${{ matrix.arch }} - - pack_debian_ubuntu: - uses: ./.github/workflows/build_template.yml - needs: [embedding_builds, check_branch, pack] - if: needs.pack.outputs.should_continue == 'true' - strategy: - fail-fast: false - matrix: - DISTR: [bionic, focal, jammy, buster, bullseye, bookworm] - arch: [x86_64, aarch64] - name: ${{ matrix.DISTR }} ${{ matrix.arch }} packing - with: - MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }} - DISTR: ${{ matrix.DISTR }} - arch: ${{ matrix.arch }} - cmake_command: | - mkdir build - cd build - cmake -DPACK=1 .. - cmake --build . --target package - cache_key: pack_${{ matrix.DISTR }}_${{ matrix.arch }} - artifact_list: "build/manticore*deb" - embeddings_artifact: embeddings_linux_${{ matrix.arch }} - version: ${{ needs.pack.outputs.version }} - package_version: ${{ needs.pack.outputs.version_full }} - - pack_rhel: - uses: ./.github/workflows/build_template.yml - needs: [embedding_builds, check_branch, pack] - if: needs.pack.outputs.should_continue == 'true' - strategy: - fail-fast: false - matrix: - DISTR: [rhel7, rhel8, rhel9, rhel10] - arch: [x86_64, aarch64] - name: ${{ matrix.DISTR }} ${{ matrix.arch }} packing - with: - MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }} - DISTR: ${{ matrix.DISTR }} - arch: ${{ matrix.arch }} - boost: boost_rhel_feb17 - cmake_command: | - ln -s $(pwd) /builds_manticoresearch_dev_usr_src_debug_manticore_component_src_0 - cd /builds_manticoresearch_dev_usr_src_debug_manticore_component_src_0 - mkdir build - cd build - CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX=/builds_manticoresearch_dev_usr_src_debug_manticore_component cmake -DPACK=1 .. - cmake --build . --target package - cache_key: pack_${{ matrix.DISTR }}_${{ matrix.arch }} - artifact_list: "build/manticore*rpm" - embeddings_artifact: embeddings_linux_${{ matrix.arch }} - version: ${{ needs.pack.outputs.version }} - package_version: ${{ needs.pack.outputs.version_full }} - rpm_package_version: ${{ needs.pack.outputs.version_rpm }} - - - pack_macos: - uses: ./.github/workflows/build_template.yml - needs: [embedding_builds, check_branch, pack] - if: needs.pack.outputs.should_continue == 'true' - strategy: - fail-fast: false - matrix: - DISTR: [macos] - arch: [x86_64, arm64] - name: ${{ matrix.DISTR }} ${{ matrix.arch }} packing - with: - MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }} - DISTR: ${{ matrix.DISTR }} - arch: ${{ matrix.arch }} - HOMEBREW_PREFIX: /opt/homebrew - cmake_command: | - mkdir build - cd build - cmake -DPACK=1 .. - cmake --build . --target package - cache_key: pack_${{ matrix.DISTR }}_${{ matrix.arch }} - artifact_list: "build/manticore*tar.gz" - embeddings_artifact: embeddings_macos_${{ matrix.arch == 'arm64' && 'aarch64' || matrix.arch }} - version: ${{ needs.pack.outputs.version }} - package_version: ${{ needs.pack.outputs.version_full }} - - pack_windows: - name: Windows x64 package - uses: ./.github/workflows/build_template.yml - needs: [embedding_builds, check_branch, pack] - if: needs.pack.outputs.should_continue == 'true' - with: - MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }} - DISTR: windows - arch: x64 - cmake_command: | - mkdir build - cd build - cmake -DPACK=1 .. - cmake --build . --target package - cache_key: pack_${{ matrix.DISTR }}_${{ matrix.arch }} - artifact_list: "build/manticore*exe build/manticore*zip" - embeddings_artifact: embeddings_windows_x86_64 - version: ${{ needs.pack.outputs.version }} - package_version: ${{ needs.pack.outputs.version_full }} - skip_avx2_move: true - - # virtual job to simplify the CI - # This job depends on all the package preparation jobs that have to pass before we can start publishing packages - publish: - name: OK to publish? - runs-on: ubuntu-22.04 - needs: [pack_debian_ubuntu, pack_rhel, pack_macos, pack_windows, pack] - if: | - (github.repository == 'manticoresoftware/columnar') - && ( - (github.event_name == 'pull_request' && (contains(github.event.pull_request.labels.*.name, 'publish'))) - || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') - || (github.event_name == 'push' && startsWith(github.ref, 'refs/heads/columnar-')) - || (github.event_name == 'push' && contains(github.ref, 'refs/tags/pack_publish')) - || (needs.pack.outputs.target == 'release') - ) - outputs: - target: ${{ steps.set-target.outputs.target }} - steps: - - run: echo "Ready to publish" - - publish_debian_ubuntu: - needs: - - pack - - publish - runs-on: ubuntu-22.04 - strategy: - fail-fast: true - matrix: - DISTR: [bionic, focal, jammy, buster, bullseye, bookworm] - arch: [x86_64, aarch64] - name: ${{ matrix.DISTR }} ${{ matrix.arch }} publishing - steps: - - uses: manticoresoftware/publish_to_repo@main - with: - ssh_key: ${{ secrets.REPO_SSH_KEY }} - distr: ${{ matrix.DISTR }} - arch: ${{ matrix.arch }} - artifact: build_${{ matrix.DISTR }}_RelWithDebInfo_${{ matrix.arch }} - type: deb - delimiter: "-" - target: ${{ needs.pack.outputs.target }} - - publish_rhel: - needs: - - pack - - publish - runs-on: ubuntu-22.04 - strategy: - fail-fast: true - matrix: - DISTR: [7, 8, 9, 10] - arch: [x86_64, aarch64] - name: RHEL ${{ matrix.DISTR }} ${{ matrix.arch }} publishing - steps: - - uses: manticoresoftware/publish_to_repo@main - with: - ssh_key: ${{ secrets.REPO_SSH_KEY }} - distr: ${{ matrix.DISTR }} - arch: ${{ matrix.arch }} - artifact: build_rhel${{ matrix.DISTR }}_RelWithDebInfo_${{ matrix.arch }} - type: rpm - delimiter: "_" - target: ${{ needs.pack.outputs.target }} - - publish_macos: - needs: - - pack - - publish - runs-on: ubuntu-22.04 - strategy: - fail-fast: true - matrix: - arch: [x86_64, arm64] - name: macos ${{ matrix.arch }} publishing - steps: - - uses: manticoresoftware/publish_to_repo@main - with: - ssh_key: ${{ secrets.REPO_SSH_KEY }} - distr: macos - arch: ${{ matrix.arch }} - artifact: build_macos_RelWithDebInfo_${{ matrix.arch }} - type: arc - delimiter: "-" - target: ${{ needs.pack.outputs.target }} - - publish_windows: - name: Publishing Windows packages to repo.manticoresearch.com - needs: - - pack - - publish - runs-on: ubuntu-22.04 - steps: - - uses: manticoresoftware/publish_to_repo@main - with: - ssh_key: ${{ secrets.REPO_SSH_KEY }} - distr: windows - arch: x64 - artifact: build_windows_RelWithDebInfo_x64 - type: arc - delimiter: "-" - target: ${{ needs.pack.outputs.target }} - - update-manticoresearch-deps: - needs: - - pack - - publish - runs-on: ubuntu-22.04 - if: needs.publish.outputs.target != 'release' - steps: - - name: Update deps - uses: manticoresoftware/manticoresearch/actions/update-deps@master - with: - name: mcl - version: ${{ needs.pack.outputs.version_full }} - token: ${{ secrets.PR_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 7cbb9943..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,728 +0,0 @@ -name: 🔬 Test -run-name: "🔬 Test ${{ (github.event.schedule == '00 20 * * *' || github.event_name == 'workflow_dispatch') && 'master HEAD in daemon + ' || 'commit' }} ${{ github.sha }}" - -#on: workflow_call - -on: - push: - branches: - - master - - columnar-* - pull_request: - branches: [ master ] - types: [opened, synchronize, reopened, labeled, unlabeled] - workflow_dispatch: - schedule: - - cron: '00 20 * * *' - -# cancels the previous workflow run when a new one appears in the same branch (e.g. master or a PR's branch) -concurrency: - group: CI_${{ github.ref }} - cancel-in-progress: true - -jobs: - - commit_info: - name: Commit info - runs-on: ubuntu-22.04 - steps: - - run: | - echo "# Automated Tests of commit ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY - [[ "${{ github.event.schedule }}" == "00 20 * * *" || "${{ github.event_name }}" == "workflow_dispatch" ]] && echo "* **Nightly/manual test, testing master's head with the daemon's master head**" >> $GITHUB_STEP_SUMMARY - echo "* Commit URL: [${{ github.sha }}](/${{ github.repository }}/commit/${{ github.sha }})" >> $GITHUB_STEP_SUMMARY - echo "* Initiated by: [@${{ github.actor }}](https://github.com/${{ github.actor }})" >> $GITHUB_STEP_SUMMARY - echo "* Ref: ${{ github.ref_type }} \"${{ github.ref_name }}\"" >> $GITHUB_STEP_SUMMARY - echo "* Attempt: ${{ github.run_attempt }}" >> $GITHUB_STEP_SUMMARY - - check_branch: - name: Check branch existence - runs-on: ubuntu-22.04 - outputs: - manticore_locator: ${{ steps.set_locator.outputs.manticore_locator }} - ref: ${{ steps.ref.outputs.ref }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Check if branch exists in manticoresoftware/manticoresearch - id: check_branch - if: github.ref_name != 'master' - run: | - # Extract the actual branch name for pull requests - if [[ "${{ github.event_name }}" == "pull_request" ]]; then - BRANCH_NAME="${{ github.event.pull_request.head.ref }}" - echo "Pull request detected, using head ref: $BRANCH_NAME" - else - BRANCH_NAME="${{ github.ref_name }}" - echo "Direct push detected, using ref name: $BRANCH_NAME" - fi - - echo "Checking if branch '$BRANCH_NAME' exists in manticoresoftware/manticoresearch..." - HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" https://api.github.com/repos/manticoresoftware/manticoresearch/branches/$BRANCH_NAME) - echo "GitHub API response status: $HTTP_STATUS" - - if [ "$HTTP_STATUS" -eq "200" ]; then - echo "✅ Branch '$BRANCH_NAME' EXISTS in manticoresoftware/manticoresearch" - echo "branch_exists=true" >> $GITHUB_OUTPUT - echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT - else - echo "❌ Branch '$BRANCH_NAME' does NOT exist in manticoresoftware/manticoresearch (HTTP $HTTP_STATUS)" - echo "branch_exists=false" >> $GITHUB_OUTPUT - echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT - fi - - - name: Resolve Manticore ref from locator - id: locator_ref - run: | - set -e - REF="" - if [[ -f manticore_src.txt ]]; then - TAG=$(sed -nE 's/.*GIT_TAG[[:space:]]+"?([^" ]+)"?.*/\1/p' manticore_src.txt | head -n1) - if [[ -n "$TAG" && "$TAG" != *'$'* ]]; then - REF="$TAG" - if [[ "$REF" =~ ^[0-9a-fA-F]{40}$ ]]; then - MATCH=$(git ls-remote --heads https://github.com/manticoresoftware/manticoresearch.git | awk -v sha="${REF,,}" 'tolower($1)==sha { sub("refs/heads/", "", $2); print $2; exit }') - if [[ -n "$MATCH" ]]; then - REF="$MATCH" - fi - fi - fi - fi - echo "ref=${REF}" >> $GITHUB_OUTPUT - if [[ -n "$REF" ]]; then - echo "Manticore ref from locator: ${REF}" >> $GITHUB_STEP_SUMMARY - fi - - - name: Detect ref for CLT tests - id: ref - run: | - set -e - if [[ "${{ github.ref_name }}" == "master" && -n "${{ steps.locator_ref.outputs.ref }}" ]]; then - REF="${{ steps.locator_ref.outputs.ref }}" - elif [[ "${{ github.ref_name }}" == "master" ]]; then - REF="master" - elif [[ "${{ steps.check_branch.outputs.branch_exists }}" == "true" ]]; then - REF="${{ steps.check_branch.outputs.branch_name }}" - else - REF="master" - fi - echo "ref=${REF}" >> $GITHUB_OUTPUT - echo "Ref for CLT tests: ${REF}" >> $GITHUB_STEP_SUMMARY - - - name: Set Manticore Locator - id: set_locator - run: | - if [[ "${{ github.event.schedule }}" == "00 20 * * *" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then - echo "manticore_locator=GIT_REPOSITORY https://github.com/manticoresoftware/manticoresearch.git GIT_TAG master GIT_SUBMODULES \"\"" >> $GITHUB_OUTPUT - echo "Set manticore_locator: GIT_REPOSITORY https://github.com/manticoresoftware/manticoresearch.git GIT_TAG master" - elif [[ "${{ github.ref_name }}" != "master" && "${{ steps.check_branch.outputs.branch_exists }}" == "true" ]]; then - echo "manticore_locator=GIT_REPOSITORY https://github.com/manticoresoftware/manticoresearch.git GIT_TAG ${{ steps.check_branch.outputs.branch_name }} GIT_SUBMODULES \"\"" >> $GITHUB_OUTPUT - echo "Set manticore_locator: GIT_REPOSITORY https://github.com/manticoresoftware/manticoresearch.git GIT_TAG ${{ steps.check_branch.outputs.branch_name }}" - else - echo "manticore_locator=" >> $GITHUB_OUTPUT - echo "Set manticore_locator: (empty)" - fi - - embedding_linux_pack: - needs: commit_info - uses: ./.github/workflows/embedding_build_template.yml - strategy: - fail-fast: false - matrix: - distr: [linux] - arch: [x86_64] - name: ${{ matrix.distr }} ${{ matrix.arch }} embedding packing - with: - distr: ${{ matrix.distr }} - arch: ${{ matrix.arch }} - - embedding_windows_pack: - needs: commit_info - uses: ./.github/workflows/embedding_build_template.yml - strategy: - fail-fast: false - matrix: - distr: [windows] - arch: [x86_64] - name: ${{ matrix.distr }} ${{ matrix.arch }} embedding packing - with: - distr: ${{ matrix.distr }} - arch: ${{ matrix.arch }} - - build_linux_release: - name: Linux release build - needs: [embedding_linux_pack, check_branch] - uses: ./.github/workflows/build_template.yml - with: - MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }} - embeddings_artifact: embeddings_linux_x86_64 - - build_linux_debug: - name: Linux debug build - needs: [embedding_linux_pack, check_branch] - uses: ./.github/workflows/build_template.yml - with: - CTEST_CONFIGURATION_TYPE: "Debug" - MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }} - embeddings_artifact: embeddings_linux_x86_64 - - build_windows: - name: Windows x64 build - needs: [embedding_windows_pack, check_branch] - uses: ./.github/workflows/build_template.yml - with: - DISTR: windows - arch: x64 - CTEST_CMAKE_GENERATOR: "Ninja Multi-Config" - CTEST_CONFIGURATION_TYPE: Debug - cache_key: build_windows_x64 - artifact_list: "build/columnar/Debug/lib_manticore_columnar.dll build/secondary/Debug/lib_manticore_secondary.dll build/knn/Debug/lib_manticore_knn.dll build/columnar/Debug/lib_manticore_columnar_avx2.dll build/secondary/Debug/lib_manticore_secondary_avx2.dll build/knn/Debug/lib_manticore_knn_avx2.dll build/columnar/Debug/lib_manticore_columnar_avx512.dll build/secondary/Debug/lib_manticore_secondary_avx512.dll build/knn/Debug/lib_manticore_knn_avx512.dll build/_deps/manticore-build/src/Debug/indexer.exe build/_deps/manticore-build/src/Debug/searchd.exe" # - MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }} - embeddings_artifact: embeddings_windows_x86_64 - - test_linux_debug: - name: Linux debug mode tests - needs: [build_linux_debug, check_branch] - uses: ./.github/workflows/test_template.yml - strategy: - fail-fast: false - matrix: - name: [1_54,55_108,109_162,163_216,217_270,271_324,325_378,379_and_on] - include: - - name: 1_54 - start: 1 - end: 54 - - name: 55_108 - start: 55 - end: 108 - - name: 109_162 - start: 109 - end: 162 - - name: 163_216 - start: 163 - end: 216 - - name: 217_270 - start: 217 - end: 270 - - name: 271_324 - start: 271 - end: 324 - - name: 325_378 - start: 325 - end: 378 - - name: 379_and_on - start: 379 - end: 999999 - with: - CTEST_CONFIGURATION_TYPE: "Debug" - CTEST_START: ${{ matrix.start }} - CTEST_END: ${{ matrix.end }} - MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }} - artifact_name: debug_test_${{ matrix.name }} - xml_command: "cd build; cp -r Testing/2*/Test.xml .; xsltproc -o junit_tests_${{ matrix.name }}.xml ../misc/junit/ctest2junit.xsl Test.xml" - timeout: 10 - USE_AVX2: true - - debug_tests_report: - name: Debug mode tests summary and report - needs: test_linux_debug - runs-on: ubuntu-22.04 - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: Download test report artifact 1_54 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: debug_test_1_54 - path: . - - name: Download test report artifact 55_108 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: debug_test_55_108 - path: . - - name: Download test report artifact 109_162 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: debug_test_109_162 - path: . - - name: Download test report artifact 163_216 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: debug_test_163_216 - path: . - - name: Download test report artifact 217_270 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: debug_test_217_270 - path: . - - name: Download test report artifact 271_324 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: debug_test_271_324 - path: . - - name: Download test report artifact 325_378 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: debug_test_325_378 - path: . - - name: Download test report artifact 379_and_on - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: debug_test_379_and_on - path: . - - name: Publish test results - uses: EnricoMi/publish-unit-test-result-action@v2 - continue-on-error: true - with: - check_name: Linux debug test results - files: build/junit*.xml - compare_to_earlier_commit: false - - name: Per-test results - run: | - for file in build/status*; do echo -n "$file: "; cat "$file"; done - grep -o "success" build/status* | wc -l | awk '{if ($1==8) exit 0; else {print "Found only "$1" successful runs out of 8"; exit 1}}' - shell: bash - - name: Delete unneded per-shard artifacts - if: always() - uses: geekyeggo/delete-artifact@v5 - with: - name: debug_test_* - failOnError: false - - name: Upload combined artifacts - if: always() - continue-on-error: true - uses: manticoresoftware/upload_artifact_with_retries@v4 - with: - name: debug_test_resuls - path: build - - test_linux_release: - name: Linux release mode tests - needs: [build_linux_release, check_branch] - uses: ./.github/workflows/test_template.yml - strategy: - fail-fast: false - matrix: - name: [1_54,55_108,109_162,163_216,217_270,271_324,325_378,379_and_on] - include: - - name: 1_54 - start: 1 - end: 54 - - name: 55_108 - start: 55 - end: 108 - - name: 109_162 - start: 109 - end: 162 - - name: 163_216 - start: 163 - end: 216 - - name: 217_270 - start: 217 - end: 270 - - name: 271_324 - start: 271 - end: 324 - - name: 325_378 - start: 325 - end: 378 - - name: 379_and_on - start: 379 - end: 999999 - with: - CTEST_CONFIGURATION_TYPE: "RelWithDebInfo" - CTEST_START: ${{ matrix.start }} - CTEST_END: ${{ matrix.end }} - MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }} - artifact_name: release_test_${{ matrix.name }} - xml_command: "cd build; cp -r Testing/2*/Test.xml .; xsltproc -o junit_tests_${{ matrix.name }}.xml ../misc/junit/ctest2junit.xsl Test.xml" - timeout: 10 - USE_AVX2: true - - release_tests_report: - name: release mode tests summary and report - needs: test_linux_release - runs-on: ubuntu-22.04 - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: Download test report artifact 1_54 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: release_test_1_54 - path: . - - name: Download test report artifact 55_108 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: release_test_55_108 - path: . - - name: Download test report artifact 109_162 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: release_test_109_162 - path: . - - name: Download test report artifact 163_216 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: release_test_163_216 - path: . - - name: Download test report artifact 217_270 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: release_test_217_270 - path: . - - name: Download test report artifact 271_324 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: release_test_271_324 - path: . - - name: Download test report artifact 325_378 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: release_test_325_378 - path: . - - name: Download test report artifact 379_and_on - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: release_test_379_and_on - path: . - - name: Publish test results - uses: EnricoMi/publish-unit-test-result-action@v2 - continue-on-error: true - with: - check_name: Linux release test results - files: build/junit*.xml - compare_to_earlier_commit: false - - name: Per-test results - run: | - for file in build/status*; do echo -n "$file: "; cat "$file"; done - grep -o "success" build/status* | wc -l | awk '{if ($1==8) exit 0; else {print "Found only "$1" successful runs out of 8"; exit 1}}' - shell: bash - - name: Delete unneded per-shard artifacts - if: always() - uses: geekyeggo/delete-artifact@v5 - with: - name: release_test_* - failOnError: false - - name: Upload combined artifacts - if: always() - continue-on-error: true - uses: manticoresoftware/upload_artifact_with_retries@v4 - with: - name: release_test_resuls - path: build - - test_windows: - name: Windows tests - needs: [build_windows, check_branch] - uses: ./.github/workflows/win_test_template.yml - strategy: - fail-fast: false - matrix: - name: [1_86,87_172,173_258,259_344,345_and_on] - include: - - name: 1_86 - start: 1 - end: 86 - - name: 87_172 - start: 87 - end: 172 - - name: 173_258 - start: 173 - end: 258 - - name: 259_344 - start: 259 - end: 344 - - name: 345_and_on - start: 345 - end: 999999 - with: - CTEST_START: ${{ matrix.start }} - CTEST_END: ${{ matrix.end }} - MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }} - artifact_name: windows_test_${{ matrix.name }} - USE_AVX2: true - - windows_tests_report: - name: Windows tests summary and report - needs: test_windows - runs-on: ubuntu-22.04 - container: - image: manticoresearch/ubertests_ctest:3263_mar_2024 - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: Download test report artifact 1_86 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: windows_test_1_86 - path: . - - name: Download test report artifact 87_172 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: windows_test_87_172 - path: . - - name: Download test report artifact 173_258 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: windows_test_173_258 - path: . - - name: Download test report artifact 259_344 - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: windows_test_259_344 - path: . - - name: Download test report artifact 345_and_on - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: windows_test_345_and_on - path: . - - name: Convert the XML to JUnit format - run: for dir in build/xml_*; do xsltproc -o $dir/junit_tests.xml misc/junit/ctest2junit.xsl $dir/Test.xml; done; - shell: bash - - name: Publish test results - uses: EnricoMi/publish-unit-test-result-action@v2 - with: - check_name: Windows test results - compare_to_earlier_commit: false - files: build/xml_*/junit_tests.xml - - name: Per-test results - run: | - for file in build/status*; do echo -n "$file: "; cat "$file"; done - grep -o "success" build/status* | wc -l | awk '{if ($1==5) exit 0; else {print "Found only "$1" successful runs out of 5"; exit 1}}' - shell: bash - - name: Delete unneded per-shard artifacts - if: always() - uses: geekyeggo/delete-artifact@v5 - with: - name: windows_test_* - failOnError: false - - name: Upload combined artifacts - if: always() - continue-on-error: true - uses: manticoresoftware/upload_artifact_with_retries@v4 - with: - name: windows_test_resuls - path: build - - pack_jammy: - needs: [embedding_linux_pack, check_branch] - name: Jammy x86_64 package [build] - uses: ./.github/workflows/build_template.yml - with: - DISTR: jammy - MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }} - arch: x86_64 - cmake_command: | - mkdir build - cd build - cmake -DPACK=1 .. - cmake --build . --target package - cache_key: pack_jammy_x86_64 - artifact_list: "build/manticore*deb" - embeddings_artifact: embeddings_linux_x86_64 - artifact_name: jammy_package_RelWithDebInfo_x86_64 - - test_kit_docker_build: - name: Test Kit docker image - needs: [pack_jammy, check_branch] - if: always() && needs.pack_jammy.result != 'failure' - runs-on: ubuntu-22.04 - env: - GHCR_REPO: ghcr.io/manticoresoftware/manticoresearch - outputs: - docker_repo: ${{ steps.vars.outputs.docker_repo }} - docker_tag: ${{ steps.vars.outputs.docker_tag }} - image: ${{ steps.vars.outputs.image }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - - name: Checkout daemon repository for branch detection - uses: actions/checkout@v4 - with: - repository: manticoresoftware/manticoresearch - path: manticoresearch - fetch-depth: 0 - - - name: Get current branch name - id: current_branch - run: | - BRANCH=${{ github.head_ref || github.ref_name }} - echo "branch=${BRANCH}" >> $GITHUB_OUTPUT - - - name: Check if branch exists in manticoresearch - id: check_branch_exists - working-directory: manticoresearch - run: | - git ls-remote --heads origin ${{ steps.current_branch.outputs.branch }} | grep -q . && \ - echo "exists=true" >> $GITHUB_OUTPUT || echo "exists=false" >> $GITHUB_OUTPUT - - - name: Determine base test-kit image - id: base_image - run: | - set -e - # Always use test-kit-latest when the branch is master - if [[ "${{ steps.current_branch.outputs.branch }}" == "master" ]]; then - IMAGE="${GHCR_REPO}:test-kit-latest" - elif [[ "${{ steps.check_branch_exists.outputs.exists }}" == "true" ]]; then - BRANCH="${{ steps.current_branch.outputs.branch }}" - fix=$(echo "$BRANCH" | tr '/' '_') - DOCKER_TAG=$(echo "$fix" | sed 's/[^a-zA-Z0-9_.-]//g') - IMAGE="${GHCR_REPO}:test-kit-${DOCKER_TAG,,}" - else - IMAGE="${GHCR_REPO}:test-kit-latest" - fi - echo "image=${IMAGE}" >> "$GITHUB_OUTPUT" - echo "Base Test-kit Image: ${IMAGE}" >> "$GITHUB_STEP_SUMMARY" - - - name: Set variables - id: vars - run: | - set -e - BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}} - fix=$(echo "$BRANCH" | tr '/' '_') - DOCKER_TAG=$(echo "$fix" | sed 's/[^a-zA-Z0-9_.-]//g') - DOCKER_TAG="test-kit-${DOCKER_TAG,,}-mcl" - COMMIT=${GITHUB_SHA::8} - IMAGE="${GHCR_REPO}:${DOCKER_TAG}" - IMAGE_COMMIT="${GHCR_REPO}:test-kit-${COMMIT,,}-mcl" - BASE_IMAGE="${{ steps.base_image.outputs.image }}" - echo "docker_repo=${GHCR_REPO}" >> $GITHUB_OUTPUT - echo "docker_tag=${DOCKER_TAG}" >> $GITHUB_OUTPUT - echo "image=${IMAGE}" >> $GITHUB_OUTPUT - echo "image_commit=${IMAGE_COMMIT}" >> $GITHUB_OUTPUT - echo "base_image=${BASE_IMAGE}" >> $GITHUB_OUTPUT - echo "Docker Image: ${IMAGE}" >> $GITHUB_STEP_SUMMARY - echo "Docker Image commit: ${IMAGE_COMMIT}" >> $GITHUB_STEP_SUMMARY - echo "Base Image: ${BASE_IMAGE}" >> $GITHUB_STEP_SUMMARY - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ vars.GHCR_USER }} - password: ${{ secrets.GHCR_PASSWORD }} - - - name: Pull base image - run: | - echo "📥 PULLING Base Image: ${{ steps.vars.outputs.base_image }}" - docker pull ${{ steps.vars.outputs.base_image }} - - - name: Download built x86_64 Ubuntu Jammy packages - uses: manticoresoftware/download_artifact_with_retries@v3 - with: - name: jammy_package_RelWithDebInfo_x86_64 - path: . - - - name: Create test kit image with MCL package - run: | - # Find the MCL package we built - MCL_PACKAGE=$(find build -name "manticore-columnar-lib*.deb" | head -n 1) - if [ -z "$MCL_PACKAGE" ]; then - echo "ERROR: MCL package not found in build/" - find . -name "*.deb" || echo "No .deb files found" - exit 1 - fi - echo "Found MCL package: $MCL_PACKAGE" - - # Create temporary container from the base image - docker create --name temp ${{ steps.vars.outputs.base_image }} tail -f /dev/null - docker start temp - - # Copy our MCL package to the container - docker cp "$MCL_PACKAGE" temp:/build/ - - # Remove old MCL package and install the new one - docker exec temp bash -c \ - 'apt-get remove -y manticore-columnar-lib || true && \ - dpkg -i --force-confnew /build/manticore-columnar-lib*.deb && \ - apt-get install -f -y' - - # Debug: check installed libraries and their dependencies - docker exec temp bash -c \ - 'echo "=== Installed MCL files ===" && \ - dpkg -L manticore-columnar-lib && \ - echo "=== Embeddings lib dependencies ===" && \ - find / -name "lib_manticore_knn_embeddings*" 2>/dev/null && \ - for f in $(find / -name "lib_manticore_knn_embeddings*" 2>/dev/null); do ldd "$f" 2>&1; done && \ - echo "=== KNN lib dependencies ===" && \ - find / -name "lib_manticore_knn.*" 2>/dev/null && \ - for f in $(find / -name "lib_manticore_knn.*" 2>/dev/null); do ldd "$f" 2>&1; done' - - # Commit the modified test-kit images - docker commit temp ${{ steps.vars.outputs.image }} - docker commit temp ${{ steps.vars.outputs.image_commit }} - - # Cleanup - docker rm -f temp - - - name: Push test kit images - run: | - echo "🚀 PUSHING Docker Images:" - echo " Branch image: ${{ steps.vars.outputs.image }}" - echo " Commit image: ${{ steps.vars.outputs.image_commit }}" - docker push ${{ steps.vars.outputs.image }} - docker push ${{ steps.vars.outputs.image_commit }} - - - name: Export test kit image for artifact - run: | - # Export the container as a tar file for the CLT workflow (fallback) - # Create a temporary container to export - docker create --name temp-export ${{ steps.vars.outputs.image_commit }} tail -f /dev/null - docker export temp-export > manticore_test_kit.img - docker rm temp-export - - - name: Upload docker image artifact - uses: manticoresoftware/upload_artifact_with_retries@v4 - with: - name: manticore_test_kit.img - path: manticore_test_kit.img - - clt: - if: always() && needs.test_kit_docker_build.result != 'failure' - name: CLT - needs: [test_kit_docker_build, check_branch] - uses: ./.github/workflows/clt_tests.yml - secrets: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }} - JINA_API_KEY: ${{ secrets.JINA_API_KEY }} - with: - docker_image: ${{ needs.test_kit_docker_build.outputs.image }} - artifact_name: manticore_test_kit.img - repository: manticoresoftware/manticoresearch - ref: ${{ needs.check_branch.outputs.ref }} diff --git a/.github/workflows/test_template.yml b/.github/workflows/test_template.yml deleted file mode 100644 index 9fa5ef4b..00000000 --- a/.github/workflows/test_template.yml +++ /dev/null @@ -1,183 +0,0 @@ -name: Test Template - -on: - workflow_call: - inputs: - CTEST_CONFIGURATION_TYPE: - required: true - type: string - CTEST_START: - required: false - type: number - default: 1 - CTEST_END: - required: false - type: number - default: 999999 - artifact_name: - required: true - type: string - build_artifact_name: - required: false - type: string - default: "" - timeout: - required: false - type: number - default: 60 - xml_command: - required: false - type: string - default: "cd build; cp -r Testing/2*/Test.xml .; xsltproc -o junit_tests.xml ../misc/junit/ctest2junit.xsl Test.xml" - MANTICORE_LOCATOR: - required: false - type: string - default: "" - USE_AVX2: - required: false - type: boolean - default: true - USE_AVX512: - required: false - type: boolean - default: false - -jobs: - test: - name: ${{ inputs.CTEST_CONFIGURATION_TYPE }}_${{ inputs.CTEST_START }}_${{ inputs.CTEST_END }} - runs-on: ubuntu-22.04 - timeout-minutes: ${{ inputs.timeout }} - continue-on-error: true - defaults: - run: - shell: bash - container: - image: manticoresearch/ubertests_ctest:3263_mar_2024 - env: - DIAGNOSTIC: 1 - CACHEB: ../cache - NO_BUILD: 1 - CTEST_START: ${{ inputs.CTEST_START }} - CTEST_END: ${{ inputs.CTEST_END }} - MANTICORE_LOCATOR: ${{ inputs.MANTICORE_LOCATOR }} - # The following is useful to test a specific test, just uncomment it, no need to disable CTEST_START/END - # CTEST_REGEX: test_234 - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Print Manticore Locator - run: | - echo "Using MANTICORE_LOCATOR: '${{ inputs.MANTICORE_LOCATOR }}'" - if [[ -n "${{ inputs.MANTICORE_LOCATOR }}" ]]; then - echo "Manticore locator is set and will be used for testing" - else - echo "Manticore locator is empty, using default manticore sources" - fi - - - name: Determine branch name for cache - id: branch - run: | - if [ "${{ github.event_name }}" = "pull_request" ]; then - echo "branch_name=${{ github.head_ref }}" >> $GITHUB_OUTPUT - else - echo "branch_name=${{ github.ref_name }}" >> $GITHUB_OUTPUT - fi - - - name: Tests container entrypoint - run: bash /entry_point.sh & - - - name: Set build artifact name - id: build_artifact - run: | - if [ -n "${{ inputs.build_artifact_name }}" ]; then - echo "name=${{ inputs.build_artifact_name }}" >> $GITHUB_OUTPUT - else - echo "name=build_jammy_${{ inputs.CTEST_CONFIGURATION_TYPE }}_x86_64" >> $GITHUB_OUTPUT - fi - - - name: Download build artifacts - uses: manticoresoftware/download_artifact_with_retries@v4 - with: - name: ${{ steps.build_artifact.outputs.name }} - path: . - - - name: Download embeddings lib - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: embeddings_linux_x86_64 - path: ./embeddings-lib/ - - - name: Initialization of embeddings lib - run: | - mkdir -p embeddings/target/release - mv ./embeddings-lib/build/* embeddings/target/release/ - rm -fr ./embeddings-lib - - - name: Check out main cache before building - uses: actions/cache@v4 - with: - path: cache - enableCrossOsArchive: true - key: build_linux_x86_64_${{ steps.branch.outputs.branch_name }} - restore-keys: build_linux_x86_64_master - - - name: Check out deps cache before building - uses: actions/cache@v4 - with: - path: build/_deps/cache - enableCrossOsArchive: true - key: build_linux_x86_64_deps_${{ steps.branch.outputs.branch_name }} - restore-keys: build_linux_x86_64_deps_master - - - name: List files - run: find . - - - name: List of modules - run: find / -name "*.so" 2>/dev/null || echo "No *.so files found" - - - name: Replace standard libs to AVX2 - if: ${{ inputs.USE_AVX2 == true }} - run: | - if [ -d /__w/columnar/columnar/build/columnar ]; then - echo "Move linux based AVX2 libs" - mv /__w/columnar/columnar/build/columnar/lib_manticore_columnar_avx2.so /__w/columnar/columnar/build/columnar/lib_manticore_columnar.so - mv /__w/columnar/columnar/build/secondary/lib_manticore_secondary_avx2.so /__w/columnar/columnar/build/secondary/lib_manticore_secondary.so - mv /__w/columnar/columnar/build/knn/lib_manticore_knn_avx2.so /__w/columnar/columnar/build/knn/lib_manticore_knn.so - fi - - - name: Replace standard libs to AVX-512 - if: ${{ inputs.USE_AVX512 == true }} - run: | - if [ -d /__w/columnar/columnar/build/columnar ]; then - echo "Move linux based AVX-512 libs" - mv /__w/columnar/columnar/build/columnar/lib_manticore_columnar_avx512.so /__w/columnar/columnar/build/columnar/lib_manticore_columnar.so - mv /__w/columnar/columnar/build/secondary/lib_manticore_secondary_avx512.so /__w/columnar/columnar/build/secondary/lib_manticore_secondary.so - mv /__w/columnar/columnar/build/knn/lib_manticore_knn_avx512.so /__w/columnar/columnar/build/knn/lib_manticore_knn.so - fi - - - name: 🚀 Test - id: test - # --timeout may be not working https://gitlab.kitware.com/cmake/cmake/-/issues/23979 - run: ctest -VV -S cmake/citest.cmake --no-compress-output --timeout 60 - continue-on-error: true - - - name: Remember status - if: always() - run: echo "${{ steps.test.outcome }}" > build/status_${{ inputs.artifact_name }} - - - name: Prepare test report xmls - if: always() - continue-on-error: true - run: ${{ inputs.xml_command }} - - - name: Upload test artifacts - if: always() - continue-on-error: true - uses: manticoresoftware/upload_artifact_with_retries@v4 - with: - name: ${{ inputs.artifact_name }} - path: "build/junit*.xml build/_deps/manticore-build/test/test_*/report.* build/_deps/manticore-build/test/error*.txt build/_deps/manticore-build/test/*log build/status*" diff --git a/.github/workflows/win_test_template.yml b/.github/workflows/win_test_template.yml deleted file mode 100644 index be3d0400..00000000 --- a/.github/workflows/win_test_template.yml +++ /dev/null @@ -1,214 +0,0 @@ -name: Windows Test Template - -on: - workflow_call: - inputs: - CTEST_START: - required: false - type: number - default: 1 - CTEST_END: - required: false - type: number - default: 999999 - artifact_name: - required: true - type: string - MANTICORE_LOCATOR: - required: false - type: string - default: "" - USE_AVX2: - required: false - type: boolean - default: true - USE_AVX512: - required: false - type: boolean - default: false - -jobs: - test_windows: - name: ${{ inputs.CTEST_START }}_${{ inputs.CTEST_END }} - runs-on: windows-2022 - env: - CTEST_CMAKE_GENERATOR: "Visual Studio 17 2022" - CTEST_CONFIGURATION_TYPE: Debug - CTEST_START: ${{ inputs.CTEST_START }} - CTEST_END: ${{ inputs.CTEST_END }} - MANTICORE_LOCATOR: ${{ inputs.MANTICORE_LOCATOR }} - # The following is useful to test a specific test, just uncomment it, no need to disable CTEST_START/END - # CTEST_REGEX: test_234 - NO_BUILD: 1 - steps: - - name: Checkout repository # We have to checkout to access .github/workflows/ in further steps - uses: actions/checkout@v4 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: Print Manticore Locator - shell: pwsh - run: | - $locator = "${{ inputs.MANTICORE_LOCATOR }}" - Write-Host "Using MANTICORE_LOCATOR: '$locator'" - if ($locator) { - Write-Host "Manticore locator is set and will be used for testing" - } else { - Write-Host "Manticore locator is empty, using default manticore sources" - } - - name: Determine branch name for cache - id: branch - run: | - if ("${{ github.event_name }}" -eq "pull_request") { - echo "branch_name=${{ github.head_ref }}" >> $env:GITHUB_OUTPUT - } else { - echo "branch_name=${{ github.ref_name }}" >> $env:GITHUB_OUTPUT - } - - name: Determine current disk and set environment variables - shell: powershell - run: | - # Get the current working directory and extract the drive letter - $currentPath = Get-Location - $driveLetter = $currentPath.Drive.Name - Write-Host "Current drive: $driveLetter" - - # Set environment variables with the correct drive letter - $env:CACHEB = "$driveLetter`:/a/columnar/columnar/cache" - $env:LIBS_BUNDLE = "$driveLetter`:/a/columnar/columnar/bundle" - $env:BOOST_ROOT = "$driveLetter`:/a/columnar/columnar/boost_1_75_0" - - # Output the environment variables for debugging - Write-Host "CACHEB: $env:CACHEB" - Write-Host "LIBS_BUNDLE: $env:LIBS_BUNDLE" - Write-Host "BOOST_ROOT: $env:BOOST_ROOT" - - # Set these as step outputs for use in subsequent steps - "CACHEB=$env:CACHEB" >> $env:GITHUB_ENV - "LIBS_BUNDLE=$env:LIBS_BUNDLE" >> $env:GITHUB_ENV - "BOOST_ROOT=$env:BOOST_ROOT" >> $env:GITHUB_ENV - - name: Download build artifacts - uses: manticoresoftware/download_artifact_with_retries@v4 - with: - name: build_windows_Debug_x64 - path: . - - name: Cache Windows bundle - id: bundle_cache - uses: actions/cache@v4 - with: - path: | - bundle - boost_1_75_0 - key: win_bundle_v1 - - name: Download bundle if cache missed - if: steps.bundle_cache.outputs.cache-hit != 'true' - shell: bash - run: | - curl -sL -o sysroot_windows_x64.tar.xz https://repo.manticoresearch.com/repository/sysroots/roots_nov22/sysroot_windows_x64.tar.xz - tar -xf sysroot_windows_x64.tar.xz diskc/winbundle - mv diskc/winbundle bundle - curl -sL -o boost_1_75_0.tgz https://repo.manticoresearch.com/repository/ci/boost_1_75_0.tgz - tar -xf boost_1_75_0.tgz - - name: Remove ZSTD # cross-OS build/deps caches saved on Linux without zstd; must match compression method - run: rm C:\tools\zstd\zstd.exe - - name: Check out main cache - uses: actions/cache@v4 - with: - path: cache - enableCrossOsArchive: true - key: build_windows_x64_${{ steps.branch.outputs.branch_name }} - restore-keys: build_windows_x64_master - - name: Check out deps cache - uses: actions/cache@v4 - with: - path: build/_deps/cache - enableCrossOsArchive: true - key: build_windows_x64_deps_${{ steps.branch.outputs.branch_name }} - restore-keys: build_windows_x64_deps_master - - run: mv build/_deps/cache/windows-amd64/* cache/windows-amd64/ - - name: Copy libraries close to the binaries - run: | - cp bundle/mysql-x64/lib/debug/libmysql* build/_deps/manticore-build/src/Debug/ - cp bundle/Expat-1.95.8-x64/Libs/libexpat* build/_deps/manticore-build/src/Debug/ - cp bundle/libiconv-1.9.1-x64/lib/iconv* build/_deps/manticore-build/src/Debug/ - - name: Setup Flex and Bison - run: Install-ChocoPackage winflexbison3 - - name: Run mysql - run: | - C:\PROGRA~1\MySQL\"MySQL Server 8.0"\bin\mysqld.exe --initialize-insecure - C:\PROGRA~1\MySQL\"MySQL Server 8.0"\bin\mysqld.exe --install mysql - net start mysql - mysql -e "create user 'test'@'localhost' IDENTIFIED WITH mysql_native_password BY ''; create database test; grant all on test.* to 'test'@'localhost'; flush privileges;" -uroot - # TODO: Uncomment the below if there's no more OpenSSL in the runner like it happened in Sep 2023 (https://github.com/actions/runner-images/issues/8344) - # - name: Install OpenSSL - # run: powershell.exe ./.github/workflows/Install-OpenSSL.ps1 - - name: Setup PHP and mysqli - uses: shivammathur/setup-php@v2 - with: - php-version: '8.2' - extensions: mysqli - - - name: Download embeddings lib - uses: manticoresoftware/download_artifact_with_retries@v4 - continue-on-error: true - with: - name: embeddings_windows_x86_64 - path: ./embeddings-lib/ - - - name: Initialization of embeddings lib - run: | - mkdir embeddings\target - mkdir embeddings\target\release - mv .\embeddings-lib\build\* embeddings\target\release\ - rm -r ./embeddings-lib - $embeddingsDir = (Resolve-Path "embeddings\target\release").Path - $env:PATH = "$embeddingsDir;$env:PATH" - echo $embeddingsDir | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - Write-Host "Added embeddings runtime directory to PATH: $embeddingsDir" - - - name: Replace standard libs to AVX2 - if: ${{ inputs.USE_AVX2 == true }} - run: | - if (Test-Path -Path "D:\a\columnar\columnar\build\columnar\Debug\") { - Write-Host "Moving Windows-based AVX2 libs" - Move-Item -Path "D:\a\columnar\columnar\build\columnar\Debug\lib_manticore_columnar_avx2.dll" -Destination "D:\a\columnar\columnar\build\columnar\Debug\lib_manticore_columnar.dll" -Force - Move-Item -Path "D:\a\columnar\columnar\build\secondary\Debug\lib_manticore_secondary_avx2.dll" -Destination "D:\a\columnar\columnar\build\secondary\Debug\lib_manticore_secondary.dll" -Force - Move-Item -Path "D:\a\columnar\columnar\build\knn\Debug\lib_manticore_knn_avx2.dll" -Destination "D:\a\columnar\columnar\build\knn\Debug\lib_manticore_knn.dll" -Force - } else { - Write-Host "Directory D:\a\columnar\columnar\build\columnar\Debug\ does not exist" - } - - - name: Replace standard libs to AVX-512 - if: ${{ inputs.USE_AVX512 == true }} - run: | - if (Test-Path -Path "D:\a\columnar\columnar\build\columnar\Debug\") { - Write-Host "Moving Windows-based AVX-512 libs" - Move-Item -Path "D:\a\columnar\columnar\build\columnar\Debug\lib_manticore_columnar_avx512.dll" -Destination "D:\a\columnar\columnar\build\columnar\Debug\lib_manticore_columnar.dll" -Force - Move-Item -Path "D:\a\columnar\columnar\build\secondary\Debug\lib_manticore_secondary_avx512.dll" -Destination "D:\a\columnar\columnar\build\secondary\Debug\lib_manticore_secondary.dll" -Force - Move-Item -Path "D:\a\columnar\columnar\build\knn\Debug\lib_manticore_knn_avx512.dll" -Destination "D:\a\columnar\columnar\build\knn\Debug\lib_manticore_knn.dll" -Force - } else { - Write-Host "Directory D:\a\columnar\columnar\build\columnar\Debug\ does not exist" - } - - - name: 🚀 Test - id: test - # --timeout may be not working https://gitlab.kitware.com/cmake/cmake/-/issues/23979 - # Add -VV to ctest to display extra debug info - run: ctest -VV -S cmake/citest.cmake -C $CTEST_CONFIGURATION_TYPE --no-compress-output --timeout 60 - continue-on-error: true - - name: Remember status - if: always() - run: echo "${{ steps.test.outcome }}" > build/status_${{ inputs.CTEST_START }}_${{ inputs.CTEST_END }} - continue-on-error: true - - name: Prepare test results - if: always() - run: | - mkdir build/xml_${{ inputs.CTEST_START }}_${{ inputs.CTEST_END }} - cp build/Testing/2*/Test.xml build/xml_${{ inputs.CTEST_START }}_${{ inputs.CTEST_END }}/ - mv build/_deps/manticore-build/test build/test_${{ inputs.CTEST_START }}_${{ inputs.CTEST_END }} - continue-on-error: true - - name: Upload test artifacts - if: always() - continue-on-error: true - uses: manticoresoftware/upload_artifact_with_retries@main - with: - name: ${{ inputs.artifact_name }} - path: "build/xml* build/test_*/test_*/report* build/test_*/error*.txt build/test_*/*log build/status*" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 4c3ff27d..00000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,227 +0,0 @@ -workflow: - rules: - - if: $CI_MERGE_REQUEST_ID - - if: $CI_COMMIT_TAG - - if: $CI_COMMIT_BRANCH == "master" && $CI_COMMIT_TITLE != $DOCS_AUTOCOMMIT_TITLE - - if: $CI_COMMIT_BRANCH =~ /^columnar-.*$/ && $CI_COMMIT_TITLE != $DOCS_AUTOCOMMIT_TITLE - - if: $CI_PIPELINE_SOURCE == "schedule" - -# to skip ci: git push -o ci.skip ... - -# global settings will be applied to all jobs -variables: - VERBOSE: 1 - DIAGNOSTIC: 1 - -stages: - - mirroring - - routines - - ubertests - - publish - - hooks - -# ==================== Mirror repo to github (only 'master' branch) ====================== - -job_mirror: - stage: mirroring - rules: - - if: $CI_COMMIT_BRANCH == "master" - - if: $CI_COMMIT_BRANCH =~ /^columnar-.*$/ - tags: - - dev-host - dependencies: [] - script: - - ./gitsync_columnar.sh - cache: {} - -# ==================== .helper build ====================== - -.build: - stage: routines - interruptible: true - needs: [ ] - tags: - - docker-free - image: manticoresearch/external_toolchain:clang15_cmake3263 - variables: - CACHEB: $CI_PROJECT_DIR/cache - NO_TESTS: 1 - CTEST_CMAKE_GENERATOR: Ninja - DISTR: bionic # our default build for tests - arch: x86_64 - boost: boost_nov22 - sysroot: roots_nov22 - script: - - ctest -VV -S cmake/citest.cmake --no-compress-output - after_script: - - cd build && mkdir xml - - cp -r Testing/2*/*.xml xml - cache: - paths: - - $CI_PROJECT_DIR/cache - key: bionic_test - artifacts: - when: on_success - expire_in: 12 hrs - paths: - - build/xml - - build/columnar/lib_manticore_columnar.so - - build/secondary/lib_manticore_secondary.so - - build/knn/lib_manticore_knn.so - - build/_deps/manticore-build/src/searchd - - build/_deps/manticore-build/src/indexer - - build/_deps/manticore-build/src/indextool - - build/_deps/manticore-build/config/*.c - - build/_deps/manticore-build/config/*.h - -# ==================== .helper testing ====================== - -.tests: - stage: ubertests - needs: [ ] - tags: - - specdocker - image: manticoresearch/ubertests_ctest:3263 - variables: - CACHEB: $CI_PROJECT_DIR/cache - NO_BUILD: 1 - script: - - ctest -V -S cmake/citest.cmake --no-compress-output - after_script: - - cd build - - cp -r Testing/2*/Test.xml . - - xsltproc -o junit_tests.xml ../misc/junit/ctest2junit.xsl Test.xml - cache: - key: bionic_test - paths: - - $CI_PROJECT_DIR/cache - artifacts: - expire_in: 12 hrs - when: always - reports: - junit: build/junit_tests.xml - paths: - - build/junit_tests.xml - - build/_deps/manticore-build/test/test_*/report.txt - - build/_deps/manticore-build/test/test_*/report.bin - - build/_deps/manticore-build/test/error*.txt - - build/_deps/manticore-build/test/searchd.log - retry: 1 - -# ==================== Simple build, then test ====================== - -simple_build: - extends: [ .build ] - variables: - CTEST_CONFIGURATION_TYPE: Debug - -simple_tests: - extends: [ .tests ] - needs: [ simple_build ] - variables: - CTEST_CONFIGURATION_TYPE: Debug - -# ==================== Release build, then test ====================== - -release_build: - extends: [ .build ] - variables: - CMAKE_INTERPROCEDURAL_OPTIMIZATION: 1 - CTEST_CONFIGURATION_TYPE: RelWithDebInfo - -release_tests: - extends: [ .tests ] - needs: [ release_build ] - variables: - CTEST_CONFIGURATION_TYPE: RelWithDebInfo - -# ==================== Windows build, then test ====================== - -win_build: - extends: [ .build ] - variables: - DISTR: windows - arch: x64 - CTEST_CMAKE_GENERATOR: "Ninja Multi-Config" - CTEST_CONFIGURATION_TYPE: Debug - artifacts: - paths: - - build/xml - - build/columnar/$CTEST_CONFIGURATION_TYPE/lib_manticore_columnar.dll - - build/secondary/$CTEST_CONFIGURATION_TYPE/lib_manticore_secondary.dll - - build/knn/$CTEST_CONFIGURATION_TYPE/lib_manticore_knn.dll - - build/_deps/manticore-build/src/$CTEST_CONFIGURATION_TYPE/indexer.exe - - build/_deps/manticore-build/src/$CTEST_CONFIGURATION_TYPE/searchd.exe -# Commented out usually need, but not on our dev where everything is installed into system -# - build/_deps/manticore-build/src/$CTEST_CONFIGURATION_TYPE/*.dll -# - build/_deps/manticore-build/src/gtests/$CTEST_CONFIGURATION_TYPE/*.dll - cache: - key: win_test - -win_tests: - stage: ubertests - needs: [ win_build ] - tags: - - windev19 - variables: - LIBS_BUNDLE: "x:/bundle" - CACHEB: "x:/cache" - BOOST_ROOT: "x:/boost_1_75_0" - CTEST_CMAKE_GENERATOR: "Visual Studio 16 2019" - CTEST_CONFIGURATION_TYPE: Debug - CTEST_EXCLUDE_REGEX: "_466" - NO_BUILD: 1 - script: - - 'net use x: \\\\VBOXSRV\\shared' - - ctest -VV -S cmake/citest.cmake -C $CTEST_CONFIGURATION_TYPE --no-compress-output - after_script: - - cd build - - cp -r Testing/2*/*.xml xml - artifacts: - expire_in: 12 hrs - when: always - paths: - - build/xml - - build/_deps/manticore-build/test/test_*/report.txt - - build/_deps/manticore-build/test/test_*/report.bin - - build/_deps/manticore-build/test/error*.txt - - build/_deps/manticore-build/test/searchd.log - cache: { } - - -# ==================== Publish (push) pipeline results ====================== - -# build and upload on regular commits -convert_win_junit: - stage: publish - tags: - - docker - image: manticoresearch/ubertests_ctest:3263 - needs: - - win_tests - variables: - GIT_STRATEGY: fetch - script: - - xsltproc -o build/junit_tests.xml misc/junit/ctest2junit.xsl build/xml/Test.xml - artifacts: - paths: - - build/junit_tests.xml - reports: - junit: build/junit_tests.xml - cache: { } - -# ==================== Run secondary pipeline ====================== - -#deploy_hook: -# stage: hooks -# rules: -# - if: $CI_PIPELINE_SOURCE != "schedule" -# trigger: -# include: packaging.yml -# needs: -# - job: simple_tests -# artifacts: false -# - job: release_tests -# artifacts: false -# - job: win_tests -# artifacts: false diff --git a/CMakeLists.txt b/CMakeLists.txt index fc706e3d..66ec0584 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -136,14 +136,9 @@ if (API_ONLY) # below is for support find_package include ( CMakePackageConfigHelpers ) - set ( pkgconfin "${CMAKE_CURRENT_BINARY_DIR}/columnar-config.cmake.in" ) - file ( WRITE "${pkgconfin}" "@PACKAGE_INIT@ - -include(\"\${CMAKE_CURRENT_LIST_DIR}/columnar-targets.cmake\")" ) - - configure_package_config_file ( "${pkgconfin}" - "${CMAKE_CURRENT_BINARY_DIR}/columnar-config.cmake" INSTALL_DESTINATION "${API_CMAKE_DIR}" ) - install ( FILES "${CMAKE_CURRENT_BINARY_DIR}/columnar-config.cmake" DESTINATION "${API_CMAKE_DIR}" ) + set ( pkgconf "${CMAKE_CURRENT_BINARY_DIR}/columnar-config.cmake" ) + file ( WRITE "${pkgconf}" "include(\"\${CMAKE_CURRENT_LIST_DIR}/columnar-targets.cmake\")\n" ) + install ( FILES "${pkgconf}" DESTINATION "${API_CMAKE_DIR}" ) write_basic_package_version_file ( "columnar-config-version.cmake" VERSION "${API_VER}.${SI_API_VER}.${KNN_API_VER}" COMPATIBILITY ExactVersion ARCH_INDEPENDENT ) install ( FILES "${CMAKE_CURRENT_BINARY_DIR}/columnar-config-version.cmake" DESTINATION "${API_CMAKE_DIR}" ) diff --git a/gitsync_columnar.sh b/gitsync_columnar.sh deleted file mode 100755 index 30d84ee0..00000000 --- a/gitsync_columnar.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -# To be run from gitlab CI runner. - -autotag() { - API_VER=$(grep -Po 'LIB_VERSION.* \K\d+' columnar/columnar.h) - SI_API_VER=$(grep -Po 'LIB_VERSION.* \K\d+' secondary/secondary.h) - KNN_API_VER=$(grep -Po 'LIB_VERSION.* \K\d+' knn/knn.h) - AUTO_TAG="c$API_VER-s$SI_API_VER-k$KNN_API_VER" - NUMS=3 - - # check whether all the numbers are available - if [[ $(echo $AUTO_TAG | grep -Po '[0-9]+' | wc -l) = $NUMS ]]; then - # tag is correct - if [[ ! $(git ls-remote github | grep $AUTO_TAG) ]]; then - echo "no tag - will add $AUTO_TAG" - echo "> git tag $AUTO_TAG" - git tag "$AUTO_TAG" - echo "> git push github $AUTO_TAG" - git push github "$AUTO_TAG" - echo "> git status" - git status - else - echo "repo github already has tag $AUTO_TAG, exiting..." - fi - else - echo "generated tag $AUTO_TAG is not valid, do nothing" - fi -} - -echo "> rm -fr gitlab_github_sync" -rm -fr gitlab_github_sync -echo "> git clone git@gitlab.com:manticoresearch/columnar.git gitlab_github_sync" -git clone git@gitlab.com:manticoresearch/columnar.git gitlab_github_sync -echo "> cd gitlab_github_sync" -cd gitlab_github_sync -echo "> git checkout $CI_COMMIT_BRANCH" -git checkout $CI_COMMIT_BRANCH -echo "> git remote add github git@github.com:manticoresoftware/columnar.git" -git remote add github git@github.com:manticoresoftware/columnar.git -echo "> git fetch github" -git fetch github -echo "> git push -u github $CI_COMMIT_BRANCH" -git push -u github "$CI_COMMIT_BRANCH" -if [[ $CI_COMMIT_BRANCH == "master" ]]; then autotag; fi diff --git a/manticore_src.txt b/manticore_src.txt deleted file mode 100644 index 1bf3af66..00000000 --- a/manticore_src.txt +++ /dev/null @@ -1 +0,0 @@ -GIT_REPOSITORY https://github.com/manticoresoftware/manticoresearch.git GIT_TAG fcd62c09bd672092894b0a7eaf847d08d819b5f9 diff --git a/packaging.yml b/packaging.yml deleted file mode 100644 index e7e1c2c8..00000000 --- a/packaging.yml +++ /dev/null @@ -1,298 +0,0 @@ -stages: - - build - - repoupload - -# global settings will be applied to all jobs -variables: - CMAKE_INTERPROCEDURAL_OPTIMIZATION: 1 - VERBOSE: 1 - DIAGNOSTIC: 1 - -workflow: - rules: - - when: always - -# ==================== .helper generic build ====================== - -.build: - stage: build - interruptible: true - rules: - - if: $CI_MERGE_REQUEST_ID - when: manual - - if: $CI_MERGE_REQUEST_ID == null - needs: [ ] - tags: - - docker - image: manticoresearch/external_toolchain:clang15_cmake3263 - variables: - CACHEB: $CI_PROJECT_DIR/cache - arch: x86_64 - boost: boost_nov22 - sysroot: roots_nov22 - script: - - mkdir build && cd build - - cmake -DPACK=1 $XTRAFLAGS .. - - cmake --build . --target package - cache: - key: release_$DISTR$arch - paths: - - $CI_PROJECT_DIR/cache - artifacts: - paths: - - build/manticore*deb - - build/manticore-*.tar.gz - - build/manticore-*.zip - - build/manticore-*.exe - when: on_success - expire_in: 12 hrs - - -.build_rhel: - extends: [ .build ] - before_script: - - ln -s $(pwd) /builds_manticoresearch_dev_usr_src_debug_manticore_component_src_0 - - cd /builds_manticoresearch_dev_usr_src_debug_manticore_component_src_0 - artifacts: - paths: - - build/manticore*rpm - -# ==================== .helper generic and specific collect of built artifacts ====================== - -.collect_job: - variables: - GIT_STRATEGY: fetch - stage: repoupload - interruptible: true - cache: { } - tags: - - dev-host - rules: - - if: $CI_MERGE_REQUEST_ID - when: manual - allow_failure: true - - if: $CI_COMMIT_BRANCH =~ /^columnar-.*$/ - when: manual - - if: $CI_COMMIT_BRANCH == "master" - when: on_success - -.collect_deb: - extends: [ .collect_job ] - script: - - wget https://raw.githubusercontent.com/manticoresoftware/repo_scripts/main/upload_repo_deb - - chmod +x upload_repo_deb - - /bin/bash ./upload_repo_deb - -.collect_rpm: - extends: [ .collect_job ] - script: - - wget https://raw.githubusercontent.com/manticoresoftware/repo_scripts/main/upload_repo_rpm - - chmod +x upload_repo_rpm - - /bin/bash ./upload_repo_rpm - -.collect_arc: - extends: [ .collect_job ] - script: - - wget https://raw.githubusercontent.com/manticoresoftware/repo_scripts/main/upload_repo_arc - - chmod +x upload_repo_arc - - /bin/bash ./upload_repo_arc - -# ==================== Mac OS X ====================== - -osx: - extends: [ .build ] - variables: - DISTR: macos - -osxm1: - extends: [ osx ] - variables: - arch: arm64 - -collect_osx: - extends: [ .collect_arc ] - needs: [ osx, osxm1 ] - - -# ==================== Windows ====================== - -windows: - extends: [ .build ] - variables: - arch: x64 - DISTR: windows - -collect_windows: - extends: [ .collect_arc ] - needs: [ windows ] - - -# ==================== Centos 7 ====================== - -rhel7: - extends: [ .build_rhel ] - variables: - DISTR: rhel7 - -rhel7_arm: - extends: [ rhel7 ] - variables: - arch: aarch64 - -collect_rhel7: - extends: [ .collect_rpm ] - needs: [ rhel7, rhel7_arm ] - variables: - DISTRO: 7 - - -# ==================== Centos 8 ====================== - -rhel8: - extends: [ .build_rhel ] - variables: - DISTR: rhel8 - -rhel8_arm: - extends: [ rhel8 ] - variables: - arch: aarch64 - -collect_rhel8: - extends: [ .collect_rpm ] - needs: [ rhel8, rhel8_arm ] - variables: - DISTRO: 8 - -# ==================== RHEL 9 ====================== - -rhel9: - extends: [ .build_rhel ] - variables: - DISTR: rhel9 - -rhel9_arm: - extends: [ rhel9 ] - variables: - arch: aarch64 - -collect_rhel9: - extends: [ .collect_rpm ] - needs: [ rhel9, rhel9_arm ] - variables: - DISTRO: 9 - -# ==================== Debian-based ====================== - -# ==================== Ubuntu Bionic ====================== - -bionic: - extends: [ .build ] - variables: - DISTR: bionic - -bionic_arm: - extends: [ bionic ] - variables: - arch: aarch64 - -collect_bionic: - extends: [ .collect_deb ] - needs: [ bionic, bionic_arm ] - variables: - DISTRO: bionic - - -# ==================== Ubuntu Focal ====================== - -focal: - extends: [ .build ] - variables: - DISTR: focal - -focal_arm: - extends: [ focal ] - variables: - arch: aarch64 - -collect_focal: - extends: [ .collect_deb ] - needs: [ focal, focal_arm ] - variables: - DISTRO: focal - - -# ==================== Debian Buster ====================== - -buster: - extends: [ .build ] - variables: - DISTR: buster - -buster_arm: - extends: [ buster ] - variables: - arch: aarch64 - -collect_buster: - extends: [ .collect_deb ] - needs: [ buster, buster_arm ] - variables: - DISTRO: buster - - -# ==================== Debian Bullseye ====================== - -bullseye: - extends: [ .build ] - variables: - DISTR: bullseye - -bullseye_arm: - extends: [ bullseye ] - variables: - arch: aarch64 - -collect_bullseye: - extends: [ .collect_deb ] - needs: [ bullseye, bullseye_arm ] - variables: - DISTRO: bullseye - - -# ==================== Ubuntu Jammy ====================== - -jammy: - extends: [ .build ] - variables: - DISTR: jammy - -jammy_arm: - extends: [ jammy ] - variables: - arch: aarch64 - -collect_jammy: - extends: [ .collect_deb ] - needs: [ jammy, jammy_arm ] - variables: - DISTRO: jammy - - -# ==================== Debian Bookworm ====================== - -bookworm: - extends: [ .build ] - variables: - DISTR: bookworm - -bookworm_arm: - extends: [ bookworm ] - variables: - arch: aarch64 - -collect_bookworm: - extends: [ .collect_deb ] - needs: [ bookworm, bookworm_arm ] - variables: - DISTRO: bookworm diff --git a/testing.cmake b/testing.cmake index c57b50f7..fa95bdf0 100644 --- a/testing.cmake +++ b/testing.cmake @@ -22,7 +22,6 @@ function ( special_ubertest_addtest testN tst_name REQUIRES ) endif () endfunction () -# cb called by manticore ubertest - filter out non-columnar or non-secondary here. function ( special_ubertest_filter accept_var explain_var REQUIRES ) if ((NOT COLUMNAR IN_LIST REQUIRES) AND (NOT SECONDARY IN_LIST REQUIRES)) set ( ${accept_var} 0 PARENT_SCOPE ) @@ -30,7 +29,6 @@ function ( special_ubertest_filter accept_var explain_var REQUIRES ) endif () endfunction () -# cb called by manticore ubertest - append path to columnar to given test properties function ( special_ubertest_properties test ) set_property ( TEST "${test}" APPEND PROPERTY ENVIRONMENT "LIB_MANTICORE_COLUMNAR=$" ) set_property ( TEST "${test}" APPEND PROPERTY ENVIRONMENT "LIB_MANTICORE_SECONDARY=$" ) @@ -57,31 +55,18 @@ elseif ( TARGET embeddings ) endif () endif () -# this will switch off pure manticore-specific tests: google, api, keyword consistency and benches (we don't need them here) set ( TEST_SPECIAL_EXTERNAL ON ) -message ( STATUS "Checking MANTICORE_LOCATOR sources..." ) if (DEFINED ENV{MANTICORE_LOCATOR} AND NOT "$ENV{MANTICORE_LOCATOR}" STREQUAL "") set ( MANTICORE_LOCATOR $ENV{MANTICORE_LOCATOR} ) message ( STATUS "Using MANTICORE_LOCATOR from environment: '${MANTICORE_LOCATOR}'" ) -elseif (EXISTS "${columnar_SOURCE_DIR}/local_manticore_src.txt") - file ( READ "${columnar_SOURCE_DIR}/local_manticore_src.txt" MANTICORE_LOCATOR ) - message ( STATUS "Using MANTICORE_LOCATOR from local_manticore_src.txt: '${MANTICORE_LOCATOR}'" ) else () - file ( READ "${columnar_SOURCE_DIR}/manticore_src.txt" MANTICORE_LOCATOR ) - message ( STATUS "Using MANTICORE_LOCATOR from manticore_src.txt: '${MANTICORE_LOCATOR}'" ) + message ( FATAL_ERROR "BUILD_TESTING in standalone MCL repo now requires MANTICORE_LOCATOR to point to a daemon checkout. CI-based cross-repo autodiscovery via manticore_src.txt has been removed." ) endif () -message ( STATUS "MANTICORE_LOCATOR before configure: '${MANTICORE_LOCATOR}'" ) -string ( CONFIGURE "${MANTICORE_LOCATOR}" MANTICORE_LOCATOR ) # that is to expand possible inside variables -message ( STATUS "MANTICORE_LOCATOR after configure: '${MANTICORE_LOCATOR}'" ) - +string ( CONFIGURE "${MANTICORE_LOCATOR}" MANTICORE_LOCATOR ) file ( WRITE "${columnar_BINARY_DIR}/manticore-get.cmake" "FetchContent_Declare ( manticore ${MANTICORE_LOCATOR} )\n" ) -message ( STATUS "Written to ${columnar_BINARY_DIR}/manticore-get.cmake: 'FetchContent_Declare ( manticore ${MANTICORE_LOCATOR} )'" ) include ( FetchContent ) include ( "${columnar_BINARY_DIR}/manticore-get.cmake" ) - -# add manticore sources to the tree. All testing will be done on manticore side; necessary additional tests/properties will -# be set by cb functions defined above. FetchContent_MakeAvailable ( manticore )