ci.yml

name: CI

# Concurrency control:
# - PRs: new commits on a feature branch will cancel in-progress (outdated) runs.
# - Push to develop: runs queue sequentially, never cancelled. This allows us to have benchmarks
#   run on every commit for our benchmarks website.
# - `workflow_dispatch`: groups by branch and queues if run on develop.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }}
on:
  push:
    branches: [develop]
  pull_request: { }
  workflow_dispatch: { }

permissions:
  actions: read
  contents: read
  checks: write  # audit-check creates checks
  issues: write  # audit-check creates issues

env:
  CARGO_TERM_COLOR: auto
  RUST_BACKTRACE: 1
  NIGHTLY_TOOLCHAIN: nightly-2026-02-05

jobs:
  lint-toml:
    runs-on: ubuntu-latest
    timeout-minutes: 40
    steps:
      - uses: actions/checkout@v6
      - uses: spiraldb/actions/.github/actions/lint-toml@0.18.5

  validate-workflow-yaml:
    runs-on: ubuntu-latest
    timeout-minutes: 40
    steps:
      - uses: actions/checkout@v6
      - name: Validate YAML file
        run: |
          # Lint the workflows and yamllint's configuration file.
          yamllint \
            --strict \
            -c .yamllint.yaml \
            .github/

  python-lint:
    name: "Python (lint)"
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=python-lint', github.run_id)
          || 'ubuntu-latest' }}
    timeout-minutes: 40
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-prebuild
      # Use uvx for ruff to avoid building the Rust extension (saves ~4.5 min)
      - name: Python Lint - Format
        run: uvx ruff format --check .
      - name: Python Lint - Ruff
        run: uvx ruff check .
      # PyRight needs the project for type information, so use uv run
      - name: Python Lint - PyRight
        env:
          MATURIN_PEP517_ARGS: "--profile dev"
        run: |
          uv sync --all-packages
          uv run basedpyright vortex-python

  python-test:
    name: "Python (test)"
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=python-test', github.run_id)
          || 'ubuntu-latest' }}
    timeout-minutes: 40
    env:
      RUST_LOG: "info,uv=debug"
      MATURIN_PEP517_ARGS: "--profile dev"
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-prebuild

      - name: Pytest - Vortex
        run: |
          uv run --all-packages pytest --benchmark-disable -n auto test/
        working-directory: vortex-python/

      - name: Setup benchmark environment
        run: sudo bash scripts/setup-benchmark.sh

      - name: Pytest Benchmarks - Vortex
        run: |
          bash ../scripts/bench-taskset.sh uv run --all-packages pytest --benchmark-only benchmark/
        working-directory: vortex-python/

      - name: Doctest - PyVortex
        run: |
          uv run --all-packages make doctest
        working-directory: docs/

      - name: Ensure docs build - PyVortex
        run: |
          uv run --all-packages make html
        working-directory: docs/

  python-wheel-build:
    name: "Python (wheel build)"
    runs-on: ubuntu-latest
    timeout-minutes: 40
    steps:
      - uses: actions/checkout@v6
      - name: Rust Dependency Cache
        uses: Swatinem/rust-cache@v2
        with:
          save-if: ${{ github.ref_name == 'develop' }}
      - uses: ./.github/actions/setup-rust
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
          enable-sccache: "false"
      - uses: mlugg/setup-zig@v2.2.1
      - name: Install uv
        uses: spiraldb/actions/.github/actions/setup-uv@0.18.5
        with:
          sync: false
          prune-cache: false
      - name: Ensure wheel and sdist can be built on Linux - PyVortex
        shell: bash
        run: |
          echo "Clearing wheel target directory"
          rm -rf ../target/wheels/

          uv venv
          uv tool run maturin@1.10 build --interpreter python3.11 --zig
          uv tool run maturin@1.10 build --interpreter python3.11 --zig --sdist

          file_count=$(ls -1 ../target/wheels/ | wc -l)

          if [[ $file_count -ne 2 ]]; then
            echo "Unexpected number of files detected ${file_count}:"
            ls ../target/wheels/
            exit 1
          else
            echo "Generated two files"
          fi
        working-directory: vortex-python/

  rust-docs:
    name: "Rust (docs)"
    timeout-minutes: 40
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-small/image=ubuntu24-full-x64-pre-v2/tag=rust-docs', github.run_id)
          || 'ubuntu-latest' }}
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-prebuild
      - name: Docs
        run: |
          RUSTDOCFLAGS="-D warnings" cargo doc --no-deps
          # nextest doesn't support doc tests, so we run it here
          cargo test --doc --workspace --all-features --exclude vortex-cxx --exclude vortex-jni --exclude vortex-ffi --exclude xtask --no-fail-fast

  build-rust:
    name: "Rust build (${{matrix.config.name}})"
    timeout-minutes: 40
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner={1}/image=ubuntu24-full-x64-pre-v2/tag={2}', github.run_id, matrix.config.runner, matrix.config.name)
          || 'ubuntu-latest' }}
    env:
      # disable lints for build, they will be caught in Rust lint job.
      RUSTFLAGS: "-A warnings"
    strategy:
      fail-fast: false
      matrix:
        config:
          - name: "all-features"
            runner: amd64-large
            args: "--all-features --all-targets"
          - name: "default features"
            runner: amd64-large
            args: "--all-targets"
          - name: "with tokio dispatcher"
            runner: amd64-small
            # Only build the crates that have the tokio features, not re-building other crates with no-default-features
            args: "--no-default-features --features tokio --all-targets -p vortex -p vortex-io -p vortex-file -p vortex-layout"
          - name: "wasm32 with default features"
            runner: amd64-medium
            target: wasm32-unknown-unknown
            env:
              rustflags: "RUSTFLAGS='-A warnings --cfg getrandom_backend=\"wasm_js\"'"
            args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-cuda --exclude vortex-cub --exclude vortex-nvcomp --exclude vortex-datafusion --exclude vortex-duckdb --exclude vortex-tui --exclude vortex-zstd --exclude vortex-test-e2e-cuda --exclude vortex-sqllogictest"
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-prebuild
      - name: Install wasm32 target
        if: ${{ matrix.config.target == 'wasm32-unknown-unknown' }}
        run: rustup target add wasm32-unknown-unknown
      - uses: ./.github/actions/check-rebuild
        with:
          command: "${{matrix.config.env.rustflags}} cargo hack build --locked ${{matrix.config.args}} --ignore-private"
      - name: "Make sure no files changed after build"
        run: |
          git status --porcelain
          test -z "$(git status --porcelain)"

  check-min-deps:
    name: "Check build with minimal dependencies"
    timeout-minutes: 40
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-min-deps', github.run_id)
          || 'ubuntu-latest' }}
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-prebuild
      - run: cargo minimal-versions check --direct --workspace --ignore-private

  rust-lint:
    name: "Rust (lint)"
    timeout-minutes: 40
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=rust-lint', github.run_id)
          || 'ubuntu-latest' }}
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-prebuild
      - name: Install nightly for fmt
        run: rustup toolchain install $NIGHTLY_TOOLCHAIN --component rustfmt
      - name: Rust Lint - Format
        run: cargo +$NIGHTLY_TOOLCHAIN fmt --all --check
      - name: Rustc check
        run: RUSTFLAGS="-D warnings" cargo check --locked --all-features --all-targets
      - name: Rustc check (release)
        run: RUSTFLAGS="-D warnings" cargo check --locked --all-features --all-targets --release
      - name: Rust Lint - Clippy All Features
        run: cargo clippy --locked --all-features --all-targets -- -D warnings
      - name: Rust Lint - Clippy Default Features
        run: cargo clippy --locked --all-targets -- -D warnings

  cpp-lint:
    name: "C/C++ (lint)"
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v6
      - name: C/C++ Lint - clang-format
        run: |
          git ls-files vortex-cuda vortex-cxx vortex-duckdb vortex-ffi \
            | grep -E '\.(cpp|hpp|cu|cuh|h)$' \
            | grep -v 'kernels/src/bit_unpack_.*\.cu$' \
            | xargs clang-format --dry-run --Werror --style=file

  rust-lint-no-default:
    name: "Rust (lint, no default)"
    timeout-minutes: 40
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-lint-no-default', github.run_id)
          || 'ubuntu-latest' }}
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-prebuild
      - name: Rust Lint - Clippy No Default Features
        shell: bash
        run: |
          cargo hack --no-dev-deps --ignore-private clippy --no-default-features -- -D warnings

  public-api:
    name: "Public API lock files"
    timeout-minutes: 40
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-xsmall/image=ubuntu24-full-x64-pre-v2/tag=public-api', github.run_id)
          || 'ubuntu-latest' }}
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-prebuild
      - name: Install nightly for public-api
        run: rustup toolchain install $NIGHTLY_TOOLCHAIN
      - name: Regenerate public API lock files
        run: cargo +$NIGHTLY_TOOLCHAIN xtask public-api
      - name: Verify lock files are up to date
        run: |
          if ! git diff --quiet '**/public-api.lock'; then
            git diff -U0 '**/public-api.lock'
            echo ""
            echo ""
            echo "Changed files:"
            git diff --name-only '**/public-api.lock'
            echo ""
            echo ""
            echo "Public API lock files are out of date."
            echo "Run './scripts/public-api.sh' locally to"
            echo "regenerate them, then commit the changes."
            exit 1
          fi

  rust-coverage:
    name: "Rust tests (coverage) (${{ matrix.suite }})"
    timeout-minutes: 40
    permissions:
      id-token: write
    strategy:
      matrix:
        include:
          - suite: tests
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=rust-coverage-suite-{1}', github.run_id, matrix.suite)
          || 'ubuntu-latest' }}
    env:
      RUSTFLAGS: "-Cinstrument-coverage -A warnings"
      CARGO_INCREMENTAL: 0  # Disable incremental compilation to get accurate coverage
      LLVM_PROFILE_FILE: "target/coverage/vortex-%p-%m.profraw"
      GRCOV_OUTPUT_FILE: "target/coverage/vortex.lcov"
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-prebuild
      - name: Rust Tests
        if: ${{ matrix.suite == 'tests' }}
        run: |
          cargo nextest run --locked --workspace --all-features --no-fail-fast
      - name: Generate coverage report
        run: |
          grcov . --binary-path target/debug/ -s . -t lcov --llvm --ignore-not-existing \
            --threads $(nproc) \
            --ignore '../*' --ignore '/*' --ignore 'fuzz/*' --ignore 'vortex-bench/*' \
            --ignore 'home/*' --ignore 'xtask/*' --ignore 'target/*' --ignore 'vortex-error/*' \
            --ignore 'vortex-python/*' --ignore 'vortex-jni/*' --ignore 'vortex-flatbuffers/*' \
            --ignore 'vortex-proto/*' --ignore 'vortex-tui/*' --ignore 'vortex-datafusion/examples/*' \
            --ignore 'vortex-ffi/examples/*' --ignore '*/arbitrary/*' --ignore '*/arbitrary.rs' --ignore 'vortex-cxx/*' \
            --ignore benchmarks/* --ignore 'vortex-test/*' \
            -o ${{ env.GRCOV_OUTPUT_FILE }}
      - name: Codecov
        uses: codecov/codecov-action@v5
        with:
          name: run-${{ matrix.suite }}
          files: ${{ env.GRCOV_OUTPUT_FILE }}
          disable_search: true
          flags: ${{ matrix.suite }}
          use_oidc: true

  rust-test-sanitizer:
    strategy:
      fail-fast: false
      matrix:
        include:
          # We don't run memory sanitizer as it provides many false positives
          # for std
          - sanitizer: asan
            sanitizer_flags: "-Zsanitizer=address,leak"
          - sanitizer: tsan
            sanitizer_flags: "-Zsanitizer=thread"
    name: "Rust tests (${{ matrix.sanitizer }})"
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/pool=amd64-medium-pre-v2/tag=rust-test-sanitizer', github.run_id)
          || 'ubuntu-latest' }}
    timeout-minutes: 40
    env:
      ASAN_OPTIONS: "symbolize=1:check_initialization_order=1:detect_leaks=1:leak_check_at_exit=1"
      LSAN_OPTIONS: "report_objects=1"
      ASAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
      MSAN_OPTIONS: "symbolize=1"
      MSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
      TSAN_OPTIONS: "symbolize=1"
      TSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
      VORTEX_SKIP_SLOW_TESTS: "1"
      # -Cunsafe-allow-abi-mismatch=sanitizer: libraries like compiler_builtins
      # unset -Zsanitizer flag and we should allow that.
      RUSTFLAGS: "-A warnings -Cunsafe-allow-abi-mismatch=sanitizer --cfg disable_loom --cfg vortex_nightly -C debuginfo=2 -C opt-level=0 -C strip=none"
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-prebuild
      - name: Install Rust nightly toolchain
        run: |
          rustup toolchain install $NIGHTLY_TOOLCHAIN
          rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview
      - name: Build tests with sanitizer
        run: |
          RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \
          cargo +$NIGHTLY_TOOLCHAIN build --locked --all-features \
            --target x86_64-unknown-linux-gnu -Zbuild-std \
            -p vortex-buffer -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array

      - name: Run tests with sanitizer
        run: |
          RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \
          cargo +$NIGHTLY_TOOLCHAIN nextest run --locked --all-features \
            --target x86_64-unknown-linux-gnu --no-fail-fast -Zbuild-std \
            -p vortex-buffer -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array

        # vortex-ffi requires --no-default-features as otherwise we pull in
        # Mimalloc which interferes with sanitizers
        # cargo nextest reports less sanitizer issues than cargo test
        # TODO(myrrc): remove --no-default-features once we make Mimalloc opt-in
      - name: Run vortex-ffi tests with sanitizer
        run: |
          RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \
          cargo +$NIGHTLY_TOOLCHAIN test --locked --no-default-features \
            --target x86_64-unknown-linux-gnu --no-fail-fast -Zbuild-std \
            -p vortex-ffi -- --no-capture

  rust-ffi-test-sanitizer:
    strategy:
      fail-fast: false
      matrix:
        include:
          # We don't run memory sanitizer as it's clang-only and provides many
          # false positives for Catch2
          - sanitizer: asan
            sanitizer_flags: "-Zsanitizer=address,leak"
          - sanitizer: tsan
            sanitizer_flags: "-Zsanitizer=thread"
    name: "Rust/C++ FFI tests (${{ matrix.sanitizer }})"
    timeout-minutes: 40
    env:
      ASAN_OPTIONS: "symbolize=1:check_initialization_order=1:detect_leaks=1:leak_check_at_exit=1"
      LSAN_OPTIONS: "report_objects=1"
      ASAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
      MSAN_OPTIONS: "symbolize=1"
      MSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
      TSAN_OPTIONS: "symbolize=1"
      TSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
      VORTEX_SKIP_SLOW_TESTS: "1"
      # -Cunsafe-allow-abi-mismatch=sanitizer: libraries like compiler_builtins
      # unset -Zsanitizer flag and we should allow that.
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-ffi-test-sanitizer', github.run_id)
          || 'ubuntu-latest' }}
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-prebuild
      - name: Install rustfilt
        run: |
          cargo install rustfilt
      - name: Install Rust nightly toolchain
        run: |
          rustup toolchain install $NIGHTLY_TOOLCHAIN
          rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview
      - name: Build FFI library
        run: |
          # TODO(myrrc): remove --no-default-features
          RUSTFLAGS="-A warnings -Cunsafe-allow-abi-mismatch=sanitizer \
                --cfg disable_loom --cfg vortex_nightly -C debuginfo=2 \
                -C opt-level=0 -C strip=none -Zexternal-clangrt \
                ${{ matrix.sanitizer_flags }}" \
          cargo +$NIGHTLY_TOOLCHAIN build --locked --no-default-features \
            --target x86_64-unknown-linux-gnu -Zbuild-std \
            -p vortex-ffi
      - name: Build FFI library tests
        run: |
          cd vortex-ffi
          cmake -Bbuild -DBUILD_TESTS=1 -DSANITIZER=${{ matrix.sanitizer }} -DTARGET_TRIPLE="x86_64-unknown-linux-gnu"
          cmake --build build -j
      - name: Run tests
        run: |
          set -o pipefail
          ./vortex-ffi/build/test/vortex_ffi_test 2>&1 | rustfilt -i-

  cuda-build-lint:
    if: github.repository == 'vortex-data/vortex'
    name: "CUDA build & lint"
    timeout-minutes: 40
    runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-build
    steps:
      - uses: runs-on/action@v2
        with:
          sccache: s3
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-rust
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - uses: ./.github/actions/check-rebuild
        with:
          command: >-
            cargo build --locked --all-features --all-targets
            -p vortex-cuda -p vortex-cub -p vortex-nvcomp
            -p gpu-scan-cli -p vortex-test-e2e-cuda
      - name: Clippy CUDA crates
        run: |
          cargo clippy --locked --all-features --all-targets \
            -p vortex-cuda \
            -p vortex-cub \
            -p vortex-nvcomp \
            -p gpu-scan-cli \
            -p vortex-test-e2e-cuda \
            -- -D warnings

  cuda-test:
    if: github.repository == 'vortex-data/vortex'
    name: "CUDA tests"
    timeout-minutes: 30
    runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-tests
    steps:
      - uses: runs-on/action@v2
        with:
          sccache: s3
      - name: Display NVIDIA SMI details
        run: |
          nvidia-smi
          nvidia-smi -L
          nvidia-smi -q -d Memory
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-rust
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Install nextest
        uses: taiki-e/install-action@v2
        with:
          tool: nextest
      - name: Rust Tests
        env:
          FLAT_LAYOUT_INLINE_ARRAY_NODE: true
        run: |
          cargo nextest run \
            --locked \
            -p vortex-file \
            -p vortex-cuda \
            -p vortex-cub \
            -p vortex-nvcomp \
            -p vortex-test-e2e-cuda \
            --all-features \
            --no-fail-fast \
            --target x86_64-unknown-linux-gnu \
            --verbose

  cuda-test-sanitizer:
    if: github.repository == 'vortex-data/vortex'
    name: "CUDA tests (${{ matrix.sanitizer }})"
    timeout-minutes: 30
    runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-sanitizer
    strategy:
      fail-fast: false
      matrix:
        include:
          - sanitizer: memcheck
            runner_flags: "--tool memcheck --leak-check=full --error-exitcode 1"
          # TODO(joe): try to re-enable racecheck, it is hanging in CI.
          # - sanitizer: racecheck
          #   runner_flags: "--tool racecheck --error-exitcode 1"
          - sanitizer: synccheck
            runner_flags: "--tool synccheck --error-exitcode 1"
          - sanitizer: initcheck
            runner_flags: "--tool initcheck --error-exitcode 1"
    steps:
      - uses: runs-on/action@v2
        with:
          sccache: s3
      - name: Display NVIDIA SMI details
        run: |
          nvidia-smi
          nvidia-smi -L
          nvidia-smi -q -d Memory
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-rust
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Build tests
        run: cargo test --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu --no-run
      - name: "CUDA - ${{ matrix.sanitizer }}"
        env:
          CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: "compute-sanitizer ${{ matrix.runner_flags }}"
        run: cargo test --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu

  cuda-test-cudf:
    if: github.repository == 'vortex-data/vortex'
    name: "CUDA tests (cudf)"
    timeout-minutes: 30
    runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-cudf
    steps:
      - uses: runs-on/action@v2
        with:
          sccache: s3
      - name: Display NVIDIA SMI details
        run: |
          nvidia-smi
          nvidia-smi -L
          nvidia-smi -q -d Memory
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-rust
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Build cudf test library
        run: cargo build --locked -p vortex-test-e2e-cuda --target x86_64-unknown-linux-gnu
      - name: Download and run cudf-test-harness
        run: |
          curl -fsSL https://github.com/vortex-data/cudf-test-harness/releases/latest/download/cudf-test-harness-x86_64.tar.gz | tar -xz
          cd cudf-test-harness-x86_64
          compute-sanitizer --tool memcheck --error-exitcode 1 ./cudf-test-harness check $GITHUB_WORKSPACE/target/x86_64-unknown-linux-gnu/debug/libvortex_test_e2e_cuda.so

  rust-test-other:
    name: "Rust tests (${{ matrix.os }})"
    timeout-minutes: 40
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: windows-x64
            runner: runs-on=${{ github.run_id }}/pool=windows-x64-pre
            fallback_runner: windows-latest
          - os: linux-arm64
            runner: runs-on=${{ github.run_id }}/runner=arm64-medium/image=ubuntu24-full-arm64-pre-v2/tag=rust-test-linux-arm64
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && matrix.runner
          || matrix.fallback_runner }}
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@v6
      - name: Setup (Windows)
        if: matrix.os == 'windows-x64'
        run: |
          $flags = '-C debuginfo=0'
          echo "RUSTFLAGS=$flags" >> $env:GITHUB_ENV
          echo "C:\rust\cargo\bin" >> $env:GITHUB_PATH
      - uses: ./.github/actions/setup-prebuild
      - name: Rust Tests (Windows)
        if: matrix.os == 'windows-x64'
        run: |
          cargo nextest run --locked --workspace --all-features --no-fail-fast `
            --exclude vortex-bench --exclude vortex-python --exclude vortex-duckdb `
            --exclude vortex-fuzz --exclude vortex-cuda --exclude vortex-nvcomp `
            --exclude vortex-cub --exclude vortex-test-e2e-cuda --exclude duckdb-bench `
            --exclude lance-bench --exclude datafusion-bench --exclude random-access-bench `
            --exclude compress-bench --exclude xtask --exclude vortex-datafusion `
            --exclude gpu-scan-cli --exclude vortex-sqllogictest
      - name: Rust Tests (Other)
        if: matrix.os != 'windows-x64'
        run: |
          cargo nextest run --locked --workspace --all-features --no-fail-fast --exclude vortex-bench --exclude xtask --exclude vortex-sqllogictest
      - uses: ./.github/actions/check-rebuild
        if: matrix.os != 'windows-x64'
        with:
          command: "cargo test --locked --workspace --all-features --no-run --exclude vortex-bench --exclude xtask --exclude vortex-sqllogictest"

      - name: Alert incident.io
        if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/develop'
        uses: ./.github/actions/alert-incident-io
        with:
          api-key: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }}
          alert-title: "Rust tests (${{ matrix.os }}) failed on develop"
          deduplication-key: ci-rust-test-${{ matrix.os }}-failure

  build-java:
    name: "Java"
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/pool=amd64-medium-pre-v2/tag=java', github.run_id)
          || 'ubuntu-latest' }}
    timeout-minutes: 40
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-prebuild
      - run: ./gradlew test --parallel
        working-directory: ./java

  bench-codspeed:
    strategy:
      matrix:
        include:
          - { shard: 1, name: "Core foundation", packages: "vortex-buffer vortex-error" }
          - { shard: 2, name: "Arrays", packages: "vortex-array", features: "--features _test-harness" }
          - { shard: 3, name: "Main library", packages: "vortex" }
          - { shard: 4, name: "Encodings 1", packages: "vortex-alp vortex-bytebool vortex-datetime-parts" }
          - { shard: 5, name: "Encodings 2", packages: "vortex-decimal-byte-parts vortex-fastlanes vortex-fsst", features: "--features _test-harness" }
          - { shard: 6, name: "Encodings 3", packages: "vortex-pco vortex-runend vortex-sequence" }
          - { shard: 7, name: "Encodings 4", packages: "vortex-sparse vortex-zigzag vortex-zstd" }
          - { shard: 8, name: "Storage formats", packages: "vortex-flatbuffers vortex-proto vortex-btrblocks" }
    name: "Benchmark with Codspeed (Shard #${{ matrix.shard }})"
    timeout-minutes: 40
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=bench-codspeed-{1}', github.run_id, matrix.shard)
          || 'ubuntu-latest' }}
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@v6
      - name: Setup benchmark environment
        run: sudo bash scripts/setup-benchmark.sh
      - uses: ./.github/actions/setup-prebuild
      - uses: ./.github/actions/system-info
      - name: Install Codspeed
        uses: taiki-e/cache-cargo-install-action@511847d4974b3aed52d8daf28cbea5313caf34f3
        with:
          tool: cargo-codspeed
      - name: Build benchmarks
        env:
          RUSTFLAGS: "-C target-feature=+avx2"
        run: cargo codspeed build ${{ matrix.features }} $(printf -- '-p %s ' ${{ matrix.packages }}) --profile bench
      - name: Run benchmarks
        uses: CodSpeedHQ/action@d872884a306dd4853acf0f584f4b706cf0cc72a2
        with:
          run: bash scripts/bench-taskset.sh cargo codspeed run
          token: ${{ secrets.CODSPEED_TOKEN }}
          mode: "simulation"

  license-check-and-audit-check:
    name: License Check and Audit Check
    runs-on: ubuntu-latest
    timeout-minutes: 40
    strategy:
      matrix:
        checks:
          - advisories
          - bans licenses sources
    # Prevent sudden announcement of a new advisory from failing ci:
    continue-on-error: ${{ matrix.checks == 'advisories' }}
    steps:
      - uses: actions/checkout@v6
      - uses: EmbarkStudios/cargo-deny-action@v2
        with:
          command: check ${{ matrix.checks }}

  cxx-test:
    name: "C++ build"
    timeout-minutes: 40
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=cxx-build', github.run_id)
          || 'ubuntu-latest' }}
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-prebuild
      - name: Build and run C++ unit tests
        run: |
          mkdir -p vortex-cxx/build
          cmake -S vortex-cxx -B vortex-cxx/build -DVORTEX_ENABLE_TESTING=ON -DVORTEX_ENABLE_ASAN=ON
          cmake --build vortex-cxx/build --parallel $(nproc)
          ctest --test-dir vortex-cxx/build -j $(nproc) -V
      - name: Build and run the example in release mode
        run: |
          cmake -S vortex-cxx/examples -B vortex-cxx/examples/build -DCMAKE_BUILD_TYPE=Release
          cmake --build vortex-cxx/examples/build --parallel $(nproc)
          vortex-cxx/examples/build/hello-vortex vortex-cxx/examples/goldenfiles/example.vortex
      - uses: ./.github/actions/check-rebuild
        with:
          command: "cargo build --locked -p vortex-cxx --lib"

  sqllogic-test:
    name: "SQL logic tests"
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=sql-logic-test', github.run_id)
          || 'ubuntu-latest' }}
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-prebuild
      - name: Run sqllogictest tests
        run: |
          ./vortex-sqllogictest/slt/tpch/generate_data.sh
          cargo test -p vortex-sqllogictest --test sqllogictests
      - uses: ./.github/actions/check-rebuild
        with:
          command: "cargo test -p vortex-sqllogictest --test sqllogictests --no-run"

  wasm-integration:
    name: "wasm-integration"
    runs-on: ubuntu-latest
    timeout-minutes: 40
    steps:
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-rust
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
          targets: "wasm32-wasip1"
      - name: Setup Wasmer
        shell: bash
        run: |
          curl https://get.wasmer.io -sSfL | sh
          echo "$HOME/.wasmer/bin" >> $GITHUB_PATH
      - run: cargo build --target wasm32-wasip1
        working-directory: ./wasm-test
      - run: wasmer run ./target/wasm32-wasip1/debug/wasm-test.wasm
        working-directory: ./wasm-test

  miri:
    name: "Rust tests (miri)"
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-miri', github.run_id)
          || 'ubuntu-latest' }}
    timeout-minutes: 40
    env:
      MIRIFLAGS: -Zmiri-strict-provenance -Zmiri-symbolic-alignment-check -Zmiri-disable-isolation -Zmiri-env-forward=RUST_BACKTRACE
      RUSTFLAGS: "-A warnings --cfg vortex_nightly"
      RUST_BACKTRACE: full
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-prebuild
      - name: Install nightly with miri
        run: rustup toolchain install $NIGHTLY_TOOLCHAIN --component rust-src,rustfmt,clippy,miri
      - name: Run Miri
        run: cargo +$NIGHTLY_TOOLCHAIN miri nextest run --no-fail-fast -p vortex-buffer -p vortex-ffi

  generated-files:
    name: "Check generated source files are up to date"
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=generated-files', github.run_id)
          || 'ubuntu-latest' }}
    timeout-minutes: 40
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-prebuild
      - name: Install nightly for cbindgen macro expansion
        run: rustup toolchain install $NIGHTLY_TOOLCHAIN
      - name: "regenerate all .fbs/.proto Rust code"
        run: |
          cargo xtask generate-fbs
          cargo xtask generate-proto
      - name: "regenerate FFI header file"
        run: |
          cargo +$NIGHTLY_TOOLCHAIN build -p vortex-ffi
      - name: "Make sure no files changed after regenerating"
        run: |
          git status --porcelain
          test -z "$(git status --porcelain)"

      - name: "Checkout develop flatbuffers"
        working-directory: vortex-flatbuffers/
        run: |
          cp -R flatbuffers flatbuffers.HEAD
          git fetch origin develop --depth 1
          git checkout origin/develop -- flatbuffers
      - name: "Verify flatbuffer back-compat"
        working-directory: vortex-flatbuffers/
        run: |
          find flatbuffers/ -type f -name "*.fbs" | sed 's/^flatbuffers\///' | xargs -I{} -n1 flatc -I flatbuffers.HEAD --conform-includes flatbuffers --conform flatbuffers/{} flatbuffers.HEAD/{}

  ffi-c-test:
    name: "C API test build"
    timeout-minutes: 10
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=cxx-build', github.run_id)
          || 'ubuntu-latest' }}
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-prebuild
      - name: "regenerate FFI header file"
        run: |
          cargo +$NIGHTLY_TOOLCHAIN build -p vortex-ffi
      - name: Build and run C++ unit tests
        run: |
          cd vortex-ffi
          mkdir build
          cmake -Bbuild
          cmake --build build -j $(nproc)
          ctest --test-dir build -j $(nproc)

  check-java-publish-build:
    runs-on: ${{ matrix.target.runs-on }}
    container:
      image: "ubuntu:20.04"
    timeout-minutes: 40
    strategy:
      fail-fast: false
      matrix:
        target:
          - { os: ubuntu, runs-on: "ubuntu-24.04-arm", target: aarch64-unknown-linux-gnu }
          - { os: ubuntu, runs-on: "ubuntu-24.04", target: x86_64-unknown-linux-gnu }
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
      - uses: ./.github/actions/prepare-java-linux
      - uses: actions/setup-java@v5
        with:
          distribution: "corretto"
          java-version: "17"
      - uses: ./.github/actions/setup-rust
        with:
          targets: ${{ matrix.target.target }}
          repo-token: ${{ secrets.GITHUB_TOKEN }}
          enable-sccache: "false"
      - run: cargo build --package vortex-jni

  compat-check:
    name: "Compat check"
    uses: ./.github/workflows/compat-validation.yml
    with:
      mode: last

  rust-publish-dry-run:
    name: "Rust publish dry-run"
    timeout-minutes: 40
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-xsmall/image=ubuntu24-full-x64-pre-v2/tag=rust-publish-dry-run', github.run_id)
          || 'ubuntu-latest' }}
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@v6
      - uses: ./.github/actions/setup-prebuild
      - name: Install cargo-edit
        uses: taiki-e/cache-cargo-install-action@511847d4974b3aed52d8daf28cbea5313caf34f3
        with:
          tool: cargo-edit
      - name: Set Version
        run: |
          # This is mostly a dummy version, we don't actually publish anything but it cannot exist in crates.io
          cargo set-version --workspace 0.100000.0
      - name: Test Release
        run: |
          cargo publish --dry-run --no-verify --allow-dirty --workspace