More splits

Signed-off-by: Adam Gutglick <adam@spiraldb.com>

Author: AdamGS

.github/workflows/ci.yml

@@ -2,8 +2,7 @@ name: Linters and Tests
 
 # Concurrency control:
 # - PRs: new commits on a feature branch will cancel in-progress (outdated) runs.
-# - Push to develop: runs queue sequentially, never cancelled. This allows us to have benchmarks
-#   run on every commit for our benchmarks website.
+# - Push to develop: runs queue sequentially, never cancelled.
 # - `workflow_dispatch`: groups by branch and queues if run on develop.
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -292,313 +291,6 @@ jobs:
             exit 1
           fi
 
-  rust-coverage:
-    name: "Rust tests (coverage) (${{ matrix.suite }})"
-    timeout-minutes: 30
-    permissions:
-      id-token: write
-    strategy:
-      matrix:
-        include:
-          - suite: tests
-    runs-on: >-
-      ${{ github.repository == 'vortex-data/vortex'
-          && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=rust-coverage-suite-{1}', github.run_id, matrix.suite)
-          || 'ubuntu-latest' }}
-    env:
-      RUSTFLAGS: "-Cinstrument-coverage -A warnings"
-      CARGO_INCREMENTAL: 0  # Disable incremental compilation to get accurate coverage
-      LLVM_PROFILE_FILE: "target/coverage/vortex-%p-%m.profraw"
-      GRCOV_OUTPUT_FILE: "target/coverage/vortex.lcov"
-    steps:
-      - uses: runs-on/action@v2
-        if: github.repository == 'vortex-data/vortex'
-        with:
-          sccache: s3
-      - uses: actions/checkout@v6
-      - uses: ./.github/actions/setup-prebuild
-      - name: Rust Tests
-        if: ${{ matrix.suite == 'tests' }}
-        run: |
-          cargo nextest run --locked --workspace --all-features --no-fail-fast
-      - name: Generate coverage report
-        run: |
-          grcov . --binary-path target/debug/ -s . -t lcov --llvm --ignore-not-existing \
-            --threads $(nproc) \
-            --ignore '../*' --ignore '/*' --ignore 'fuzz/*' --ignore 'vortex-bench/*' \
-            --ignore 'home/*' --ignore 'xtask/*' --ignore 'target/*' --ignore 'vortex-error/*' \
-            --ignore 'vortex-python/*' --ignore 'vortex-jni/*' --ignore 'vortex-flatbuffers/*' \
-            --ignore 'vortex-proto/*' --ignore 'vortex-tui/*' --ignore 'vortex-datafusion/examples/*' \
-            --ignore 'vortex-ffi/examples/*' --ignore '*/arbitrary/*' --ignore '*/arbitrary.rs' --ignore 'vortex-cxx/*' \
-            --ignore benchmarks/* --ignore 'vortex-test/*' \
-            -o ${{ env.GRCOV_OUTPUT_FILE }}
-      - name: Codecov
-        uses: codecov/codecov-action@v5
-        with:
-          name: run-${{ matrix.suite }}
-          files: ${{ env.GRCOV_OUTPUT_FILE }}
-          disable_search: true
-          flags: ${{ matrix.suite }}
-          use_oidc: true
-
-  rust-test-sanitizer:
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          # We don't run memory sanitizer as it provides many false positives
-          # for std
-          - sanitizer: asan
-            sanitizer_flags: "-Zsanitizer=address,leak"
-          - sanitizer: tsan
-            sanitizer_flags: "-Zsanitizer=thread"
-    name: "Rust tests (${{ matrix.sanitizer }})"
-    runs-on: >-
-      ${{ github.repository == 'vortex-data/vortex'
-          && format('runs-on={0}/pool=amd64-medium-pre-v2/tag=rust-test-sanitizer', github.run_id)
-          || 'ubuntu-latest' }}
-    timeout-minutes: 30
-    env:
-      ASAN_OPTIONS: "symbolize=1:check_initialization_order=1:detect_leaks=1:leak_check_at_exit=1"
-      LSAN_OPTIONS: "report_objects=1"
-      ASAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
-      MSAN_OPTIONS: "symbolize=1"
-      MSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
-      TSAN_OPTIONS: "symbolize=1:suppressions=${{ github.workspace }}/vortex-ffi/tsan_suppressions.txt"
-      TSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
-      VORTEX_SKIP_SLOW_TESTS: "1"
-      # -Cunsafe-allow-abi-mismatch=sanitizer: libraries like compiler_builtins
-      # unset -Zsanitizer flag and we should allow that.
-      RUSTFLAGS: "-A warnings -Cunsafe-allow-abi-mismatch=sanitizer -C debuginfo=2 -C opt-level=0 -C strip=none"
-    steps:
-      - uses: runs-on/action@v2
-        if: github.repository == 'vortex-data/vortex'
-        with:
-          sccache: s3
-      - uses: actions/checkout@v6
-      - uses: ./.github/actions/setup-prebuild
-      - name: Install Rust nightly toolchain
-        run: |
-          rustup toolchain install $NIGHTLY_TOOLCHAIN
-          rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview
-      - name: Build tests with sanitizer
-        run: |
-          RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \
-          cargo +$NIGHTLY_TOOLCHAIN build --locked --all-features \
-            --target x86_64-unknown-linux-gnu -Zbuild-std \
-            -p vortex-buffer -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array
-
-      - name: Run tests with sanitizer
-        run: |
-          RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \
-          cargo +$NIGHTLY_TOOLCHAIN nextest run --locked --all-features \
-            --target x86_64-unknown-linux-gnu --no-fail-fast -Zbuild-std \
-            -p vortex-buffer -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array
-
-        # vortex-ffi requires --no-default-features as otherwise we pull in
-        # Mimalloc which interferes with sanitizers
-        # cargo nextest reports less sanitizer issues than cargo test
-        # TODO(myrrc): remove --no-default-features once we make Mimalloc opt-in
-      - name: Run vortex-ffi tests with sanitizer
-        run: |
-          RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \
-          cargo +$NIGHTLY_TOOLCHAIN test --locked --no-default-features \
-            --target x86_64-unknown-linux-gnu --no-fail-fast -Zbuild-std \
-            -p vortex-ffi -- --no-capture
-
-  rust-ffi-test-sanitizer:
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          # We don't run memory sanitizer as it's clang-only and provides many
-          # false positives for Catch2
-          - sanitizer: asan
-            sanitizer_flags: "-Zsanitizer=address,leak"
-          - sanitizer: tsan
-            sanitizer_flags: "-Zsanitizer=thread"
-    name: "Rust/C++ FFI tests (${{ matrix.sanitizer }})"
-    timeout-minutes: 30
-    env:
-      ASAN_OPTIONS: "symbolize=1:check_initialization_order=1:detect_leaks=1:leak_check_at_exit=1"
-      LSAN_OPTIONS: "report_objects=1"
-      ASAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
-      MSAN_OPTIONS: "symbolize=1"
-      MSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
-      TSAN_OPTIONS: "symbolize=1"
-      TSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
-      VORTEX_SKIP_SLOW_TESTS: "1"
-      # -Cunsafe-allow-abi-mismatch=sanitizer: libraries like compiler_builtins
-      # unset -Zsanitizer flag and we should allow that.
-    runs-on: >-
-      ${{ github.repository == 'vortex-data/vortex'
-          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-ffi-test-sanitizer', github.run_id)
-          || 'ubuntu-latest' }}
-    steps:
-      - uses: runs-on/action@v2
-        if: github.repository == 'vortex-data/vortex'
-        with:
-          sccache: s3
-      - uses: actions/checkout@v6
-      - uses: ./.github/actions/setup-prebuild
-      - name: Install rustfilt
-        run: |
-          cargo install rustfilt
-      - name: Install Rust nightly toolchain
-        run: |
-          rustup toolchain install $NIGHTLY_TOOLCHAIN
-          rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview
-      - name: Build FFI library
-        run: |
-          # TODO(myrrc): remove --no-default-features
-          RUSTFLAGS="-A warnings -Cunsafe-allow-abi-mismatch=sanitizer \
-                -C debuginfo=2 -C opt-level=0 -C strip=none -Zexternal-clangrt \
-                ${{ matrix.sanitizer_flags }}" \
-          cargo +$NIGHTLY_TOOLCHAIN build --locked --no-default-features \
-            --target x86_64-unknown-linux-gnu -Zbuild-std \
-            -p vortex-ffi
-      - name: Build FFI library tests
-        run: |
-          cd vortex-ffi
-          cmake -Bbuild -DBUILD_TESTS=1 -DSANITIZER=${{ matrix.sanitizer }} -DTARGET_TRIPLE="x86_64-unknown-linux-gnu"
-          cmake --build build -j
-      - name: Run tests
-        run: |
-          set -o pipefail
-          ./vortex-ffi/build/test/vortex_ffi_test 2>&1 | rustfilt -i-
-
-  cuda-build-lint:
-    if: github.repository == 'vortex-data/vortex'
-    name: "CUDA build & lint"
-    timeout-minutes: 30
-    runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-build
-    steps:
-      - uses: runs-on/action@v2
-        with:
-          sccache: s3
-      - uses: actions/checkout@v6
-      - uses: ./.github/actions/setup-rust
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: ./.github/actions/check-rebuild
-        with:
-          command: >-
-            cargo build --profile ci --locked --all-features --all-targets
-            -p vortex-cuda -p vortex-cub -p vortex-nvcomp
-            -p gpu-scan-cli -p vortex-test-e2e-cuda
-      - name: Clippy CUDA crates
-        run: |
-          cargo clippy --profile ci --locked --all-features --all-targets \
-            -p vortex-cuda \
-            -p vortex-cub \
-            -p vortex-nvcomp \
-            -p gpu-scan-cli \
-            -p vortex-test-e2e-cuda \
-            -- -D warnings
-
-  cuda-test:
-    if: github.repository == 'vortex-data/vortex'
-    name: "CUDA tests"
-    timeout-minutes: 30
-    runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-tests
-    steps:
-      - uses: runs-on/action@v2
-        with:
-          sccache: s3
-      - name: Display NVIDIA SMI details
-        run: |
-          nvidia-smi
-          nvidia-smi -L
-          nvidia-smi -q -d Memory
-      - uses: actions/checkout@v6
-      - uses: ./.github/actions/setup-rust
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Install nextest
-        uses: taiki-e/install-action@v2
-        with:
-          tool: nextest
-      - name: Rust Tests
-        env:
-          FLAT_LAYOUT_INLINE_ARRAY_NODE: true
-        run: |
-          cargo nextest run \
-            --cargo-profile ci \
-            --locked \
-            -p vortex-file \
-            -p vortex-cuda \
-            -p vortex-cub \
-            -p vortex-nvcomp \
-            -p vortex-test-e2e-cuda \
-            --all-features \
-            --no-fail-fast \
-            --target x86_64-unknown-linux-gnu \
-            --verbose
-
-  cuda-test-sanitizer:
-    if: github.repository == 'vortex-data/vortex'
-    name: "CUDA tests (${{ matrix.sanitizer }})"
-    timeout-minutes: 30
-    runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-sanitizer
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - sanitizer: memcheck
-            runner_flags: "--tool memcheck --leak-check=full --error-exitcode 1"
-          # TODO(joe): try to re-enable racecheck, it is hanging in CI.
-          # - sanitizer: racecheck
-          #   runner_flags: "--tool racecheck --error-exitcode 1"
-          - sanitizer: synccheck
-            runner_flags: "--tool synccheck --error-exitcode 1"
-          - sanitizer: initcheck
-            runner_flags: "--tool initcheck --error-exitcode 1"
-    steps:
-      - uses: runs-on/action@v2
-        with:
-          sccache: s3
-      - name: Display NVIDIA SMI details
-        run: |
-          nvidia-smi
-          nvidia-smi -L
-          nvidia-smi -q -d Memory
-      - uses: actions/checkout@v6
-      - uses: ./.github/actions/setup-rust
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Build tests
-        run: cargo test --profile ci --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu --no-run
-      - name: "CUDA - ${{ matrix.sanitizer }}"
-        env:
-          CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: "compute-sanitizer ${{ matrix.runner_flags }}"
-        run: cargo test --profile ci --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu
-
-  cuda-test-cudf:
-    if: github.repository == 'vortex-data/vortex'
-    name: "CUDA tests (cudf)"
-    timeout-minutes: 30
-    runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-cudf
-    steps:
-      - uses: runs-on/action@v2
-        with:
-          sccache: s3
-      - name: Display NVIDIA SMI details
-        run: |
-          nvidia-smi
-          nvidia-smi -L
-          nvidia-smi -q -d Memory
-      - uses: actions/checkout@v6
-      - uses: ./.github/actions/setup-rust
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Build cudf test library
-        run: cargo build --profile ci --locked -p vortex-test-e2e-cuda --target x86_64-unknown-linux-gnu
-      - name: Download and run cudf-test-harness
-        run: |
-          curl -fsSL https://github.com/vortex-data/cudf-test-harness/releases/latest/download/cudf-test-harness-x86_64.tar.gz | tar -xz
-          cd cudf-test-harness-x86_64
-          compute-sanitizer --tool memcheck --error-exitcode 1 ./cudf-test-harness check $GITHUB_WORKSPACE/target/x86_64-unknown-linux-gnu/ci/libvortex_test_e2e_cuda.so
-
   rust-test-other:
     name: "Rust tests (${{ matrix.os }})"
     timeout-minutes: 30
@@ -758,29 +450,6 @@ jobs:
       - run: wasmer run ./target/wasm32-wasip1/ci/wasm-test.wasm
         working-directory: ./wasm-test
 
-  miri:
-    name: "Rust tests (miri)"
-    runs-on: >-
-      ${{ github.repository == 'vortex-data/vortex'
-          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-miri', github.run_id)
-          || 'ubuntu-latest' }}
-    timeout-minutes: 30
-    env:
-      MIRIFLAGS: -Zmiri-strict-provenance -Zmiri-symbolic-alignment-check -Zmiri-disable-isolation -Zmiri-env-forward=RUST_BACKTRACE
-      RUSTFLAGS: "-A warnings"
-      RUST_BACKTRACE: full
-    steps:
-      - uses: runs-on/action@v2
-        if: github.repository == 'vortex-data/vortex'
-        with:
-          sccache: s3
-      - uses: actions/checkout@v6
-      - uses: ./.github/actions/setup-prebuild
-      - name: Install nightly with miri
-        run: rustup toolchain install $NIGHTLY_TOOLCHAIN --component rust-src,rustfmt,clippy,miri
-      - name: Run Miri
-        run: cargo +$NIGHTLY_TOOLCHAIN miri nextest run --no-fail-fast -p vortex-buffer -p vortex-ffi
-
   generated-files:
     name: "Check generated source files are up to date"
     runs-on: >-