Merge remote-tracking branch 'origin/develop' into ji/slots-for-arrays

Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>

# Conflicts:
#	encodings/alp/public-api.lock
#	encodings/alp/src/alp/array.rs
#	encodings/alp/src/alp_rd/array.rs
#	encodings/bytebool/public-api.lock
#	encodings/datetime-parts/public-api.lock
#	encodings/decimal-byte-parts/public-api.lock
#	encodings/fastlanes/public-api.lock
#	encodings/fsst/public-api.lock
#	encodings/pco/public-api.lock
#	encodings/runend/public-api.lock
#	encodings/sequence/public-api.lock
#	encodings/sparse/public-api.lock
#	encodings/zigzag/public-api.lock
#	encodings/zstd/public-api.lock
#	encodings/zstd/src/zstd_buffers.rs
#	vortex-array/public-api.lock
#	vortex-array/src/arrays/chunked/vtable/mod.rs
#	vortex-array/src/arrays/dict/vtable/mod.rs
#	vortex-array/src/arrays/filter/vtable.rs
#	vortex-array/src/arrays/shared/vtable.rs
#	vortex-array/src/arrays/slice/vtable.rs
#	vortex-array/src/arrays/variant/mod.rs
#	vortex-array/src/arrays/variant/vtable/mod.rs
#	vortex-array/src/executor.rs
#	vortex-array/src/vtable/mod.rs

Author: joseph-isaacs

.github/scripts/fuzz_report/cli.py

@@ -363,8 +363,9 @@ def cmd_report(args: argparse.Namespace) -> int:
         print(f"Commented on #{existing_issue}", file=sys.stderr)
         _write_github_output("issue_number", str(existing_issue))
     else:
-        fuzz_target = variables.get("FUZZ_TARGET", "unknown")
-        title = f"Fuzzing Crash: {crash_info.error_variant} in {fuzz_target}"
+        # Use FUZZ_NAME for the title (descriptive name), fall back to FUZZ_TARGET
+        fuzz_name = variables.get("FUZZ_NAME") or variables.get("FUZZ_TARGET", "unknown")
+        title = f"Fuzzing Crash: {crash_info.error_variant} in {fuzz_name}"
 
         body = render_template(str(TEMPLATES_DIR / "new_issue.md"), variables, use_env=False)
         body_file = Path("issue_body.md")

.github/workflows/bench-dispatch.yml

@@ -0,0 +1,49 @@
+# Dispatches benchmark workflows when benchmark labels are added to pull requests.
+# This is a separate workflow so that non-benchmark label events don't create
+# phantom check suites that obscure in-progress benchmark runs on the PR.
+
+name: Benchmark Dispatch
+
+on:
+  pull_request:
+    types: [labeled]
+    branches: ["develop"]
+
+permissions:
+  actions: write
+  contents: read
+  pull-requests: write  # for label removal and PR comments
+  id-token: write  # enables AWS-GitHub OIDC
+
+jobs:
+  remove-bench-label:
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    if: github.event.label.name == 'action/benchmark'
+    steps:
+      - uses: actions-ecosystem/action-remove-labels@v1
+        if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex'
+        with:
+          labels: action/benchmark
+          fail_on_error: true
+
+  bench:
+    needs: remove-bench-label
+    uses: ./.github/workflows/bench-pr.yml
+    secrets: inherit
+
+  remove-sql-label:
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    if: github.event.label.name == 'action/benchmark-sql'
+    steps:
+      - uses: actions-ecosystem/action-remove-labels@v1
+        if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex'
+        with:
+          labels: action/benchmark-sql
+          fail_on_error: true
+
+  sql-bench:
+    needs: remove-sql-label
+    uses: ./.github/workflows/sql-pr.yml
+    secrets: inherit

.github/workflows/bench-pr.yml

@@ -1,4 +1,5 @@
-# Runs all benchmarks once when we add the `action/benchmark` tag to a pull request.
+# Runs all benchmarks once for a pull request.
+# Called from bench-dispatch.yml when the `action/benchmark` label is added.
 
 name: PR Benchmarks
 
@@ -9,32 +10,16 @@ concurrency:
   cancel-in-progress: false
 
 on:
-  pull_request:
-    types: [labeled, synchronize]
-    branches: ["develop"]
+  workflow_call: { }
   workflow_dispatch: { }
 
 permissions:
-  actions: write  # for removing labels
   contents: read
   pull-requests: write  # for commenting on PRs
   id-token: write  # enables AWS-GitHub OIDC
 
 jobs:
-  label_trigger:
-    runs-on: ubuntu-latest
-    timeout-minutes: 120
-    if: ${{ contains(github.event.head_commit.message, '[benchmark]') || github.event.label.name == 'action/benchmark' && github.event_name == 'pull_request' }}
-    steps:
-      # We remove the benchmark label first so that the workflow can be re-triggered.
-      - uses: actions-ecosystem/action-remove-labels@v1
-        if: ${{ github.event.pull_request.head.repo.full_name == 'vortex-data/vortex' }}
-        with:
-          labels: action/benchmark
-          fail_on_error: true
-
   bench:
-    needs: label_trigger
     timeout-minutes: 120
     runs-on: >-
       ${{ github.repository == 'vortex-data/vortex'
@@ -48,7 +33,6 @@ jobs:
             build_args: "--features lance"
           - id: compress-bench
             name: Compression
-    if: ${{ contains(github.event.head_commit.message, '[benchmark]') || github.event.label.name == 'action/benchmark' && github.event_name == 'pull_request' }}
     steps:
       - uses: runs-on/action@v2
         if: github.event.pull_request.head.repo.fork == false
@@ -137,17 +121,16 @@ jobs:
           comment-tag: bench-pr-comment-${{ matrix.benchmark.id }}
 
       - name: Comment PR on failure
-        if: failure() && inputs.mode == 'pr' && github.event.pull_request.head.repo.fork == false
+        if: failure() && github.event.pull_request.head.repo.fork == false
         uses: thollander/actions-comment-pull-request@v3
         with:
           message: |
-            # 🚨🚨🚨❌❌❌ BENCHMARK FAILED ❌❌❌🚨🚨🚨
+            # BENCHMARK FAILED
 
             Benchmark `${{ matrix.benchmark.name }}` failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
           comment-tag: bench-pr-comment-${{ matrix.benchmark.id }}
 
   sql:
-    needs: label_trigger
     uses: ./.github/workflows/sql-benchmarks.yml
     secrets: inherit
     with:

.github/workflows/ci.yml

@@ -69,7 +69,9 @@ jobs:
       - name: Python Lint - PyRight
         env:
           MATURIN_PEP517_ARGS: "--profile dev"
-        run: uv run basedpyright vortex-python
+        run: |
+          uv sync --all-packages
+          uv run basedpyright vortex-python
 
   python-test:
     name: "Python (test)"
@@ -379,51 +381,132 @@ jobs:
           flags: ${{ matrix.suite }}
           use_oidc: true
 
-  rust-test:
-    name: "Rust tests (sanitizer)"
-    timeout-minutes: 40
+  rust-test-sanitizer:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - sanitizer: asan
+            sanitizer_flags: "-Zsanitizer=address -Zsanitize=leak"
+          - sanitizer: msan
+            sanitizer_flags: "-Zsanitizer=memory"
+          - sanitizer: tsan
+            sanitizer_flags: "-Zsanitizer=thread"
+    name: "Rust tests (${{ matrix.sanitizer }})"
     runs-on: >-
       ${{ github.repository == 'vortex-data/vortex'
           && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-test-sanitizer', github.run_id)
           || 'ubuntu-latest' }}
+    timeout-minutes: 40
     env:
-      # Add debug symbols and enable ASAN/LSAN with better output
-      ASAN_OPTIONS: "symbolize=1:print_stats=1:check_initialization_order=1:detect_leaks=1:halt_on_error=0:verbosity=1:leak_check_at_exit=1"
-      LSAN_OPTIONS: "verbosity=1:report_objects=1"
+      ASAN_OPTIONS: "symbolize=1:check_initialization_order=1:detect_leaks=1:leak_check_at_exit=1"
+      LSAN_OPTIONS: "report_objects=1"
       ASAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
-      # Link against DuckDB debug build
-      VX_DUCKDB_DEBUG: "1"
-      # Keep frame pointers for better stack traces
-      CARGO_PROFILE_DEV_DEBUG: "true"
-      CARGO_PROFILE_TEST_DEBUG: "true"
-      # Skip slow tests that are too expensive under sanitizer
+      MSAN_OPTIONS: "symbolize=1"
+      MSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
+      TSAN_OPTIONS: "symbolize=1"
+      TSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
       VORTEX_SKIP_SLOW_TESTS: "1"
+      # -Cunsafe-allow-abi-mismatch=sanitizer: libraries like compiler_builtins
+      # unset -Zsanitizer flag and we should allow that.
+      RUSTFLAGS: "-A warnings -Cunsafe-allow-abi-mismatch=sanitizer --cfg disable_loom --cfg vortex_nightly -C debuginfo=2 -C opt-level=0 -C strip=none"
     steps:
       - uses: runs-on/action@v2
         if: github.repository == 'vortex-data/vortex'
         with:
           sccache: s3
       - uses: actions/checkout@v6
       - uses: ./.github/actions/setup-prebuild
-      - name: Install nightly for sanitizer
+      - name: Install Rust nightly toolchain
         run: |
           rustup toolchain install $NIGHTLY_TOOLCHAIN
           rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview
-      - name: Rust Tests
-        env:
-          RUSTFLAGS: "-A warnings -Zsanitizer=address -Zsanitizer=leak --cfg disable_loom --cfg vortex_nightly -C debuginfo=2 -C opt-level=0 -C strip=none"
+          export RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}"
+      - name: Build tests with sanitizer
         run: |
-          # Build with full debug info first (helps with caching)
-          cargo +$NIGHTLY_TOOLCHAIN build --locked --all-features  \
-            --target x86_64-unknown-linux-gnu \
-            -p vortex-buffer -p vortex-ffi -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array
-          # Run tests with sanitizers and debug output
-          cargo +$NIGHTLY_TOOLCHAIN nextest run \
-            --locked \
-            --all-features \
-            --no-fail-fast \
-            --target x86_64-unknown-linux-gnu \
-            -p vortex-buffer -p vortex-ffi -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array
+          cargo +$NIGHTLY_TOOLCHAIN build --locked --all-features \
+            --target x86_64-unknown-linux-gnu -Zbuild-std \
+            -p vortex-buffer -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array
+
+      - name: Run tests with sanitizer
+        run: |
+          cargo +$NIGHTLY_TOOLCHAIN nextest run --locked --all-features \
+            --target x86_64-unknown-linux-gnu --no-fail-fast -Zbuild-std \
+            -p vortex-buffer -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array
+
+        # vortex-ffi requires --no-default-features as otherwise we pull in
+        # Mimalloc which interferes with sanitizers
+        # cargo nextest reports less sanitizer issues than cargo test
+        # TODO(myrrc): remove --no-default-features once we make Mimalloc opt-in
+      - name: Run vortex-ffi tests with sanitizer
+        run: |
+          cargo +$NIGHTLY_TOOLCHAIN test --locked --no-default-features \
+            --target x86_64-unknown-linux-gnu --no-fail-fast -Zbuild-std \
+            -p vortex-ffi -- --no-capture
+
+  rust-ffi-test-sanitizer:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # We don't run memory sanitizer as it's clang-only and provides many
+          # false positives for Catch2
+          - sanitizer: asan
+            sanitizer_flags: "-Zsanitizer=address -Zsanitize=leak"
+          - sanitizer: tsan
+            sanitizer_flags: "-Zsanitizer=thread"
+    name: "Rust/C++ FFI tests (${{ matrix.sanitizer }})"
+    timeout-minutes: 40
+    env:
+      ASAN_OPTIONS: "symbolize=1:check_initialization_order=1:detect_leaks=1:leak_check_at_exit=1"
+      LSAN_OPTIONS: "report_objects=1"
+      ASAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
+      MSAN_OPTIONS: "symbolize=1"
+      MSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
+      TSAN_OPTIONS: "symbolize=1"
+      TSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
+      VORTEX_SKIP_SLOW_TESTS: "1"
+      # -Cunsafe-allow-abi-mismatch=sanitizer: libraries like compiler_builtins
+      # unset -Zsanitizer flag and we should allow that.
+    runs-on: >-
+      ${{ github.repository == 'vortex-data/vortex'
+          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-ffi-test-sanitizer', github.run_id)
+          || 'ubuntu-latest' }}
+    steps:
+      - uses: runs-on/action@v2
+        if: github.repository == 'vortex-data/vortex'
+        with:
+          sccache: s3
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/setup-prebuild
+      - name: Install rustfilt
+        run: |
+          cargo install rustfilt
+      - name: Install Rust nightly toolchain
+        run: |
+          rustup toolchain install $NIGHTLY_TOOLCHAIN
+          rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview
+
+          # Export flags here so that rustfilt won't be built with sanitizers
+          export RUSTFLAGS="-A warnings -Cunsafe-allow-abi-mismatch=sanitizer \
+                --cfg disable_loom --cfg vortex_nightly -C debuginfo=2 \
+                -C opt-level=0 -C strip=none -Zexternal-clangrt \
+                ${{ matrix.sanitizer_flags }}"
+      - name: Build FFI library
+        run: |
+          # TODO(myrrc): remove --no-default-features
+          cargo +$NIGHTLY_TOOLCHAIN build --locked --no-default-features \
+            --target x86_64-unknown-linux-gnu -Zbuild-std \
+            -p vortex-ffi
+      - name: Build FFI library tests
+        run: |
+          cd vortex-ffi
+          cmake -Bbuild -DBUILD_TESTS=1 -DSANITIZER=${{ matrix.sanitizer }} -DTARGET_TRIPLE="x86_64-unknown-linux-gnu"
+          cmake --build build -j
+      - name: Run tests
+        run: |
+          set -o pipefail
+          ./vortex-ffi/build/test/vortex_ffi_test 2>&1 | rustfilt -i-
 
   cuda-build-lint:
     if: github.repository == 'vortex-data/vortex'
@@ -708,7 +791,7 @@ jobs:
           mkdir -p vortex-cxx/build
           cmake -S vortex-cxx -B vortex-cxx/build -DVORTEX_ENABLE_TESTING=ON -DVORTEX_ENABLE_ASAN=ON
           cmake --build vortex-cxx/build --parallel $(nproc)
-          ctest --test-dir vortex-cxx/build -V
+          ctest --test-dir vortex-cxx/build -j $(nproc) -V
       - name: Build and run the example in release mode
         run: |
           cmake -S vortex-cxx/examples -B vortex-cxx/examples/build -DCMAKE_BUILD_TYPE=Release
@@ -821,6 +904,31 @@ jobs:
         run: |
           find flatbuffers/ -type f -name "*.fbs" | sed 's/^flatbuffers\///' | xargs -I{} -n1 flatc -I flatbuffers.HEAD --conform-includes flatbuffers --conform flatbuffers/{} flatbuffers.HEAD/{}
 
+  ffi-c-test:
+    name: "C API test build"
+    timeout-minutes: 10
+    runs-on: >-
+      ${{ github.repository == 'vortex-data/vortex'
+          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=cxx-build', github.run_id)
+          || 'ubuntu-latest' }}
+    steps:
+      - uses: runs-on/action@v2
+        if: github.repository == 'vortex-data/vortex'
+        with:
+          sccache: s3
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/setup-prebuild
+      - name: "regenerate FFI header file"
+        run: |
+          cargo +$NIGHTLY_TOOLCHAIN build -p vortex-ffi
+      - name: Build and run C++ unit tests
+        run: |
+          cd vortex-ffi
+          mkdir build
+          cmake -Bbuild
+          cmake --build build -j $(nproc)
+          ctest --test-dir build -j $(nproc)
+
   check-java-publish-build:
     runs-on: ${{ matrix.target.runs-on }}
     container:

.github/workflows/compat-gen-upload.yml

@@ -1,6 +1,16 @@
 name: Compat Fixture Upload
 
 on:
+  workflow_call:
+    inputs:
+      git_ref:
+        description: "Git ref for version detection (e.g. v0.62.0). Defaults to HEAD."
+        required: false
+        type: string
+      confirm_upload:
+        description: "Set to 'yes' to confirm upload."
+        required: true
+        type: string
   workflow_dispatch:
     inputs:
       git_ref:

.github/workflows/docs.yml

@@ -49,7 +49,7 @@ jobs:
           mkdir -p docs/_static/vortex-jni
           mkdir -p docs/_static/vortex-spark
           cp -r java/vortex-jni/build/docs/javadoc/* docs/_static/vortex-jni/
-          cp -r java/vortex-spark/build/docs/javadoc/* docs/_static/vortex-spark/
+          cp -r java/vortex-spark/build/vortex-spark_2.13/docs/javadoc/* docs/_static/vortex-spark/
       - name: build Python and Rust docs
         run: |
           uv run --all-packages make -C docs html