Merge remote-tracking branch 'origin/develop' into claude/admiring-lichterman

Author: lwwmanning

.github/scripts/fuzz_report/cli.py

@@ -363,8 +363,9 @@ def cmd_report(args: argparse.Namespace) -> int:
         print(f"Commented on #{existing_issue}", file=sys.stderr)
         _write_github_output("issue_number", str(existing_issue))
     else:
-        fuzz_target = variables.get("FUZZ_TARGET", "unknown")
-        title = f"Fuzzing Crash: {crash_info.error_variant} in {fuzz_target}"
+        # Use FUZZ_NAME for the title (descriptive name), fall back to FUZZ_TARGET
+        fuzz_name = variables.get("FUZZ_NAME") or variables.get("FUZZ_TARGET", "unknown")
+        title = f"Fuzzing Crash: {crash_info.error_variant} in {fuzz_name}"
 
         body = render_template(str(TEMPLATES_DIR / "new_issue.md"), variables, use_env=False)
         body_file = Path("issue_body.md")

.github/workflows/ci.yml

@@ -379,51 +379,132 @@ jobs:
           flags: ${{ matrix.suite }}
           use_oidc: true
 
-  rust-test:
-    name: "Rust tests (sanitizer)"
-    timeout-minutes: 40
+  rust-test-sanitizer:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - sanitizer: asan
+            sanitizer_flags: "-Zsanitizer=address -Zsanitize=leak"
+          - sanitizer: msan
+            sanitizer_flags: "-Zsanitizer=memory"
+          - sanitizer: tsan
+            sanitizer_flags: "-Zsanitizer=thread"
+    name: "Rust tests (${{ matrix.sanitizer }})"
     runs-on: >-
       ${{ github.repository == 'vortex-data/vortex'
           && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-test-sanitizer', github.run_id)
           || 'ubuntu-latest' }}
+    timeout-minutes: 40
     env:
-      # Add debug symbols and enable ASAN/LSAN with better output
-      ASAN_OPTIONS: "symbolize=1:print_stats=1:check_initialization_order=1:detect_leaks=1:halt_on_error=0:verbosity=1:leak_check_at_exit=1"
-      LSAN_OPTIONS: "verbosity=1:report_objects=1"
+      ASAN_OPTIONS: "symbolize=1:check_initialization_order=1:detect_leaks=1:leak_check_at_exit=1"
+      LSAN_OPTIONS: "report_objects=1"
       ASAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
-      # Link against DuckDB debug build
-      VX_DUCKDB_DEBUG: "1"
-      # Keep frame pointers for better stack traces
-      CARGO_PROFILE_DEV_DEBUG: "true"
-      CARGO_PROFILE_TEST_DEBUG: "true"
-      # Skip slow tests that are too expensive under sanitizer
+      MSAN_OPTIONS: "symbolize=1"
+      MSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
+      TSAN_OPTIONS: "symbolize=1"
+      TSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
       VORTEX_SKIP_SLOW_TESTS: "1"
+      # -Cunsafe-allow-abi-mismatch=sanitizer: libraries like compiler_builtins
+      # unset -Zsanitizer flag and we should allow that.
+      RUSTFLAGS: "-A warnings -Cunsafe-allow-abi-mismatch=sanitizer --cfg disable_loom --cfg vortex_nightly -C debuginfo=2 -C opt-level=0 -C strip=none"
     steps:
       - uses: runs-on/action@v2
         if: github.repository == 'vortex-data/vortex'
         with:
           sccache: s3
       - uses: actions/checkout@v6
       - uses: ./.github/actions/setup-prebuild
-      - name: Install nightly for sanitizer
+      - name: Install Rust nightly toolchain
         run: |
           rustup toolchain install $NIGHTLY_TOOLCHAIN
           rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview
-      - name: Rust Tests
-        env:
-          RUSTFLAGS: "-A warnings -Zsanitizer=address -Zsanitizer=leak --cfg disable_loom --cfg vortex_nightly -C debuginfo=2 -C opt-level=0 -C strip=none"
+          export RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}"
+      - name: Build tests with sanitizer
         run: |
-          # Build with full debug info first (helps with caching)
-          cargo +$NIGHTLY_TOOLCHAIN build --locked --all-features  \
-            --target x86_64-unknown-linux-gnu \
-            -p vortex-buffer -p vortex-ffi -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array
-          # Run tests with sanitizers and debug output
-          cargo +$NIGHTLY_TOOLCHAIN nextest run \
-            --locked \
-            --all-features \
-            --no-fail-fast \
-            --target x86_64-unknown-linux-gnu \
-            -p vortex-buffer -p vortex-ffi -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array
+          cargo +$NIGHTLY_TOOLCHAIN build --locked --all-features \
+            --target x86_64-unknown-linux-gnu -Zbuild-std \
+            -p vortex-buffer -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array
+
+      - name: Run tests with sanitizer
+        run: |
+          cargo +$NIGHTLY_TOOLCHAIN nextest run --locked --all-features \
+            --target x86_64-unknown-linux-gnu --no-fail-fast -Zbuild-std \
+            -p vortex-buffer -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array
+
+        # vortex-ffi requires --no-default-features as otherwise we pull in
+        # Mimalloc which interferes with sanitizers
+        # cargo nextest reports less sanitizer issues than cargo test
+        # TODO(myrrc): remove --no-default-features once we make Mimalloc opt-in
+      - name: Run vortex-ffi tests with sanitizer
+        run: |
+          cargo +$NIGHTLY_TOOLCHAIN test --locked --no-default-features \
+            --target x86_64-unknown-linux-gnu --no-fail-fast -Zbuild-std \
+            -p vortex-ffi -- --no-capture
+
+  rust-ffi-test-sanitizer:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # We don't run memory sanitizer as it's clang-only and provides many
+          # false positives for Catch2
+          - sanitizer: asan
+            sanitizer_flags: "-Zsanitizer=address -Zsanitize=leak"
+          - sanitizer: tsan
+            sanitizer_flags: "-Zsanitizer=thread"
+    name: "Rust/C++ FFI tests (${{ matrix.sanitizer }})"
+    timeout-minutes: 40
+    env:
+      ASAN_OPTIONS: "symbolize=1:check_initialization_order=1:detect_leaks=1:leak_check_at_exit=1"
+      LSAN_OPTIONS: "report_objects=1"
+      ASAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
+      MSAN_OPTIONS: "symbolize=1"
+      MSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
+      TSAN_OPTIONS: "symbolize=1"
+      TSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
+      VORTEX_SKIP_SLOW_TESTS: "1"
+      # -Cunsafe-allow-abi-mismatch=sanitizer: libraries like compiler_builtins
+      # unset -Zsanitizer flag and we should allow that.
+    runs-on: >-
+      ${{ github.repository == 'vortex-data/vortex'
+          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-ffi-test-sanitizer', github.run_id)
+          || 'ubuntu-latest' }}
+    steps:
+      - uses: runs-on/action@v2
+        if: github.repository == 'vortex-data/vortex'
+        with:
+          sccache: s3
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/setup-prebuild
+      - name: Install rustfilt
+        run: |
+          cargo install rustfilt
+      - name: Install Rust nightly toolchain
+        run: |
+          rustup toolchain install $NIGHTLY_TOOLCHAIN
+          rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview
+
+          # Export flags here so that rustfilt won't be built with sanitizers
+          export RUSTFLAGS="-A warnings -Cunsafe-allow-abi-mismatch=sanitizer \
+                --cfg disable_loom --cfg vortex_nightly -C debuginfo=2 \
+                -C opt-level=0 -C strip=none -Zexternal-clangrt \
+                ${{ matrix.sanitizer_flags }}"
+      - name: Build FFI library
+        run: |
+          # TODO(myrrc): remove --no-default-features
+          cargo +$NIGHTLY_TOOLCHAIN build --locked --no-default-features \
+            --target x86_64-unknown-linux-gnu -Zbuild-std \
+            -p vortex-ffi
+      - name: Build FFI library tests
+        run: |
+          cd vortex-ffi
+          cmake -Bbuild -DBUILD_TESTS=1 -DSANITIZER=${{ matrix.sanitizer }} -DTARGET_TRIPLE="x86_64-unknown-linux-gnu"
+          cmake --build build -j
+      - name: Run tests
+        run: |
+          set -o pipefail
+          ./vortex-ffi/build/test/vortex_ffi_test 2>&1 | rustfilt -i-
 
   cuda-build-lint:
     if: github.repository == 'vortex-data/vortex'

.github/workflows/docs.yml

@@ -49,7 +49,7 @@ jobs:
           mkdir -p docs/_static/vortex-jni
           mkdir -p docs/_static/vortex-spark
           cp -r java/vortex-jni/build/docs/javadoc/* docs/_static/vortex-jni/
-          cp -r java/vortex-spark/build/docs/javadoc/* docs/_static/vortex-spark/
+          cp -r java/vortex-spark/build/vortex-spark_2.13/docs/javadoc/* docs/_static/vortex-spark/
       - name: build Python and Rust docs
         run: |
           uv run --all-packages make -C docs html

.github/workflows/fuzz.yml

@@ -106,7 +106,8 @@ jobs:
     name: "Array Operations Fuzz (unstable)"
     uses: ./.github/workflows/run-fuzzer.yml
     with:
-      fuzz_target: array_ops_unstable_encodings
+      fuzz_target: array_ops
+      fuzz_name: array_ops_unstable_encodings
       extra_features: "vortex/unstable_encodings"
       jobs: 16
     secrets:
@@ -124,7 +125,8 @@ jobs:
       pull-requests: read
     uses: ./.github/workflows/report-fuzz-crash.yml
     with:
-      fuzz_target: array_ops_unstable_encodings
+      fuzz_target: array_ops
+      fuzz_name: array_ops_unstable_encodings
       crash_file: ${{ needs.ops_fuzz_unstable.outputs.first_crash_name }}
       artifact_url: ${{ needs.ops_fuzz_unstable.outputs.artifact_url }}
       artifact_name: array_ops_unstable_encodings-crash-artifacts

.github/workflows/report-fuzz-crash.yml

@@ -6,6 +6,9 @@ on:
       fuzz_target:
         required: true
         type: string
+      fuzz_name:
+        required: false
+        type: string
       crash_file:
         required: true
         type: string
@@ -148,6 +151,7 @@ jobs:
             --dedup-result dedup_result.json \
             --claude-analysis claude_analysis.txt \
             -v "FUZZ_TARGET=${{ inputs.fuzz_target }}" \
+            -v "FUZZ_NAME=${{ inputs.fuzz_name || inputs.fuzz_target }}" \
             -v "CRASH_FILE=${{ inputs.crash_file }}" \
             -v "BRANCH=${{ inputs.branch }}" \
             -v "COMMIT=${{ inputs.commit }}" \

.github/workflows/run-fuzzer.yml

@@ -4,9 +4,14 @@ on:
   workflow_call:
     inputs:
       fuzz_target:
-        description: "The cargo fuzz target name (e.g., file_io, array_ops, compress_roundtrip)"
+        description: "The cargo fuzz binary target name (e.g., file_io, array_ops)"
         required: true
         type: string
+      fuzz_name:
+        description: "Display/storage name for this fuzz run. Defaults to fuzz_target if not set."
+        required: false
+        type: string
+        default: ""
       max_time:
         description: "Maximum fuzzing time in seconds"
         required: false
@@ -48,11 +53,13 @@ env:
 
 jobs:
   fuzz:
-    name: "Run ${{ inputs.fuzz_target }}"
+    name: "Run ${{ inputs.fuzz_name || inputs.fuzz_target }}"
+    env:
+      FUZZ_NAME: ${{ inputs.fuzz_name || inputs.fuzz_target }}
     timeout-minutes: 230  # almost 4 hours
     runs-on: >-
       ${{ github.repository == 'vortex-data/vortex'
-          && format('runs-on={0}/runner={1}/disk=large/tag={2}-fuzz', github.run_id, inputs.runner, inputs.fuzz_target)
+          && format('runs-on={0}/runner={1}/disk=large/tag={2}-fuzz', github.run_id, inputs.runner, inputs.fuzz_name || inputs.fuzz_target)
           || 'ubuntu-latest' }}
     outputs:
       crashes_found: ${{ steps.check.outputs.crashes_found }}
@@ -89,8 +96,8 @@ jobs:
           AWS_REGION: "us-east-1"
           AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com"
         run: |
-          CORPUS_KEY="${{ inputs.fuzz_target }}_corpus.tar.zst"
-          CORPUS_DIR="fuzz/corpus/${{ inputs.fuzz_target }}"
+          CORPUS_KEY="${FUZZ_NAME}_corpus.tar.zst"
+          CORPUS_DIR="fuzz/corpus/${FUZZ_NAME}-${{ inputs.extra_features }}"
 
           # Try to download corpus
           if python3 scripts/s3-download.py "s3://vortex-fuzz-corpus/$CORPUS_KEY" "$CORPUS_KEY"; then
@@ -163,15 +170,15 @@ jobs:
         if: steps.check.outputs.crashes_found == 'true'
         uses: actions/upload-artifact@v7
         with:
-          name: ${{ inputs.fuzz_target }}-crash-artifacts
+          name: ${{ env.FUZZ_NAME }}-crash-artifacts
           path: fuzz/artifacts
           retention-days: 180
 
       - name: Archive fuzzer output log
         if: steps.check.outputs.crashes_found == 'true'
         uses: actions/upload-artifact@v7
         with:
-          name: ${{ inputs.fuzz_target }}-logs
+          name: ${{ env.FUZZ_NAME }}-logs
           path: fuzz_output.log
           retention-days: 90
 
@@ -183,8 +190,8 @@ jobs:
           AWS_REGION: "us-east-1"
           AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com"
         run: |
-          CORPUS_KEY="${{ inputs.fuzz_target }}_corpus.tar.zst"
-          CORPUS_DIR="fuzz/corpus/${{ inputs.fuzz_target }}"
+          CORPUS_KEY="${FUZZ_NAME}_corpus.tar.zst"
+          CORPUS_DIR="fuzz/corpus/${FUZZ_NAME}"
 
           tar -acf "$CORPUS_KEY" "$CORPUS_DIR"
 

.github/workflows/web.yml

@@ -0,0 +1,63 @@
+---
+
+name: Vortex Web
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+  push:
+    branches: [develop]
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    working-directory: vortex-web
+
+jobs:
+  changes:
+    name: Detect Changes
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: read
+    outputs:
+      web: ${{ steps.filter.outputs.web }}
+    steps:
+      - uses: actions/checkout@v6
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            web:
+              - "vortex-web/**"
+              - ".github/workflows/web.yml"
+
+  build:
+    name: Build & Check
+    needs: [changes]
+    if: needs.changes.outputs.web == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/setup-rust
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          enable-sccache: "false"
+      - name: Install wasm-pack
+        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
+        working-directory: .
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+          cache: "npm"
+          cache-dependency-path: vortex-web/package-lock.json
+      - run: npm ci
+      - run: npm run wasm
+      - run: npm run lint
+      - run: npm run typecheck
+      - run: npm run build-storybook