Merge remote-tracking branch 'origin/develop' into ji/slots-for-arrays

Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>

Author: joseph-isaacs

.github/workflows/ci.yml

@@ -386,10 +386,10 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          # We don't run memory sanitizer as it provides many false positives
+          # for std
           - sanitizer: asan
-            sanitizer_flags: "-Zsanitizer=address -Zsanitize=leak"
-          - sanitizer: msan
-            sanitizer_flags: "-Zsanitizer=memory"
+            sanitizer_flags: "-Zsanitizer=address,leak"
           - sanitizer: tsan
             sanitizer_flags: "-Zsanitizer=thread"
     name: "Rust tests (${{ matrix.sanitizer }})"
@@ -421,15 +421,16 @@ jobs:
         run: |
           rustup toolchain install $NIGHTLY_TOOLCHAIN
           rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview
-          export RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}"
       - name: Build tests with sanitizer
         run: |
+          RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \
           cargo +$NIGHTLY_TOOLCHAIN build --locked --all-features \
             --target x86_64-unknown-linux-gnu -Zbuild-std \
             -p vortex-buffer -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array
 
       - name: Run tests with sanitizer
         run: |
+          RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \
           cargo +$NIGHTLY_TOOLCHAIN nextest run --locked --all-features \
             --target x86_64-unknown-linux-gnu --no-fail-fast -Zbuild-std \
             -p vortex-buffer -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array
@@ -440,6 +441,7 @@ jobs:
         # TODO(myrrc): remove --no-default-features once we make Mimalloc opt-in
       - name: Run vortex-ffi tests with sanitizer
         run: |
+          RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \
           cargo +$NIGHTLY_TOOLCHAIN test --locked --no-default-features \
             --target x86_64-unknown-linux-gnu --no-fail-fast -Zbuild-std \
             -p vortex-ffi -- --no-capture
@@ -452,7 +454,7 @@ jobs:
           # We don't run memory sanitizer as it's clang-only and provides many
           # false positives for Catch2
           - sanitizer: asan
-            sanitizer_flags: "-Zsanitizer=address -Zsanitize=leak"
+            sanitizer_flags: "-Zsanitizer=address,leak"
           - sanitizer: tsan
             sanitizer_flags: "-Zsanitizer=thread"
     name: "Rust/C++ FFI tests (${{ matrix.sanitizer }})"
@@ -486,15 +488,13 @@ jobs:
         run: |
           rustup toolchain install $NIGHTLY_TOOLCHAIN
           rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview
-
-          # Export flags here so that rustfilt won't be built with sanitizers
-          export RUSTFLAGS="-A warnings -Cunsafe-allow-abi-mismatch=sanitizer \
-                --cfg disable_loom --cfg vortex_nightly -C debuginfo=2 \
-                -C opt-level=0 -C strip=none -Zexternal-clangrt \
-                ${{ matrix.sanitizer_flags }}"
       - name: Build FFI library
         run: |
           # TODO(myrrc): remove --no-default-features
+          RUSTFLAGS="-A warnings -Cunsafe-allow-abi-mismatch=sanitizer \
+                --cfg disable_loom --cfg vortex_nightly -C debuginfo=2 \
+                -C opt-level=0 -C strip=none -Zexternal-clangrt \
+                ${{ matrix.sanitizer_flags }}" \
           cargo +$NIGHTLY_TOOLCHAIN build --locked --no-default-features \
             --target x86_64-unknown-linux-gnu -Zbuild-std \
             -p vortex-ffi

.github/workflows/compat-gen-upload.yml

@@ -28,6 +28,7 @@ jobs:
           && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=compat-gen-dry-run', github.run_id)
           || 'ubuntu-latest' }}
     permissions:
+      id-token: write
       contents: read
     outputs:
       version: ${{ steps.detect.outputs.version }}
@@ -41,6 +42,12 @@ jobs:
           fetch-depth: 0
       - uses: ./.github/actions/setup-prebuild
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v6
+        with:
+          role-to-assume: "arn:aws:iam::245040174862:role/GitHubCompatUploadRole"
+          aws-region: us-east-1
+
       - name: Detect version
         id: detect
         run: |
@@ -87,7 +94,7 @@ jobs:
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v6
         with:
-          role-to-assume: "arn:aws:iam::245040174862:role/GitHubBenchmarkRole"
+          role-to-assume: "arn:aws:iam::245040174862:role/GitHubCompatUploadRole"
           aws-region: us-east-1
 
       - name: Upload fixtures for v${{ needs.dry-run.outputs.version }}

Cargo.lock

@@ -315,6 +315,7 @@ unexpected_cfgs = { level = "deny", check-cfg = [
     "cfg(codspeed)",
     "cfg(disable_loom)",
     "cfg(vortex_nightly)",
+    'cfg(target_os, values("unknown"))',
 ] }
 warnings = "warn"
 

Cargo.toml

@@ -64,7 +64,9 @@ pub use native_runtime::RUNTIME;
 #[cfg(not(target_arch = "wasm32"))]
 pub use native_runtime::SESSION;
 
-#[cfg(target_arch = "wasm32")]
+// target_os = "unknown" matches wasm32-unknown-unknown (browser), excluding WASI targets
+// where wasm-bindgen's JS interop is not available.
+#[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
 mod wasm_runtime {
     use std::sync::LazyLock;
 
@@ -77,5 +79,20 @@ mod wasm_runtime {
         LazyLock::new(|| VortexSession::default().with_handle(WasmRuntime::handle()));
 }
 
-#[cfg(target_arch = "wasm32")]
+#[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
 pub use wasm_runtime::SESSION;
+
+/// WASI targets get a session without an async runtime handle since wasm-bindgen
+/// is not available. The fuzz workloads are synchronous, so this is sufficient.
+#[cfg(all(target_arch = "wasm32", not(target_os = "unknown")))]
+mod wasi_runtime {
+    use std::sync::LazyLock;
+
+    use vortex::VortexSessionDefault;
+    use vortex_session::VortexSession;
+
+    pub static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::default);
+}
+
+#[cfg(all(target_arch = "wasm32", not(target_os = "unknown")))]
+pub use wasi_runtime::SESSION;

fuzz/src/lib.rs

@@ -28,6 +28,7 @@ async-trait = { workspace = true }
 bytes = { workspace = true }
 cudarc = { workspace = true, features = ["f16"] }
 futures = { workspace = true, features = ["executor"] }
+itertools = { workspace = true }
 kanal = { workspace = true }
 object_store = { workspace = true, features = ["fs"] }
 parking_lot = { workspace = true }

vortex-cuda/Cargo.toml

@@ -40,8 +40,8 @@ use vortex_cuda::CudaDeviceBuffer;
 use vortex_cuda::CudaExecutionCtx;
 use vortex_cuda::CudaSession;
 use vortex_cuda::dynamic_dispatch::CudaDispatchPlan;
+use vortex_cuda::dynamic_dispatch::DispatchPlan;
 use vortex_cuda::dynamic_dispatch::MaterializedPlan;
-use vortex_cuda::dynamic_dispatch::UnmaterializedPlan;
 use vortex_cuda_macros::cuda_available;
 use vortex_cuda_macros::cuda_not_available;
 
@@ -123,13 +123,15 @@ struct BenchRunner {
 
 impl BenchRunner {
     fn new(array: &vortex::array::ArrayRef, len: usize, cuda_ctx: &CudaExecutionCtx) -> Self {
+        let plan = match DispatchPlan::new(array).vortex_expect("build_dyn_dispatch_plan") {
+            DispatchPlan::Fused(plan) => plan,
+            _ => unreachable!("encoding not fusable"),
+        };
         let MaterializedPlan {
             dispatch_plan,
             device_buffers,
             shared_mem_bytes,
-        } = UnmaterializedPlan::new(array)
-            .and_then(|p| p.materialize(cuda_ctx))
-            .vortex_expect("build_dyn_dispatch_plan");
+        } = plan.materialize(cuda_ctx).vortex_expect("materialize plan");
 
         let device_plan = Arc::new(
             cuda_ctx